diff --git "a/layer-15-part-tail-0.json" "b/layer-15-part-tail-0.json" new file mode 100644--- /dev/null +++ "b/layer-15-part-tail-0.json" @@ -0,0 +1,26086 @@ +{ + "bytes": 922818568, + "expert_reports": [ + { + "expert": 0, + "hessian": { + "down_diagonal_mean": 73.64610290527344, + "gate_up_diagonal_mean": 948.7332153320312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10585766098862692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023460412109165458, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10581776979555371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023650532513696833, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10592331500471351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02368174836903076, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1058338508098542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023638387677374858, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602966884692402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025672349792808413, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10567398557776467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02581666161363364, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10598399758704267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026271015318980045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10560115918255054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02581828664223845, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07750424573606798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05455891784338999, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07781549213663203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05175052576903214, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0776854830670935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05438632249132328, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07753065806513575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05427924126989837, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 2, + "hessian": { + "down_diagonal_mean": 75.78314208984375, + "gate_up_diagonal_mean": 936.4705810546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1043550793034986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024262101197902834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10403602145535683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024255404706332265, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416309419093069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02389326300974899, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437583829501687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02390187497859906, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10407351548231997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028706105896381166, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10427076994766282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028799858933000938, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10438219156208503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02825727259058487, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10439557868085689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028286621410717642, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07747234057442708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05664968536763653, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0773525260796903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05517666519741044, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07789999263420351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054928255653557245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07820265742385996, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053628808555060935, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 4, + "hessian": { + "down_diagonal_mean": 86.05967712402344, + "gate_up_diagonal_mean": 947.9122314453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1045030724870584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022490751736144555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10465631556867086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022046082599533588, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10462555351841395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02253649442260004, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10442747486206098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02262803537670671, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10461902785851598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02639596703880247, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452795505524382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026991182720260954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10460182326243217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026978779256880858, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10461608182403441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02691391578669031, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0771049152244732, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05659787193771445, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07685822229764165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05756104717389606, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07684694107870708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05749495431935773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07733671126008494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056108569043880056, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 6, + "hessian": { + "down_diagonal_mean": 77.39239501953125, + "gate_up_diagonal_mean": 950.6351318359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10397360839869656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022759459764923242, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10392943256899108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022465190320425862, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.104086468297888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022017319805298138, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10380446543868552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022626596552540765, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038167490418601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02897769099721696, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10420263387183193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028480977527622554, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10379593889405539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028660872661013748, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10376428242815394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02866078131987119, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07608179342169154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05846646879888999, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07658300917428967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0568250454924828, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07609644922492505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057436350395184105, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07619596562618058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057972180511943465, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 8, + "hessian": { + "down_diagonal_mean": 62.89159393310547, + "gate_up_diagonal_mean": 893.7671508789062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10648204613381776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017182051108537683, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10657973032770389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01721933237794977, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10649192228087831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016810786955078625, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10651182101086912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017118539405717823, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10650106386325758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021337888052064123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10651275975300845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02135094206729743, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1065094705740878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020964028061953723, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10634970257583284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021103377456601825, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08121138712776864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047677504085336685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08045217708367783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04899207216597304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08166778220527608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046244903822005794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08231054194830666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047018831101563484, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 10, + "hessian": { + "down_diagonal_mean": 85.2914810180664, + "gate_up_diagonal_mean": 888.8673706054688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1069344651624511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01319786630396069, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683088111222802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01354059958828921, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10711854216676538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013677142019301489, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10704508638524513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014004549176886991, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10714359785266342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01761605701006758, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10712709770995862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01754986522749042, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1071569701050785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01760435805543952, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683203912505197, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017699793481302524, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08533466922633699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03998570256478793, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08460626043311464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0424555570920433, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08419077243245054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04412150957241732, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08514998812763241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04255907713596879, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 12, + "hessian": { + "down_diagonal_mean": 119.2367935180664, + "gate_up_diagonal_mean": 945.0033569335938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10622231344819176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01718444778275323, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1062217029599647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01646036805454959, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10641729023305703, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016921103521461374, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10631879950916692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0166781544607295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10639950406145111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02212382300646207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10610792233117773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021596609330372406, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10635717747921214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022169380275996405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10617101231396867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022036200254651212, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08522940191787322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.038730152325865706, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0868369245500959, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.02843661345708367, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08518758168380589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04022648637585785, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0851501453217604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03901427415398626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 14, + "hessian": { + "down_diagonal_mean": 78.808837890625, + "gate_up_diagonal_mean": 946.6099853515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1126974387017849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02052537423877877, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1128233569887171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020642040484337314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11276240315484277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02018169221726166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11277027277255364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020622110371679055, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11279398601084024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024063935265454372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11265089172538703, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02434376574723722, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11252965488381377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024002915392113047, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11277247684591482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0246365943604443, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07937584079927185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05239216562253785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07911124278838023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052666135343158636, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07951026815868897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05053622225968782, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07877323074251942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053258527598693234, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 16, + "hessian": { + "down_diagonal_mean": 97.90715026855469, + "gate_up_diagonal_mean": 945.9564819335938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436635802303741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019980459364531102, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10407116618654194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020353336105720075, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1040574813608057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019970841380545286, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10396786026606966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020792542950173354, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10402225404892453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025895518335720034, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10406352092708283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024972346569903693, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10379643647620514, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02556015877605, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10396769249018317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025963855953787977, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07795065380572067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05517690214419814, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0785668384296653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05506177694098417, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07756196098381378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05585862781263011, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07793793324172778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055491811186188025, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 18, + "hessian": { + "down_diagonal_mean": 59.14828872680664, + "gate_up_diagonal_mean": 933.5765380859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10539998921622992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02408171258284815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10547563709913883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02428836537131916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10546678919412876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024803919491360484, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10548391218718804, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024086670107874995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10537159817128196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02918428311276171, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10546083280941687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029399358792464643, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1054314983795433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02914926818767121, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521989544970882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029302536341881742, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07584730068365614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056841397529796056, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07551844302431823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05893591118578817, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07564010293059575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05894162327060729, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07583436646210923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05663730155545568, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 20, + "hessian": { + "down_diagonal_mean": 121.9405746459961, + "gate_up_diagonal_mean": 919.6381225585938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10855487834196047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017049940886127112, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10850455082102965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017092090051822103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1082629651263111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01641171967284202, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10862672011581516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016508779779953534, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1085440266270194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01910994117586581, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1085012950320483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019316232977470957, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10848344919891659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019258894174298862, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10835750245190195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01960773867845605, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0832671573598493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05031233245438692, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08316656512908455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04974421846529619, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0839097075647754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04938776634287536, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08358628643128267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05039680212037164, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 22, + "hessian": { + "down_diagonal_mean": 78.50912475585938, + "gate_up_diagonal_mean": 942.787109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1054772967298195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025427490941998393, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.1053865816851695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025119500239136304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10548089971369909, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02533038161536751, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1053531454358726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024983710593246687, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10551511392761077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029067824591455738, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10535467065632823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029650360763278767, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10545797404277933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02924457862193808, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10541827036973873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02908517338532899, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0774306993003623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05356928608865639, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07741894479213972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054438231390067766, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07827143458109864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053211888803777745, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07835494207205988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052683832646196, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 24, + "hessian": { + "down_diagonal_mean": 95.2047119140625, + "gate_up_diagonal_mean": 942.4881591796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10524623818413313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02134474511566068, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10535102138221268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021618734945872874, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10517844527859198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021687796623118955, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10531977892332635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02131226152913416, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10529308820853812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025576851137084803, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10519210354405004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025883587177252847, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1053022750675099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025906413163984524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10507256744498106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02544900914864373, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0780070301328779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05354713749785415, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07893375067161644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05338924168905048, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.07917063356429081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05076553939415934, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07937538546087196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05187640650798767, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 26, + "hessian": { + "down_diagonal_mean": 103.26248168945312, + "gate_up_diagonal_mean": 917.4208374023438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10602828588554866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017309188224288362, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10583974678555491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017580341314590964, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10595238853594093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017509578577659612, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10582039273409422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01694852469315568, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10577940861914716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018699183228366154, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10575247078446698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0182768359065971, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10565564823855687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01997415024697128, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10593985928605378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018779224910076314, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0832555107174228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04924073600565532, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.082795539565446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04783031928791994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08264046576799969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049885977942604125, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08308893517964057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047878789266966086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 28, + "hessian": { + "down_diagonal_mean": 93.23096466064453, + "gate_up_diagonal_mean": 939.2559814453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10501711653740795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0211096964988426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10477792999963963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021250984326847557, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10492391778712616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02183413368497782, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500037312065275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02137986546803718, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10474014428693029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02573724615957051, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498059433866082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02525998929592362, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10500972914059373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025728871043816437, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10490436541264342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025731031081204583, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08017113117719457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05196908498246129, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08001583244771011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05040605255130618, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07975469393537091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051471418578455004, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07944535363628057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05312014918602815, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 30, + "hessian": { + "down_diagonal_mean": 69.0694351196289, + "gate_up_diagonal_mean": 934.0906372070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10642857889675249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024119999440887812, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10657307559664346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024130076744679662, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065341581650475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02418954753618872, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10652948998791462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024194868260890526, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10643798350688281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028224592165266973, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1066479811231777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028079564540601374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10646749865418823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02790494005348807, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10658508522513863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027992269205185914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07690020798942285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05709688966473019, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07718719871025435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05598949437725648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.076937444562284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0564096342880967, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07676760188295251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05717215521121496, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 32, + "hessian": { + "down_diagonal_mean": 78.0974349975586, + "gate_up_diagonal_mean": 947.8861083984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10799901488641656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02202147613602292, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10791312063054066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021919270716402876, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1078861231333153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02170839813039736, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10784329890770145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022062303632185636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10773495011419132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02629270697776781, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1078233169291686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025893844854982077, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10769579772507243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026006359977207967, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10764678473092333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02612866397493974, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07913421773054231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05406523471604477, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07921075103639573, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05137962637895455, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07899534390019129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05435642111466777, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07866743125878155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05361010291219537, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 34, + "hessian": { + "down_diagonal_mean": 112.76305389404297, + "gate_up_diagonal_mean": 937.5656127929688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10545459743458997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017905849464886183, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10526950622853574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017807653057772772, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10533133042909554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018239161329931673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1054387487225292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018068610772734826, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10545747352099864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020872151461910907, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10538052444947758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020086226544143763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1052139031656241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020859953761420576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10537528836978818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020546963182626964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07888343833120598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0527106898348827, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0791116978471639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05152670719555319, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07873854556583378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05325920714388015, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07875248233639139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05290097891275108, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 36, + "hessian": { + "down_diagonal_mean": 70.77516174316406, + "gate_up_diagonal_mean": 941.1778564453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10673153054021318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023950726748442244, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10645114109186837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024346671894166335, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10664606616511517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024021132998436417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10659817819225041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024012853097223584, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10667422483510397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028767878720752076, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10653711728870298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02887844700210764, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10655866349548547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02845411767060812, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10653027988409419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028544462058364427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07763408263602072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05379683668824353, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07625829233234707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058073682207251835, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07732084312396198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05072204146779417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07727844130764666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05439251358587019, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 38, + "hessian": { + "down_diagonal_mean": 50.883460998535156, + "gate_up_diagonal_mean": 929.752197265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048685311149183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026064317980255745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10494421832907505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026544027661922136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10491877437171049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026274166153578592, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10484800226860042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026900674352331334, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10502010545500462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03290672499764716, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10490114797983567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03249957425753467, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10496386259509459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03213342433953223, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10489492470965266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032489550853502504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0745364408104786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06040270953796302, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07478198624815023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05876777485244167, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0743356169008105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06030867027863597, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07476107945520687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059903707325364666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 40, + "hessian": { + "down_diagonal_mean": 88.45849609375, + "gate_up_diagonal_mean": 946.2593383789062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10513578846415415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023209062722879353, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10546590014749786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023383860450103246, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10532423899834112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023452423900274716, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10537489522754656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02313556227421427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10535699701371076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027104367247943215, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10526319327989006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02666980623161076, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10527795695636899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026271330368732578, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1052126652080874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026508503849389718, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07778863324313902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05612459082083534, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07754682402249798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05560470132790589, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07808426048693469, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053520493348525065, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07859942625863271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05381522661783243, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 42, + "hessian": { + "down_diagonal_mean": 120.23934173583984, + "gate_up_diagonal_mean": 934.3271484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10809853274434467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017996438137224572, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10792407128750103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01850096424633567, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10802842188253539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01762939791703249, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10789123469399939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017868599918222663, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10783672384190286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020286755726371918, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10777020970644999, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021367691583996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1079748392966692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020833066689813638, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1078521737540124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021145727999068897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08257451987558663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04929214702691156, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08122217994030678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05116486040637199, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.08201235863213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050754297603527856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08203245368759342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04852121476555458, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 44, + "hessian": { + "down_diagonal_mean": 86.867431640625, + "gate_up_diagonal_mean": 948.486083984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10309043426884174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02642288389565189, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1030769338751592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025953310957169184, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10293418279946143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02566446353725064, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1028956487886161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025982583460724507, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10287725909159323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03125050943008918, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1030866706994778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030146042868175998, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10292260015382129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029843946283582674, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10304228748334089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03085000049811088, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07638628332854684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05555757028849689, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0768589790896129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053835857698763784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07727072518231509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04775729380882393, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07740409329278021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051118856458080064, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 46, + "hessian": { + "down_diagonal_mean": 99.30596923828125, + "gate_up_diagonal_mean": 935.0555419921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10629151448905202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02387595978601023, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642856535027963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024221739052103335, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10659845635305651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02408972321776838, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10635703862000039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024302648610014372, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10645443548908129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02743516831970205, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642409968484712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027214020454844345, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10653945089099867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026910615399958555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10636289270139983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027406807906793916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07913581735047029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055564008592309554, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07947504953471347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054298904886049514, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07923266526730659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055623154454752334, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07914970358878871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055652894171044465, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 48, + "hessian": { + "down_diagonal_mean": 70.80125427246094, + "gate_up_diagonal_mean": 939.2567138671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10735482606553465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021945624420670093, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10727503122343851, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02191460504779087, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10717927094347009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02216196469628938, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10727405873161092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021929079457188513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10733394424851044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02698730588847832, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10726467736921713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026130324497717353, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10729039403836542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026372875490236602, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10757131923808996, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026538548481342177, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07697572134164776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05729928911114935, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07821336032278513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05403177068462068, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07751178942436869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055910001196330376, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07722202092200973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05697804987096004, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 50, + "hessian": { + "down_diagonal_mean": 85.93025970458984, + "gate_up_diagonal_mean": 945.2205810546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10599140693668245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022785673000252818, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10611600207175427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022805338255755207, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10605347769106986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023374769116031945, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10612367348648859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02271699300089214, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10615360266746718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02568912734710473, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10586624692773985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025770468574210666, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10607323055053022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025478500575111675, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10611725386170685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026095050077396525, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07958512974950042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045088018936000465, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07937229505261467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05125151924291166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07927643671594647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04903733774548572, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07928393721222003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05038888864826156, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 52, + "hessian": { + "down_diagonal_mean": 79.57875061035156, + "gate_up_diagonal_mean": 947.217041015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11250839064697785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0199162276412552, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.11252286557457408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01999810890132867, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11234225131781433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01967433864776012, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11269895934768706, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01991018270014488, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1123761019031985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02348740717450819, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1123116722798302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023716129270581275, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1124265416377303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023424074339457843, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11248220242207731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02371317291984837, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07952846280713104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054835754388972764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07939249815241234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05520675761783912, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07934140314023591, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05496242326783676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07899879772565278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05521783686134445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 54, + "hessian": { + "down_diagonal_mean": 119.15617370605469, + "gate_up_diagonal_mean": 938.3165283203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10941503214169547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01809230455314371, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10930689768825634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018410857521145543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1094686279375986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018631259351527176, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10932353964126788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018364120454332118, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10946178339468896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020738865449069988, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10908098503228782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02105712240185828, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10920745166206912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02111550486547798, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10944756193989509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02105811849019093, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08136919215930803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049510537635656195, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.08242475960348139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04750435325081314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08173499749763831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049727058812774914, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08105398419173516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05052750441928328, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 56, + "hessian": { + "down_diagonal_mean": 88.69732666015625, + "gate_up_diagonal_mean": 891.3936157226562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10698018300781525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019319926354245853, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10698602058430631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019304883682482583, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10696909469324122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01955754331797409, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10703036381576342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018529067381831438, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10687362328411869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022588472270937156, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10675512596135062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021956815809185078, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10676381845315751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022237483180772222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10682691213051738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021699786259393394, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08300823526683884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04741011315418009, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08229571182011515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048097169671822926, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08227990394757174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0501068748854795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0837760537424022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04718767968028323, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 58, + "hessian": { + "down_diagonal_mean": 97.2948989868164, + "gate_up_diagonal_mean": 935.95751953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10674105880233271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020838598703082592, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10667944509464357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0199644321122482, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10674564763677193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020480743514477023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10668900211132155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02017919817903956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1069763952942224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02369680685614827, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10675407841534987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023508470731198106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1069586253288811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023900614068794027, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10684588339844718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023625507957489286, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08072259707931054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05124277191986785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0816415165240672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04677549232681614, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08119342635235051, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05021194321413165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08112866718959044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051091302048912185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 60, + "hessian": { + "down_diagonal_mean": 120.2400894165039, + "gate_up_diagonal_mean": 933.7113037109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10802370837215783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019027105300193737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10784412304784619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019205188020964083, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10805695188192425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019453293650508463, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10792801658182645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019080628679542563, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10784739220933998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021461228279037656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10801779375862998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02171304511448672, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1080463219674054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021596683260023904, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10781615068415168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021936570844323273, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08210344281889258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05145264575139753, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08148379668161558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05283823546354695, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08156246501058644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05257809310464521, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08166141998047272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05239274321264347, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 62, + "hessian": { + "down_diagonal_mean": 79.69110107421875, + "gate_up_diagonal_mean": 938.7864990234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10412314804876359, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02336484480075751, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10403529188279839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023145021139466704, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10410977415321535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023090018296997805, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10404400309546138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02277387882809199, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10414475975454096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028066140559369034, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10411978991857142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027945682761515815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10409804094867274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028657595288208326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041091936729589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028033640278077466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07738161516718536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04927990269545029, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07747301376183717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04831356213094096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0768473603602227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051731398544154934, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07739055501112323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05340711158647953, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 64, + "hessian": { + "down_diagonal_mean": 75.64093017578125, + "gate_up_diagonal_mean": 944.3092041015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10481369409382311, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02425863149608772, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10480622624668522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024287218401105166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10514722923297122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023992433978419974, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10468889660442846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024133963326925693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474978461294245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028129934837114786, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10467363772824766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02823620237582697, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047370069825188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028177814076821746, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1046892708008257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028096688426718242, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07746340667743494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054583585250297925, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07727340183114131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055205748984161716, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07713011245080667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05627911032800338, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0778308472807269, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05407287562426281, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 66, + "hessian": { + "down_diagonal_mean": 84.65718841552734, + "gate_up_diagonal_mean": 946.8775634765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11493532937229793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018256290991447113, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11489314313211893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018109555868997363, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11498004696217037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018351920411515565, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1149458106779022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018466526032755774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11473198859324625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021520350676433632, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1146967042167336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021224025513525088, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11475808272020942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021197491401270422, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11473751720078706, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02123661943397207, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08091204457445758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047915320473043234, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.08320316132345405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04454787609729067, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08192677715522963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04590588188630119, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.08163503426816114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05058625093369074, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 68, + "hessian": { + "down_diagonal_mean": 75.43392181396484, + "gate_up_diagonal_mean": 954.439697265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10357087396154807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021871611067649788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10339631433183735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021741099151166485, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10338756827377345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021830843276396766, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10347192163176416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021898285125925877, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10347991300585699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02651570268122829, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.103486828383608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026852913593818157, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10361190119078363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026600677178748082, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10341502758291214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026651604767949384, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07590760510278652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05808659059095816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07551300100840777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05821621477556533, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07558517947508095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05844691690817995, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07593659055935165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05782352890881934, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 70, + "hessian": { + "down_diagonal_mean": 76.9500503540039, + "gate_up_diagonal_mean": 954.8723754882812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10639793000717876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025666882318578655, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10617683635346589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025509293563340557, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10638507622315505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025581519109326538, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10647538498761275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025529715929036367, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10617932104623795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02817501605277139, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10615489704787964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028193860105150258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10623157784639015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028126421924107668, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10617577251813742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028055358217935887, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07562754349912146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05927211134053592, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0753875070313623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0594810577382685, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07596465064087908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05829974584898726, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07560162967768244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059259078972734094, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 72, + "hessian": { + "down_diagonal_mean": 86.06189727783203, + "gate_up_diagonal_mean": 949.126953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10575926892445577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023551787918206318, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1060061404195685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023533231124577014, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10599836772333351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02301571212344483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10599491152055363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023374900679446434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10596747601133458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027442705044641837, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10576400665427294, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027729650889744665, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10580550033883157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0273144626684143, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10568756024382985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0274608237326587, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07779384614073517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055413076411262266, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07722404029201517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05656377159532493, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07730265678279562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056136674926384925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07748057110949322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056439115645175404, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 74, + "hessian": { + "down_diagonal_mean": 96.65754699707031, + "gate_up_diagonal_mean": 947.71240234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10338159389702417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02080179384532565, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1033798748002587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021113299320638525, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10349851215699539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020781676560635395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10356933564463738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02082446275703417, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10325818370197787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02762849950815497, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10332780525222475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02799854252734563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10341524597002852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027652273476983128, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1032144139630572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027410142075990213, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07609416193913565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058159965695399996, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0766060477954823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056686003296145045, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07694777396131602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056554823555468085, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07671400514074214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056707430583937735, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 76, + "hessian": { + "down_diagonal_mean": 89.69775390625, + "gate_up_diagonal_mean": 941.9677734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1045389593700665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022794220684757966, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10466117623699929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022527997596931207, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10457575390861633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02294448889060711, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10444162828485802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022402791657341815, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10438876674732922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02729098912281078, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1044875719952062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027532607159826594, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10454043384777872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026723495104068348, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10437779877309526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026826768271981473, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07698395584482653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056533622462996626, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07723602009824343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054283060047999064, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0768048164678794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0561442842662664, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07660582972674887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05659647479039086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 78, + "hessian": { + "down_diagonal_mean": 70.00118255615234, + "gate_up_diagonal_mean": 940.10595703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10459674412986615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020206294287898212, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10445057573577257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02040878171189431, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440363441441283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020547863115496248, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10431251107599725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02040131695869504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10449198411752654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025494845030392486, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10427438597163476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02656847689252651, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10439953996518524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026102590738773516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1043777788566024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026011265698837382, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07827164594138952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048014163952838605, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07739959069574281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0485889618917055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0783140269534508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04435785414814625, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07729657535872402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05197258295177025, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 80, + "hessian": { + "down_diagonal_mean": 73.81034851074219, + "gate_up_diagonal_mean": 953.6112060546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10755392371527997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024020860519595056, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10787010028897223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023893781575354052, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10787846785354577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023950873840298748, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10761907921804902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023928699332460706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10766360303838511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027036560208351147, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10798368287096433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026935411186838194, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10769768697759606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026939343701507638, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10745290696915531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026795450944349822, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07632034998378172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0572821527696239, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07708230432210701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05389482720768431, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07650853288273894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05745149815904386, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07653713750919591, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05686622601159389, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 82, + "hessian": { + "down_diagonal_mean": 129.8821563720703, + "gate_up_diagonal_mean": 925.3272094726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10692281618264424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02117100853647763, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10689461154456618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02056381539955831, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10690618008534934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021737363727540236, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672845380339899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021522466233685648, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10691416912458884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023397676449476563, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10679061651898622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022837605195005096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10700193204572381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023832029885993273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10681394530700818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023121778326462332, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08109657530118916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04369990353742246, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08163126872352007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03470493979158534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08074744804647152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04536938553800897, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08186974022390171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041635564429742936, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 84, + "hessian": { + "down_diagonal_mean": 91.86802673339844, + "gate_up_diagonal_mean": 934.9052734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10895126782879924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020396221686265724, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10918752064959904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020300773737053194, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10913564926754969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0202710485422978, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10901928585554563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020481361161647886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10902370528488865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0234934414343076, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10898342421190062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02333951397266262, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10901703307670897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023117904207333013, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.109070683183334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023339499789465215, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0801630746974096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04552029393141867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08032760531722172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04814338867379817, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08053582988359445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04811550344220691, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08042720876472616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05018248439305183, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 86, + "hessian": { + "down_diagonal_mean": 80.44498443603516, + "gate_up_diagonal_mean": 953.1031494140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.11009265670335898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01809712265753905, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11019271204670064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01875596155012042, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.11038722546485288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018382556989600833, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1101577908607524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0186376175564666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11018034141293745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02027124369690696, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133597, + "packed_nmse_vs_bf16": 0.11014343268880769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02124405577342422, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11027679320885816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020375520467591075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11024881170246291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020758319497452413, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0807865027284803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049917233994020156, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08021345100031756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05109101577236139, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08111237756597998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04789085820003261, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08027608637658029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05028715706235823, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 88, + "hessian": { + "down_diagonal_mean": 94.4471206665039, + "gate_up_diagonal_mean": 945.3013916015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041873415652748, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02094135835718859, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10423640813224357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02146938229575945, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10445881896415583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021052161401716847, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10433304684378429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021745948109288146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10437760302366775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024604518410308925, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10430916246485765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024594815840617627, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10442933910443472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024238371462761407, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434562944563575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025237154667304675, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07746883779707917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05576264202931913, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07724936454716386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05584891099496942, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07729932331021772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611271041170178, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0767735958625735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057069863855459334, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 90, + "hessian": { + "down_diagonal_mean": 80.77271270751953, + "gate_up_diagonal_mean": 952.6093139648438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10911573895674884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020880202385905496, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10920748216142721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020659009299529157, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10915962212719788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020871432154968093, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10933903006919046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020906336605401483, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10908540126509872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02372523504900344, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10926636059318508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023711642710763745, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10913341184115717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023763541962961023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10909981892072632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023746875962361483, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07892224854899843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046046645162406946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0784238559460784, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05474148691368248, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0787372032841367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05392020141982308, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07903567926036148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05435696394048604, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 92, + "hessian": { + "down_diagonal_mean": 91.60929870605469, + "gate_up_diagonal_mean": 924.2393798828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10542155483189644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019667307525763327, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10514628038480561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019435435040051276, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536936705909906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019772718150951264, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10522867777257493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02009722443150244, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10539226933977416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022884756010437005, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10537140562367182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023334424806905903, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10529750231671403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022944073693229896, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051063489627714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023782706146971918, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08000534835678988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05390989695735369, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08004871540018374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053541145026368475, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0794271063589085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05237084351770615, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08044482107332379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053563335411425834, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 94, + "hessian": { + "down_diagonal_mean": 65.35416412353516, + "gate_up_diagonal_mean": 946.1636962890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500245566359898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024311033169828987, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10500653818621962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024365106475858614, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482820186630809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024205801470110182, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.1051289184352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02409106844469031, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10490551231395019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030596770473135894, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10494998049218818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03068057558084622, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10488055750765167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030615343433077608, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10489844989378565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030206583685037883, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.07636621769371169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05637101311873207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0766884990854134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05389052159381712, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07609419051067326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05703531253939958, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07642938196818351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054456740823879475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 96, + "hessian": { + "down_diagonal_mean": 85.47240447998047, + "gate_up_diagonal_mean": 948.1558837890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1034365330861284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02193335132984411, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10347064720670254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022307858742478006, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367101887792629, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022256880900771124, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10335047736071656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02218846834132575, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10353148649384208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026897136763061043, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10338881141881746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026983981281408102, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1032652906228372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0266564208665226, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10355551628630227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026927954399233768, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07738906360661935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055700878183078985, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07662628223669189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05782465723626268, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0772042399550914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057356293470775734, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07681580658882657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05783549588812802, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 98, + "hessian": { + "down_diagonal_mean": 131.93841552734375, + "gate_up_diagonal_mean": 919.3717041015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10937289587195731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017207998729077797, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10953587983652667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016712904251409207, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10947898188897932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017241687708840776, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10918646878795177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017592837868850065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10935375248539425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020509335892923395, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1090731334666212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021018213070705404, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10916393864665418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02073569816567043, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10936560412526626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02069968089897396, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08597868294905016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04324678176115537, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08556185343082232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04378005252990122, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08627544080445004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042612153622128016, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.08644677340279994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04310170512449931, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 100, + "hessian": { + "down_diagonal_mean": 74.49531555175781, + "gate_up_diagonal_mean": 941.89013671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10473798310733244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0239085076566199, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10490809309914358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02415838806607591, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10485471449197167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02362678491284791, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10480895537744774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024050226192687527, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10471252573214186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028771983939386358, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10473451003325575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02909535725334301, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1048511037427307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02856509008397873, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10465659018180952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028666066967836804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07711579372581076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05501768565736031, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07761418667092336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05486606521429402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0775453131545352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05328944535997204, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0767849638447688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05570210538711295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 102, + "hessian": { + "down_diagonal_mean": 69.69123840332031, + "gate_up_diagonal_mean": 952.3419799804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10753730174182512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02257492846828133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1074087372321826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0230251960306285, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10743971462087377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02236834253682299, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10734118329286092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02276177505576507, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10750287875940608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026192958768216756, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1074989027955842, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025950470579513833, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10750598014037889, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026659219822430028, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1076164068335495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02606881696588614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07703954013378268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04877065014751296, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0769622488482298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05493796189195534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07673517806014961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054980851803085394, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07679106646730648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05664309267987895, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 104, + "hessian": { + "down_diagonal_mean": 85.31571960449219, + "gate_up_diagonal_mean": 889.3572998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10705170985538473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015449728031473446, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10716251276575947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015176862056758065, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10702229852971501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014957357479875455, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10706157835181157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015308924005123523, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10698993715584773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01718088378249241, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10703906547767421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01734277186438674, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10718096848938095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017309805919935885, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10709105366896979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017093721771942724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08411841560210957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.035833669473291314, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08292147571704464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04234612142376525, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08277086013701604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.040464197589279986, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08427414978399439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03895080859972496, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 106, + "hessian": { + "down_diagonal_mean": 73.01968383789062, + "gate_up_diagonal_mean": 941.8360595703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10468029829883468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023689671857141565, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1040991790459229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023600716555922673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10442165573125956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023736802389853137, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10440568589915422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023389448865924853, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10433849798491376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02818958990328451, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10447299583724584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028804418242098454, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416326902451563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028083682656188328, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10429446376679292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027868481261523102, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0783727517196759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05027830654679224, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07725318856436973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0548608582289991, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0779877449215532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052346393426868205, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0785868137730202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04826009523439708, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 108, + "hessian": { + "down_diagonal_mean": 58.233673095703125, + "gate_up_diagonal_mean": 937.8193969726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10441324574203628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027062840861970684, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.104509142361446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027285290918075012, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10434830289601418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026924770224965915, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10443216118234355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026819179516165734, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.104600640707501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03038259594840805, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10452349438625866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03059719428792584, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10445813071295168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030581279731859045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10464364010713906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030690177040943678, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07457724661531992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061010042936305034, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07451637583317214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061078093714579805, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07467052274384937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060930942148857685, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07446666935421078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06143431743605599, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 110, + "hessian": { + "down_diagonal_mean": 94.61795806884766, + "gate_up_diagonal_mean": 943.5657348632812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10659472320940751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019294849717057753, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10652031912083598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01950931330148524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10655109704951161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019186246719120094, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1067490194380346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019202952654031404, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10663178667172675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025365433180640433, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10660790961154314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02526169784492543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10644501088862945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025398017802823004, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10630002142185477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02494915411539027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07982799565108187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05152724399425794, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08089203741327235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04951409085111213, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08083995188800662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049731454915302634, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08025022563698332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0507192414997587, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 112, + "hessian": { + "down_diagonal_mean": 84.06935119628906, + "gate_up_diagonal_mean": 955.004150390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10897028960934592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023693188230884434, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10891375334762252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023315211133617612, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10890500838780703, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02337414162915578, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10908903127154176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022581813211936523, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10915583272009319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026153540166665417, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10912890218138577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02664081358426255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10892783777070478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02608099435913837, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10900907889603288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02594308638696033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07725112056265014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05763932225788138, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0774773598635034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057043867135425586, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07749506916649432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05729665804149548, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07754261486952116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05713094451534181, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 114, + "hessian": { + "down_diagonal_mean": 162.7545928955078, + "gate_up_diagonal_mean": 938.13037109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.1097394477654006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.009798506014493114, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10969552608178544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01055200393177097, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10972294211806369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.010239662971842365, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10958642190246697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.009917125626840712, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10964989442040202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014461757580384491, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10967698819378563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014234648975026092, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10966856709256106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014431703492771508, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10958412500175829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01458413085826255, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08583274974818259, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045412068385122205, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0859206686922819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043647105302994524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.08716547910272068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04287474136802544, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08733641859477875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04376960754450975, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 116, + "hessian": { + "down_diagonal_mean": 78.69078826904297, + "gate_up_diagonal_mean": 907.2232666015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069421652866404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015836925769207453, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1070806475302998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01623968806312998, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10733990771456596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016300362199876465, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10723530399170862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015608302826418605, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10692499316498448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02191315292638314, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10703255698824446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021741985787165793, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10712853469376266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02208056580700726, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10704113544665521, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02114258466467359, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08076931354017708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04963256148486056, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08083221128730476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05045144064568177, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08191255865379651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047362286190948975, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08130889527046845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05014235361179805, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 118, + "hessian": { + "down_diagonal_mean": 86.13668823242188, + "gate_up_diagonal_mean": 895.8468627929688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10680797396220736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014110455114392404, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1067547925574704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015355000848225383, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10682323446234351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014322438905256635, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10671991037748248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014646268020855443, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10663358561459732, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015672159169060076, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1067288468280264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01616825900287793, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661333331900619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015789719089710875, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10667816895050611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015907084267135944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08444097212217952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04534128556167939, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08469684970019911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04418646423595186, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08452983608647253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04584210746077961, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08446872570157622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04531252145814847, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 120, + "hessian": { + "down_diagonal_mean": 92.1963882446289, + "gate_up_diagonal_mean": 947.25048828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498801519337972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020162553274177278, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10500436200395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02016087388767877, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10514984550694348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020857335372793894, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1050411756801093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021173712286323157, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10493634531033247, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02375338535629607, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10500398034344914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023634122091844553, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10498425070425317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024209595907698653, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10499900042478173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024064182396127023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0781632254411926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05581150635977928, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07741151297119347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056397532821158086, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0778709301765088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05624879791459073, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07712895980838409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05757647289617114, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 122, + "hessian": { + "down_diagonal_mean": 93.93234252929688, + "gate_up_diagonal_mean": 941.083740234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11046626801064638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020529373846541358, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11053383567265299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020324318058509854, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11029535021460961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020486642310612795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.11032856767291845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020190413311117872, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11053827845309515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022652255436171817, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11043521260128773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02274071490839589, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11033151790108121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022519320191947363, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1106080566790217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022647511831939873, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07832602311838904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05439382128863801, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07837893847951032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05531957472077919, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07779286817069304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05642225229631843, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0778113041742746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056025734048469895, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 124, + "hessian": { + "down_diagonal_mean": 66.06449890136719, + "gate_up_diagonal_mean": 935.4498291015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10543416092898096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02427166862559913, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10546276007706057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02435518634162491, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10553571507956959, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02441568620610911, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10548940309932274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024689913181865895, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10536050633369373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02816365187727101, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10534391083286379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028747511426372996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10554572281981152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028736507934492023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10554615942423828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028420256841897636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07714739162667474, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05534585852775235, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07661166685512624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05657227334049348, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07712501470146574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05514992335678213, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07705715419418645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05552818476778308, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 126, + "hessian": { + "down_diagonal_mean": 84.87898254394531, + "gate_up_diagonal_mean": 946.948486328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10683153607302691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023050101759991615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662357650417385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022987777775343686, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10672266707315879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022925142955839414, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10682168044108928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022926496636481244, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10675945061733588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0265723845008113, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10665592710666603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026359112161045046, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10681208574869022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026506738723541094, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1068300195589758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02634041707825097, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07734347990976134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05555260404747115, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07793218338045158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05086646042832912, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07779668868950981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050681344352315326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07795018605771033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04911778782645832, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 128, + "hessian": { + "down_diagonal_mean": 78.69275665283203, + "gate_up_diagonal_mean": 949.5005493164062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10626900414697314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02268410546618465, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10625658674106173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02308270669329848, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10640962095106042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02274589153609294, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10639970815738405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023456854001551456, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10624326249570273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027694964567260526, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1061303256978381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0283195818203388, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10629688980094579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02701901665547533, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10625987965145543, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028208575769171244, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07655867858947554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05601834746740269, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07701185615359635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0558904069104537, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07742986410296858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05387056729964457, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07737563637607965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05431100230698607, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 130, + "hessian": { + "down_diagonal_mean": 87.53399658203125, + "gate_up_diagonal_mean": 947.05615234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10513215778024229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023348295428703827, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10533877344678166, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023329986805766152, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10567371063296098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02310171611847497, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10541305801523433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023356698877883473, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10554715188050282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026550434395635616, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10544777552579071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026595220167672306, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10534840182172638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026586025454849845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10533366946721699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026579616228484138, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0763588109136095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05870314048170691, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07690766449436219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058058309413594196, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0764584699402011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05856336009635043, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07611585562470792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05909838815801504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 132, + "hessian": { + "down_diagonal_mean": 57.85383605957031, + "gate_up_diagonal_mean": 931.2554931640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10497106357074028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0247758968089726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10488038114451456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02504076185563295, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1049289028557877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0248852587591904, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10505962693569693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025098045997866445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10490751958999477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029601060355134528, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10504595931664781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030136445324728478, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10505750076379487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029772212850818058, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10467971325711534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03032647952377937, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07580340245295265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05836892345190505, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07536505024407752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05942642866056934, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0758319345102596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059012908533573084, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07552624122196448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05915436457606174, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 134, + "hessian": { + "down_diagonal_mean": 80.20906066894531, + "gate_up_diagonal_mean": 942.977294921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10489831389220304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023954217519008336, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10473694326105655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023428484572693614, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10494659742038363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023492728638982014, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10487918869885912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022766432109229, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10477959927803404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028183740232318216, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10499553156145859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028565759295738565, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10477888425074912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027830991436130034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10483223082161865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027932164851725377, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07787416329880982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0507421663375083, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07734222777509533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05557268417246661, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07796979003414241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0537823313121601, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07842589131326397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052706832273105964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 136, + "hessian": { + "down_diagonal_mean": 65.43392181396484, + "gate_up_diagonal_mean": 941.406494140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10470934954949782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022888134820561463, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10457946919838224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0224508441939079, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1046826862045544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022623537863824567, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10458306269001094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02259173829269156, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10451609765189197, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02725149290562032, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454754308194183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02591517588457071, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1047342200998919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02655263495970327, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10447989882453067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026535645837764014, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07643059680864496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05078089810692302, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07702991753571768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052759862641475336, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07658075089443363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054731566006153407, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07683775291858401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04725594931380589, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 138, + "hessian": { + "down_diagonal_mean": 55.23842239379883, + "gate_up_diagonal_mean": 943.9905395507812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10352771436939716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020953038201713594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10340463960822975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022254579300527825, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10339327970394781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021664995750506465, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10355772795268048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02160954212143145, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10340647872878628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026932611425139377, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10349960212849084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026885010071813282, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10349872949655048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02639863670792132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10338583904303317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02659536718020381, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07658574549431355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05253097160683446, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07605952817077244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05493008892382977, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0766484053568927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05326406667325372, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07645191020494711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053699961178874246, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 140, + "hessian": { + "down_diagonal_mean": 78.26570129394531, + "gate_up_diagonal_mean": 947.2903442382812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10431767385986145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023511403114598615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10431460082137646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023761769786892383, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10427247076820444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023705490771973797, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430217307130034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023801181379274186, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041396269685289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02858824847037121, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10417380842113191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029230800907367228, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10441838633629118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029175798554236743, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10431583086470646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029255462628480614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0769274232427433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05414028025461125, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07705625949328272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051717481029824944, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07643479530733052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05541693205504669, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.076809687249767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054335300446987325, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 142, + "hessian": { + "down_diagonal_mean": 70.72308349609375, + "gate_up_diagonal_mean": 951.8446044921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10538800709215494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02314425485451745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10547797162874975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02314330037499016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10528017009680089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02323002951248223, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10529220783490256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022813627995199405, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1052116853699025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026506797635674588, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10525394350298951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02678535823361053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10520577924939142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026353467765833995, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10529958864088451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026445866783909606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07697451360459977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05560144030070559, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07730490085716216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05558903837794655, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07647453050344237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056784683658902116, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07736982470959278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05565058714759057, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 144, + "hessian": { + "down_diagonal_mean": 90.3609848022461, + "gate_up_diagonal_mean": 938.6778564453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10397138970793966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02256915210754427, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10382082604908781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022991193208137608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10395627005279867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02283947339070356, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10391676615681415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023305733742878743, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10414380411885926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027190230508194694, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10380069396596887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027023567364004995, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10411700599022229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027351676520180557, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10383579071660758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027020817108945473, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07711796210592836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0573059834034789, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07727646777323254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671400778126064, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07722548009794808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05700723295454425, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07725944055543163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056375968222947735, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 146, + "hessian": { + "down_diagonal_mean": 47.144046783447266, + "gate_up_diagonal_mean": 923.4988403320312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.107681002145846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018914141191660797, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10780241516759727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01929543744910513, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10769224906148044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01889286320485165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10773704010771183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019145712779026283, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10805205164405424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02548345298121346, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10773031492815936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02560902433031164, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1077094652600319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025484294936521972, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10781093014877757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02569989984017304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07743409647678105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0543142875502219, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07705128063230764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409499189117624, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07705586788206982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05504539539124011, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07734781468150698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05196447047085394, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 148, + "hessian": { + "down_diagonal_mean": 152.03427124023438, + "gate_up_diagonal_mean": 943.8192138671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1094938885942138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011596753241664192, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10953443849813468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011750393688060676, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10976019170192025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011578449565386562, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10982943840330904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011588685875650769, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10960761340845827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014617163444231897, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10966879875623807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014651319143368549, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10958183119841046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01469044886368208, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10956522063702219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014744313959496956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08385270672489681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04811533759491452, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08374722222626545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04974966063179637, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.08441442630995644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04826098190025822, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08317578354471619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0487154885161409, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 150, + "hessian": { + "down_diagonal_mean": 70.64401245117188, + "gate_up_diagonal_mean": 951.803955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10481775171821801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024487154334385263, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10496366257291037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024951513865959136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10475441735287774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02434788850685547, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10468224371594322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024652632806102245, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10480392788461781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02816174927407262, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10489828899562614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02825219871811648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10478116846788249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027845822870352355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1047220110901032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028025588085031673, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07555506777195013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05891595760614027, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07532316033632094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06017928747115451, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0753487460755912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05965283170688842, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07557554773428454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0591192479146585, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 152, + "hessian": { + "down_diagonal_mean": 111.84512329101562, + "gate_up_diagonal_mean": 922.1978759765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10678563072083244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016411429717774324, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10660266998414702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016757109332617728, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10676895242902934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01677535952358649, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10670209981632367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01636228141717144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661081636186298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018322998141899225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1066130452727567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019559085503436963, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10655279671221025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018188929650640426, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10670732888747794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018382317214367394, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08019738367862117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04877816577631728, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08020032044148727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050867804056218234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08029875166690204, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049223414692489066, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08037856975420765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04777158087921973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 154, + "hessian": { + "down_diagonal_mean": 89.91767883300781, + "gate_up_diagonal_mean": 930.0615234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10556785391452271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020490153845428426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10556315667476869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019991105302305846, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1054479150548082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02005444997745648, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551099892318648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01988767523268513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.105421402301094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023963246187543343, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10560984947204645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022673170052917264, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10543659580700934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023112430395505635, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10565816576802105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02284959239921817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08186505327077723, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04919762842368787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08291304419809191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04547012179549374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08192994462216904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048323663038421276, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08312958821722362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04413006494497376, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 156, + "hessian": { + "down_diagonal_mean": 85.49229431152344, + "gate_up_diagonal_mean": 946.9837036132812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10484643052605758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021365002144024564, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10469867333945408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021517167197070412, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10471569880555634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02131981554345845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10455156124098246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021415961591116094, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1046537228556399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025956338893283453, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10469468056712888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02574531000976763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10485585599455215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026245685630519492, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048001678679438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02565009776802747, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07820687933492378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05223801477282934, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07828283249178944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05307188026560724, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07807461729444815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05032377666253413, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07887971879648087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04879605102461519, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 158, + "hessian": { + "down_diagonal_mean": 58.56503677368164, + "gate_up_diagonal_mean": 947.5120849609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10431550567185548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019313927182827498, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1044452972765046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020254569343498736, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10448924651111696, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020175649133957075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10455188005932771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020277128752690925, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1044042827890324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022794358177152543, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10457833927768713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02175708912012436, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10438651268324683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021840527966027094, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10455324701015727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02233371660692044, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07898078914637834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051013048393239835, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07896153376101274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051333918684682574, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07907533627844471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051121700486072716, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07908461508671581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050449730078014766, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 160, + "hessian": { + "down_diagonal_mean": 57.39873123168945, + "gate_up_diagonal_mean": 930.2322998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10379939337963147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026716529927520837, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10383849956284916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02712865962014775, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10368910055581101, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027121504775453834, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10357460210602107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026975305519578673, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10364129046782786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032022487816670976, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10353607551773412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032009295691337904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10350195824932573, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031804437537342314, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10370697481321368, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03213946767196056, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07528936822181632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05874727226741484, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07509662646617228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058967995280757274, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0750321594547436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05890466406609203, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07525912980584212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05861605029888335, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 162, + "hessian": { + "down_diagonal_mean": 71.94790649414062, + "gate_up_diagonal_mean": 940.5552368164062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10752815546299063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022574598088548642, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1077202152791986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022205658280385447, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10758986346016682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022343401908663473, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10755316121256485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02268684501147727, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10734049956352523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027048365385623545, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10732090995302694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026227698678208258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10725271271953744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02560588961978245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10731026026006786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02723262303239506, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07682072074668127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05668673503767757, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07783357676441637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05352122973898999, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07781940389083924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05135336527078434, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0762112346408139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05796838234186941, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 164, + "hessian": { + "down_diagonal_mean": 137.88845825195312, + "gate_up_diagonal_mean": 930.4378662109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10584159754493519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02058457853409507, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10591636395723891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020365761234165853, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10585336187203027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02049753274030144, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10586524204130704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020509353290871227, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10576494435725221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022948013864003032, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10581632979896494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022751851742756502, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10574286625283034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02336570570199244, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10578524330089047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022805159947660695, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07972139057007045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05276568078966738, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08008242755161582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0516861800786431, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08052149171823797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051337406870151835, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07992068091258238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05236348593491451, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 166, + "hessian": { + "down_diagonal_mean": 88.71908569335938, + "gate_up_diagonal_mean": 947.1104125976562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10729432169676928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022747284335041588, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10746736127652594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023002655402256125, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10732087394972462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022709561314834616, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10769187796282942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022608924597706408, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10748102936719812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025634640632579964, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10724082952882055, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0256212618669689, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10743011962725382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025565384839271156, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1074484086763632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025544394928417515, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07782673863658313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05625457994083475, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07769101855841944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056453361558545696, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0777644821438858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055970323071333775, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07768090648003605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056771799398910895, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 168, + "hessian": { + "down_diagonal_mean": 74.89920806884766, + "gate_up_diagonal_mean": 909.3898315429688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10627196554091484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016364455512245718, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10624581755577679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016673983470126667, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10608081224975084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01654127655346485, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1060718086744385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016392828841187573, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1060814051036232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020370607875630543, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10618673176227558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02041483281979388, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10600290176601196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019966494985636508, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10599488697125038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019644351746749743, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08070868371503222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04642353207340135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08204699717859099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044641975068627375, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08192820034079158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04263276005101552, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08154354509305085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04571270583972617, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 170, + "hessian": { + "down_diagonal_mean": 73.95829772949219, + "gate_up_diagonal_mean": 951.087158203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10463891456722332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023924251062976725, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10464416163234194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023688289081219296, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10436023054454575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023836714569107773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10442017597332977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02348524403514016, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437365682814953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027902185099804137, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10443523152873795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027882115057655286, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10435708312907155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027913195161937396, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10436282725375506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027636579651198482, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07628533054007998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05789442891246856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07579191335456616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059473863920386824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07606481476326091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05857305319198985, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0760610733786292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0575463073773744, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 172, + "hessian": { + "down_diagonal_mean": 88.28030395507812, + "gate_up_diagonal_mean": 941.087158203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10460449613306438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02306855379872699, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10474246777128281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023664196978551707, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10475853091241433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02343223387375191, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10469517323334523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023261484341037267, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048405190842501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026637037384686944, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10471038785929368, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02685928458886493, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10470294012705054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02644175673582367, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10469502097119722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025918688870968175, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07680286757839688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056918839447105185, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07636265591581054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05819868284607738, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07683175929757967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05626852552050929, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07739772876051998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05614366832098717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 174, + "hessian": { + "down_diagonal_mean": 89.28707885742188, + "gate_up_diagonal_mean": 946.44677734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10850513285520377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020734374030320364, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10857245370419494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02039971753044988, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10846211259354091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020447792342476685, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10848872091454671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02089161619901302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10834137879089338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024390880069610793, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10820788998517357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024410764804574114, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10834556390586454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024774663897186078, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1083925881389293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024565184217812678, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07853946186632027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05520375856044337, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07925191425766465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054652255582914026, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07846199978409388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05547012266034702, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07856287327969919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05566654975532681, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 176, + "hessian": { + "down_diagonal_mean": 97.47415161132812, + "gate_up_diagonal_mean": 946.5562744140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10564175285497206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018719144082909322, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10592354882252432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018009970936186236, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.1058068006132667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018291589048090445, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10569742364240568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018131799039478383, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1055358450177874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02547077333231906, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10577601779518543, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024821695857100628, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1058050456984089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02497239775631008, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10568974288176328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024217742804833728, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08075696097042048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049100867977796676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08070145456138272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04780574626949954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08028522608083681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04938833900948075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08145168329520998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047903596667784494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 178, + "hessian": { + "down_diagonal_mean": 57.817230224609375, + "gate_up_diagonal_mean": 940.370361328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10573419530329779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025279193859151425, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10596525526127716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025104878265677425, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10608366642781122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02498440876926023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10592791868948434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02515579715136301, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10590153934014876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029281752470232025, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059079325391896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02970897229726936, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10583022630218526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02938974063589475, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10608856524555546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029422252363702835, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07521000449468403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059290357438621676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07491109361836405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06056984181957726, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07490195682092136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06048417633872545, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07498720173025542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05990205761049145, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 180, + "hessian": { + "down_diagonal_mean": 48.460933685302734, + "gate_up_diagonal_mean": 924.7034912109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10426398574561475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02306591859830782, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10398756927927769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023206862636996343, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1039871527151376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02299337502059586, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10397754690173376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022795580402964397, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10406032550885015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031188409308812066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10380096908266331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031322074658080984, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10400237037818395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030601850310333932, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10396140018905678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03016370204858853, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07514343660073798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05824538637525951, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0753225732911143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057807305174762555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07576316466024888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05725818411680813, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0756850422641524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055473330149529015, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 182, + "hessian": { + "down_diagonal_mean": 84.21726989746094, + "gate_up_diagonal_mean": 941.3681640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10396729350186659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023072732160841075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1040474464205846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02308861470179099, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10411077636336792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023447030567673315, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10406616127333702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023962946409331866, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10388908127458511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027521143895887924, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10383271942492044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02787274205691383, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10396389156222996, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027592072850820413, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10397365725759469, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027738494386575337, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07707888679441077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05051478326473938, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07703985899414838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04992182786112921, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07742928448466992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04931973608574607, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07700534123150228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047588753586238784, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 184, + "hessian": { + "down_diagonal_mean": 84.03047180175781, + "gate_up_diagonal_mean": 941.1907348632812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1046890513076214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02217943523930325, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467593219966927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022010514044696408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1045688131096603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021866153591427562, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10467323203631745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02231502508632883, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10469159660586609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027106582636004874, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454755582190332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027574469969863916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10471305234727264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027432321011325464, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10461027001179288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02728837693084532, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0776624651662964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05436263792678349, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07749218901763863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05541874395241889, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07745531208107195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05499622265359207, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07832069680400988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053290766260452496, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 186, + "hessian": { + "down_diagonal_mean": 59.52299118041992, + "gate_up_diagonal_mean": 930.5309448242188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1047428208748038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023004235359598793, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10479205721882438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022647856302541132, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1046292558026584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021965508593999, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10478109745320366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02274460593556375, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10466507528036284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029218361931879794, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10447766712491566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028865675001317613, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10473871018934988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02846139720271207, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10462687252948312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028553954752359233, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07620704077735155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05637427749906976, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641306578930446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05615872920125322, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07641310754685211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05403150310959122, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07619585507071276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05488862472104225, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 188, + "hessian": { + "down_diagonal_mean": 97.67453002929688, + "gate_up_diagonal_mean": 934.5527954101562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11041057504385768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016896582981440356, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11054559112503455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016669328136683155, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11064578637122262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016698808853876666, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11080574825364811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016556662241677424, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11045831601101133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01968566048914984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1103484318779007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01962955744347688, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11070051752650048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019581618232573875, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11042440257242433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019713510377659667, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08282505329215943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05082336627470215, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08283613629848492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0503142327662296, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08270902981136669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050271479175884225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08303408995978367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04918234676254749, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 190, + "hessian": { + "down_diagonal_mean": 72.65433502197266, + "gate_up_diagonal_mean": 940.4154663085938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105246488828657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021450301037509903, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10523585521278457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021355715435779406, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10532501412965756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021386779910799514, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10500879064355491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021658768697949242, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10533887936347931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713124673110938, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10499372880585302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026844906337448533, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10498540337843314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02731352789528796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1051390128464334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02699531607762101, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07918206465876815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044813819001163324, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07816880779127867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05107086293398028, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07813962858608227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05195569828068053, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07882406122410276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050570298958065016, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 192, + "hessian": { + "down_diagonal_mean": 80.25511169433594, + "gate_up_diagonal_mean": 932.4317016601562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10384170654407185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022768932906169256, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10397776853311122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022460418074879165, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10379868564960099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022874595401298006, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10388694425221974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022460992547904967, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038067047368356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026989127782868914, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10401574886666674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026667396930361027, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10380942401567679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027725392353173964, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1039037033727126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026633742447919564, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07975033605047885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049184985084432524, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07963991098579082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04775907939491689, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0796909790990451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04520373127416219, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07935571747954895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04823329116716349, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 194, + "hessian": { + "down_diagonal_mean": 80.71558380126953, + "gate_up_diagonal_mean": 942.7021484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10726401634031522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022924792912124656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10729466780110188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022649656540600187, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10740647639180465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02282563751198845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1072358066383659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022970563861116528, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10725003755522776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02715661481941015, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10735719090257528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026679509835392076, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10734521094735289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026653609900533313, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10725065920860756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02642602748309235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07697384635135308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057911461027582115, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07722012943605801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05758181363884845, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07698219249400384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05759004029222358, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07786098391277459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055947718480966774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 196, + "hessian": { + "down_diagonal_mean": 102.9488525390625, + "gate_up_diagonal_mean": 916.8692626953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661286255774993, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017271601382291772, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661754359578945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016913818389126766, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10690816110594364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017261386228910946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10671247213537868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016641841245027728, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10665748074910415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02033184548252426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10654612133941394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0199156737291204, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10660301332046321, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02006404994617222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10663398909046322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019437635652269584, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08381241632406296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04876399899580168, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08396849563741367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04777620448466683, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0836176422160094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047755940332900086, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08414480761753632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0455887887526162, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 198, + "hessian": { + "down_diagonal_mean": 65.15906524658203, + "gate_up_diagonal_mean": 929.9306640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10543082975129542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022569994054729984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10564134943325501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022082067960784067, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10546672407776889, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02205640287780286, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10558185405118503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02239058679731804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10536741101759077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028570787420903743, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10528013375964097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02770147687084995, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10535540818312915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028161067966542174, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10527606503759777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02795124981146817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07759500856527776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054494559347954476, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07794146533645493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053349941141741755, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07707540529053547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05680410507990517, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07845625543506488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0541513913838447, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 200, + "hessian": { + "down_diagonal_mean": 63.91040802001953, + "gate_up_diagonal_mean": 908.030029296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10585515163595545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01708838415064097, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10606406161811285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01691989412954154, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10587680286765273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017024395665526656, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10581934916677793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01696415874971921, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10553171685189679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023125944952459943, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105830049036829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023187874141762543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10565840629276681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022896048919795582, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10575943405991016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023628670539416304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08013149428755646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044235348856773646, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08011996983735754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048355958234211016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0800587387905063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04893772838634339, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07922089225218557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05165220921961648, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 202, + "hessian": { + "down_diagonal_mean": 91.58360290527344, + "gate_up_diagonal_mean": 947.6763305664062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10447399018372219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024760899647642012, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440531579106489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025430949914515177, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10420541554573083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02469741720556275, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10432575293061871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024762592172284905, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10426817915407036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028807268025036678, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10450870105384519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029096987677664644, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10433470691936658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028827767883874103, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10432450938185647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029144491338371065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07584561397091678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060086911104236966, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07577710377728708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05948999992099976, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.075574077714072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060121640426808884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07589086452460768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05990850519870079, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 204, + "hessian": { + "down_diagonal_mean": 64.3184585571289, + "gate_up_diagonal_mean": 943.6234741210938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448607007050664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027192905621404525, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10442942927129473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02726845589999061, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452026040748426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027184135633262895, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10449861668328891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02752112479881968, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10451868919475092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030969819960877977, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448347185912417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03092289857589783, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10437461886682828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03059797382147543, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10440431377304846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03091144777144738, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0747479274543403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06076335708680669, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07468969332575669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06016237124527588, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07453065607319327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061125868073499075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07462964923750388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06124129578867295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 206, + "hessian": { + "down_diagonal_mean": 80.86605072021484, + "gate_up_diagonal_mean": 949.0201416015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10540871177686102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02365684936848244, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10532210605733552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023397595107560534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10530881761044401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023561179527676362, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10542763727888722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023385485431740308, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10531838677710069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027165457472581644, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10539378492565644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0271426482559026, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10547692119538785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027321300300193887, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10537158487686908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027160877581398107, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07713606342232172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056440245494112896, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07685651166775562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0567768417521444, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0768476013798695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05782101242029457, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07735752827623352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05654502032439363, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 208, + "hessian": { + "down_diagonal_mean": 91.5489501953125, + "gate_up_diagonal_mean": 937.8496704101562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10575575157345611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02209249930155916, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10589788862707011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021949395303111203, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10576414788837059, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02229126633817091, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1060887506955246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022276920337251697, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10590644211051307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02517313056704396, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10603671218054961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02504611368827979, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10589212144661321, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025183173004522368, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10588026117052764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02528719889162263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0771220493252536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056851860262869924, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07742540777948267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05576970018909112, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07728665972939985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05633534967990169, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07763259567408116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05362477606188889, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 210, + "hessian": { + "down_diagonal_mean": 77.00360107421875, + "gate_up_diagonal_mean": 940.9153442382812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10466184817406694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024375142721436233, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10479677554881882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024107117482011203, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10457469769931386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02411746968572819, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1046560939724793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024021576348761686, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467265807239343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02879388825934774, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10452570824275376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02919587330168923, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10480515433031848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029451658548728773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10477031856342695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029214359331254897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07592956122055412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058694006599263396, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0763184441284481, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05801642337917702, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0758630630643866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059062989044640135, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0759352886830934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05843538685564837, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 212, + "hessian": { + "down_diagonal_mean": 113.54688262939453, + "gate_up_diagonal_mean": 945.9306640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1048462560892512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01947815906730977, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.104794746555289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019699974910641016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10449948197573124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019769387368537682, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10484019256216087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01973657107312395, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1047109551440733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025086604757111334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1046742622964464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025365782139509397, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1045014474076387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02565780128394035, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10456711124970301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025389876863060386, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08015457670632861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048232842175935785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07984163278651693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05030225624332822, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08055424960268058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043542431410981255, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08023295013518317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050213003312847344, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 214, + "hessian": { + "down_diagonal_mean": 67.14308166503906, + "gate_up_diagonal_mean": 946.090087890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10626136165371078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023913037387096166, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10629644991682892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023613844157478178, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10633886600856776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02369732754097892, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10636697949698679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023591101450378155, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10619278438172894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0284574589809395, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10616516088189035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028112087313195065, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10612399949222533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02824195925903478, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10637642489199436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028302856667192934, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0765320352985863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054583180409686065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07661211081743277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05646626179804786, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07662803975693508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05532933949996574, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0770529150666289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051581795413999404, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 216, + "hessian": { + "down_diagonal_mean": 82.92237854003906, + "gate_up_diagonal_mean": 943.5560302734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10350739368208577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02387881089629922, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1036180736831782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024147081121664186, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1036211063692024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023966151751240637, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1034454027195575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024242021311657275, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10337840269446258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029109295777292023, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1034964182020914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029173563141380095, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10369269101650118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028758160690240248, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10367296854013541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029121488719183974, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07673751929937113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05657425087426867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07628737122551563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057230438329416336, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07739437279818322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05584035860306836, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07739662077279064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05570751551213407, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 218, + "hessian": { + "down_diagonal_mean": 94.88410949707031, + "gate_up_diagonal_mean": 940.04833984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10460108440575129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02184708335632864, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048978656854158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021846929328748963, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047223095532337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02151071482578355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10462087272444538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02166955918238615, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10483253542486813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026417822320217596, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1047215900460298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025149126147604943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10465312708820967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026036202967585617, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10457104025028886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026035387450167993, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.0777989618154456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054263439588203094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07813220888650758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05168049175001245, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07791749737982184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05379448301497404, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07751207339258238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05461848947242469, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 220, + "hessian": { + "down_diagonal_mean": 62.71857833862305, + "gate_up_diagonal_mean": 923.1752319335938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10558226533519982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020176506621003714, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10550516314506632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02015384842522252, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10540620057825544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019880659249567535, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1054343771163498, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02002073196423821, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1055329644196717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028022600871837566, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10555256336766682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028322903432143116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10547064012487453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027611279105448146, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10519650708589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027772516652977815, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07903162323782681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050277985607213235, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07977036635761471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047939404962874126, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0797330111355746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04829195522638871, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07991959131434791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04906815776916304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 222, + "hessian": { + "down_diagonal_mean": 68.019287109375, + "gate_up_diagonal_mean": 936.9879760742188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438363360822184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02536059853172111, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10441761835757687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02566857097270751, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10424610194293006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025828248144330858, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435601510518788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025649449888104938, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10426790012559853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030154796899848256, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10425074768968892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029868847146530427, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10413734340413398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03037702335711364, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430302087794734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030171791930004983, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07695983532102837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056350436216598826, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0768845475132141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05584437470645291, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07667301944133656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05510538943706962, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07655108736798007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05373794180792085, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 224, + "hessian": { + "down_diagonal_mean": 97.02763366699219, + "gate_up_diagonal_mean": 945.4828491210938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10438662786522432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025621198145246604, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10434275277929192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024771226765045687, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10421820970911226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024941787907681416, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1042191731807386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025967708668071087, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10441800082024866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030005036514701985, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10416694657759296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029420917978357258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10437288442601247, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029673662341300706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10429781349531606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02986339273431421, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07788185706518332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05606395470685504, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07820367572779315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054328608448175546, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07808052222910664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054856726880895795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07800295774178186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056717192478162655, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 226, + "hessian": { + "down_diagonal_mean": 100.33316802978516, + "gate_up_diagonal_mean": 942.3101806640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10377474253799779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022745981637106043, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10381053330272826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023190642126740085, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10382610222336855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022716333019320953, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.103809388264791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02310390172013698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1037988149110314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026038887140451508, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10372425510918962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02598415975686819, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10381844257882324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026082016197315695, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10355778684453991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026048422052077583, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07615758388410779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05916807723924108, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07636528809968046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058216648351510955, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07670856229901411, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05711275097001468, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07619388167025114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05862459889136827, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 228, + "hessian": { + "down_diagonal_mean": 84.28826904296875, + "gate_up_diagonal_mean": 902.006103515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1071386299101138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013049477328788675, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10699062413120576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013940529266977333, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10713297965557306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013844839310886734, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10684869872921242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013501850268768812, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10698748753225519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01590650188503581, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10688283786818915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01678344564054183, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10717290624575364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01633770984324521, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10712891859335134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016315123413834832, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08676090370536621, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03950273529968308, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08472150714871116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04439337427170995, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08458348830442729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04326818216656379, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08514843779626814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04338210282293701, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 230, + "hessian": { + "down_diagonal_mean": 97.28128051757812, + "gate_up_diagonal_mean": 943.0371704101562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10487669530548971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022136725890796072, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10488187475384352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021927537288259618, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10502097866286511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022260084165807555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10493241339703611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022696633956361855, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10497647638224332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02539419157645407, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10500499401020284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025480015584686527, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10514259512790293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025292301650496077, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10486050949364534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025757958842964914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07734178096181256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05608346251935959, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07665565467870365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05760826803029859, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07651727188485598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057451356796991905, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07647684489875406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057988991432378026, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 232, + "hessian": { + "down_diagonal_mean": 86.74600219726562, + "gate_up_diagonal_mean": 947.924072265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10447963722262046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022744025988977894, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10442000503385172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022515062634699642, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10464595135970733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022953810981804254, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10444868408717452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02254825890517308, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10432207556540272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026788055832197104, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10443884541796838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02653652304716693, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10438387322233002, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026817354786735897, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10447244730430287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02648365048938019, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07545289400689133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05958269752538189, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0751311282217176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0601310330377901, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0753734815079257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060110044862987294, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07528380222407036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06000097722438712, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 234, + "hessian": { + "down_diagonal_mean": 101.3442153930664, + "gate_up_diagonal_mean": 934.1488037109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10456669568280516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01997668930852398, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10477460175699453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020247837102683647, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10461067914420209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02050295055154184, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10461851708477987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01990570372550065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10475037687341772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023421536247531446, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10459702580367548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02292294357716371, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10464281909354564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02327518454892836, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10450302534623987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02257920719308798, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07807127827132312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05342234716638348, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07846634447182778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052383263038572846, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0777630385589687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05457445662912419, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07855804790356852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053612554797935946, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 236, + "hessian": { + "down_diagonal_mean": 109.25973510742188, + "gate_up_diagonal_mean": 935.9844970703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10472487749585832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018907090245497884, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10505158850340775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019004489578756886, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10476180197356932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01924858579961777, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10495787019234308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019454242954233262, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10505181156916533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021745968932712707, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1049488806309278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023101819866643662, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10493495653945394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021049499505730106, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10506442715082578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02196513082894702, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07776744819160505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05438251950092741, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07771731380777043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055601372567121284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0774056112665151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051589857644720225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07885641631368824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04953118049548932, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 238, + "hessian": { + "down_diagonal_mean": 85.58096313476562, + "gate_up_diagonal_mean": 953.06005859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10473992155304812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02321621161670493, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467348612175478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023430397306764897, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10464690897188472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023077207837700855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10464283871536614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023082569868184848, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10453551045961532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026583708481216207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10455511084880305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02625772709849638, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10456777131304083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026384610629931214, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.104699977766912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026346621480942436, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07538185594824386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05954909616375092, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07499098240933799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060265861192334805, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07526476287714164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060130952076530825, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0752933334600394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05991155926282135, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 240, + "hessian": { + "down_diagonal_mean": 74.75343322753906, + "gate_up_diagonal_mean": 936.2340087890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10700478597584716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02405804231615603, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10690690927900856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024042156857379253, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10678699140992831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024165940871633276, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10686634032081412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024562595281794938, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10681005195072503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02777356354961946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069443852369632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028254681600839694, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1069060572579595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028383607049385806, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10690735952643937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028596073972617878, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07827635536198004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05220521684673685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07832718349387327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05353263323794001, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07770531441863647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05536343758895123, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07750062647186419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05449326119927414, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 242, + "hessian": { + "down_diagonal_mean": 55.130088806152344, + "gate_up_diagonal_mean": 937.2047729492188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10417570066229728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023439001839280542, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10413446183331865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024072488574312757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10417307915756042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02410344049032907, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10430261224237085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02418859659239314, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1040957392936249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02922680051354112, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10416020351129189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02950152154056077, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10428572671574456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02931968851830706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10399765657190511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029698770315876356, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07504329710264136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057235703922478814, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07506933952827105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05758892417612482, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07483315884566594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057563868303422826, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0748901019361161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0582343789517195, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 244, + "hessian": { + "down_diagonal_mean": 102.06834411621094, + "gate_up_diagonal_mean": 917.8551635742188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1091881878444405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018167276490304952, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10911888151685466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01733882739186058, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10911906889321178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01763507890771274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10945866770179484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01762418898378372, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10942562324141754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02175160078310056, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10902587257587341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020572307217587824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10937268962924197, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020819329407270232, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10929765417465537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02096665934300686, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0814251579333318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05020375647657938, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08297168414753667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046429767739700194, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08322148385910869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04606292075618453, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.08226085578894084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04889155773988859, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 246, + "hessian": { + "down_diagonal_mean": 97.79701232910156, + "gate_up_diagonal_mean": 948.2132568359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10528842876639836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023020441917172597, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1053007876409515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022944974193534643, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10536065596822462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022795836359714397, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1055743252116283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023027703023597447, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10551036992385812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026817751372017872, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10523884931396768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02641455087976136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10536773164147886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026213186770945633, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10531164010043481, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026599424377677417, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07841025603802489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05208674368625779, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07862471192261335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05347362167729763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07902252135173175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04715956614309445, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07937565323300327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04482077246183515, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 248, + "hessian": { + "down_diagonal_mean": 81.3575439453125, + "gate_up_diagonal_mean": 939.6381225585938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10515179604395651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024528940472374253, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517435677199895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023989557789571663, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1053784512689649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023576324366240405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1052506760171862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02373560808938296, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10511868961008243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0285735457576047, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10515859238937665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028217548073797775, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502957898679441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028234759968306947, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10505028408900877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027843616376276966, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.076264364331234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058358701924247615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07669780757493416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0566853899392055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07720117535732249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05605356044959909, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0769677810211379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05571610058603588, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 250, + "hessian": { + "down_diagonal_mean": 150.09234619140625, + "gate_up_diagonal_mean": 943.6627197265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10941396541099714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0165469785131092, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10922401083441637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016573493440000135, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1090498148305342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017025348065572977, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10933939877176768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01731345521918357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10950193891105382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021270489760594802, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1092975945141997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020961262219097526, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10925882020617829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021095920842200078, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10926354697204255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020487022235020427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08392565518158243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.039908643845543816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08405001675050934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.015370235199971988, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0828102861599845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03901899866068604, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08353046812626413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041105941451138696, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 252, + "hessian": { + "down_diagonal_mean": 78.64859008789062, + "gate_up_diagonal_mean": 944.9188842773438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10367316165280727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025421195307988986, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.103605233605539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024921455170943404, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1033644680925576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025726660381882566, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10355634118864344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024268552519640565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10357918493605901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030859271663551922, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10352426906609007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0315445919713949, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10352446742633978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030730550155017595, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10383693364758181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030570314998481545, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07654111181567547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052851445374833946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07720067749573994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05085723227166154, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07588368522838539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057927322687127296, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07705318215578867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049609405775246314, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 254, + "hessian": { + "down_diagonal_mean": 81.78147888183594, + "gate_up_diagonal_mean": 947.7046508789062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10381409208787558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023863736099120854, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10366866508757525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02338570312236922, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10371691299096726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02375202113513855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10359231087800393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023531337176404897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10375199856286568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02760988763890908, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10358772023855173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02760549904507019, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10369660640282827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027456931995905667, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1035877597181919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027424949794337954, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07547023650496192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058827934502733645, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07591439120595218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058149533794590605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07641466524867105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05669875532532069, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07551912812582208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05912789597300921, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 256, + "hessian": { + "down_diagonal_mean": 93.6646499633789, + "gate_up_diagonal_mean": 931.4044799804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10644040899232275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018588640296559152, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10666526526116257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019413603652363637, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10661702570393014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019185943807804775, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10668125471759417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019701676262009898, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683322420804978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02176021150948753, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10673862842313843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02216177125909584, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10656135853120415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022128814565337177, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10673772290013973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022290810325002606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0809723848272275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049834067852730625, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07981866712077676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05188357428924798, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07997246135466111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05085244100813658, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0795595186915384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053279452230757605, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 258, + "hessian": { + "down_diagonal_mean": 90.67298889160156, + "gate_up_diagonal_mean": 949.9397583007812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10648487665454277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021870731448317863, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1063991564630959, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021972513946783543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10661880196233944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021628024583390502, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10650758235559125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021776524484058073, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10646679447091963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026578101574680425, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10637510937394974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02649125661024203, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10625994810116969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025880140719007294, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1063976998115203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026104036455858987, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07775234528560451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053858677218601284, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.077601241885099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05282626264763412, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07852639930171389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052362036375768625, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07818569350179393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05300722213008295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 260, + "hessian": { + "down_diagonal_mean": 92.66806030273438, + "gate_up_diagonal_mean": 934.06396484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10806349750045316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021459033866192388, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10809519596532172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021085488656549738, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10789482402466986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021482316384644994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10772656726604682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021318440132034974, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10773665377836465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023569639688397707, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10784359297597018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02309613658215543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10794493641263278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02368851136454975, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1080242321915935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02353159858378919, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07905227605498878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05499835416012945, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07986333386562801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052033106859759114, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07991387694454156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05297744115482052, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0799090132815902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051493939338415534, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 262, + "hessian": { + "down_diagonal_mean": 83.6792984008789, + "gate_up_diagonal_mean": 952.1032104492188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10410161448092577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02306495249898671, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10431218265551147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022674544623009954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10411060125882739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022776711133154638, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10403232790877948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02290951518327956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10383815068860745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02777613398359215, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10406492807073765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027674047949435163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10409053780178044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027595003962901796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10407927157918644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028029472870543894, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07693712752632693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05727283540918146, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.077062163769945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05772530171937587, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07660855926045901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058034556951086794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07650884492458176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0585640863686012, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 264, + "hessian": { + "down_diagonal_mean": 105.15691375732422, + "gate_up_diagonal_mean": 928.9295654296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10433242321806008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020446065147793815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10458289143572638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020440810807228635, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10438068947096867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020771748550586305, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10442136028172955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020229326864074183, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10420346621747849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023234236787189125, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1044044644833934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022521939685902142, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436208648161134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022672126736635684, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10453577154336977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023152279822154953, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07893798445784238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052969518245164, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07933040587823544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05160342709980211, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07908303411732515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05286985240751325, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07999442601048744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05047128825552232, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 266, + "hessian": { + "down_diagonal_mean": 81.87537384033203, + "gate_up_diagonal_mean": 950.798095703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10576531565761377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017088534719164734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1058222172851349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017256535282983813, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10583762724732748, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016533661357469552, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1056266597141551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017132466927504666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10576111494055075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020184142563745414, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1055751713408721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02026608924094593, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10580800174619788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019611184087991337, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572321088819053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0200056516602185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0810019293901258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048135927902564095, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08249748373081452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043694115070191754, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08228596270257794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041728287340574266, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0818536714265194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04364668734127794, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 268, + "hessian": { + "down_diagonal_mean": 76.54148864746094, + "gate_up_diagonal_mean": 947.8062133789062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10461931305000122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02445179358242882, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10450046977625317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024859413609291427, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10451781444790283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02459213336135842, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10455752957135724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024299676357115902, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10441707713710595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029070599146634485, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10434198560246692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029088421755296492, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10433307300354161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029096334039067466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10447837622183881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02915848291459504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07558385214782683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05982014982481306, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07587499938014704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905858350446681, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07584125656030512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05930046477653533, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07590029667779138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05864719869474706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 270, + "hessian": { + "down_diagonal_mean": 54.46979522705078, + "gate_up_diagonal_mean": 922.0174560546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10553317972316188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019984784691515704, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10562980334317319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019823632432476312, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105577994269988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020109794671489063, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055539842081799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01937175653826237, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10559179807950196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02469264640199687, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10558631118454889, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024664999455221735, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10560340605231056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02496815828049506, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10570369691699884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024509337345416965, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07795859094466817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050326469959285865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07812170995891175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05276762340606588, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07785972984173915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04942457052246688, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133594, + "packed_nmse_vs_bf16": 0.07894686185919737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04963519458559936, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 272, + "hessian": { + "down_diagonal_mean": 62.22691345214844, + "gate_up_diagonal_mean": 941.752197265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10274253094276183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024287330248373335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10303981806473424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02395827360919738, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10297830895638434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023078903794284417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10285088645439772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024062636384908027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10271199539313282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02933225185961432, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10274928615453482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02951264598074133, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10280856781247616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029048151649694088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10285627777031214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02959556817342612, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07656735495142258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056108685090077604, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07645217352805612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05562146320440782, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07658353187350747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05408482924815351, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07576090422185533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05790350146646137, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 274, + "hessian": { + "down_diagonal_mean": 190.79721069335938, + "gate_up_diagonal_mean": 931.3555908203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11205767505095059, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013501955900465911, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.11168077074756735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013993721419899555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11174947135029378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014101327266037824, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.11189986789742776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013627726980460778, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11184963516179917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015433784457414905, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11182543758998863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016350909084717823, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11184036298167367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0163375657073295, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11190086077961527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014849284067684355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0892855300041786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.038119882847555936, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0885651624901955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03680680394654994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08850508163638744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.039123099109530254, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0893454586261848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03265774252957233, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 276, + "hessian": { + "down_diagonal_mean": 68.54522705078125, + "gate_up_diagonal_mean": 941.998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10342691238465755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023577853859028882, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.103615287272675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023948131898986452, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10340379305388858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024159422771530988, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10380018035530639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023459156255282398, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10354184042900387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028361369685772766, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10337109008041644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028981232363350047, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10344745169881378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027955926643697584, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10356148161043663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028411609826996267, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0756009984567049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0576314618353954, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07545017514533485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05851214218029567, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07631556921922988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050557665411677534, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.07559137151814033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05820578220311687, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 278, + "hessian": { + "down_diagonal_mean": 49.03890609741211, + "gate_up_diagonal_mean": 929.22021484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10347999897752201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024994966906758457, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10330970310348564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025341891128621594, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10346587248721018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025380612816583145, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10348089846062379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025109253520059098, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10362531798809227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03031923200298093, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10327263584434926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02906664141677473, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10367702579304797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030395548656279713, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10348241095336316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030133323446140295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07494446523997719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058120011125228736, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07541999761119982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05638473323609103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07486253114707199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0577809684791585, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07533848482618404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05683629139138724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 280, + "hessian": { + "down_diagonal_mean": 87.97015380859375, + "gate_up_diagonal_mean": 923.2188720703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10836087831919437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018920903614990135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10859824610119607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01890122025552882, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10848066385981572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01949718277231358, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10855450710531299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01910165200025164, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10843102081602131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02271431254461359, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10838602445374564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022829853920970634, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10841463777818634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02357073686345693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10828742686801911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022978928599823537, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08037517528905849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05185006563518742, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07976730648889432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05181892822265494, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07991056274294588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05275702927033704, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07918564157156519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05392542168983714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 282, + "hessian": { + "down_diagonal_mean": 103.34507751464844, + "gate_up_diagonal_mean": 949.048583984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10620330920803828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017493720992456087, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10592930872597855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01692449142829871, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10597821339041694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017013481665904575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10600021709104558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016872027588176013, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10598191320081579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02165923575271714, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10596420710851477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022010476582530122, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10613858541399156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021429295770240015, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602847876916789, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021700807799734295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08086640436470818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04677410012887222, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08067490731831287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04738071128013121, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08076673538830278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043248002170247435, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08127101723217191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04344549590023549, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 284, + "hessian": { + "down_diagonal_mean": 77.74371337890625, + "gate_up_diagonal_mean": 947.7078857421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10526760382871113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0242218247650278, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10545787898569088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023977267177690692, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10529948002928942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024096520664889255, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10535032073814884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024363337630409956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10527910767188615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027176129670937653, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1053923042953622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02687635059861257, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10550095682180566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027036489224753938, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10546808531930907, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026989871782033083, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07638328428171623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058929888169123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07651518473035644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0577316167105935, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0764529861959031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05873310928006505, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07618118503518045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05952996955146778, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 286, + "hessian": { + "down_diagonal_mean": 75.894287109375, + "gate_up_diagonal_mean": 935.9466552734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10429642561514364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023072294325564926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435627561646381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023261277843285393, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448014796767856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023551484526639806, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10447079649358344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023557862399388847, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10428621362553607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028424801302526444, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10443938285409962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028318782086307552, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10460727954245265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028605057217058506, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10434897695306984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02848885064666385, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07868508166359162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0524393620558733, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.079144369651292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05198538280328402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07913660428135032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051447865877036196, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07887227857116887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05184023579742132, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + } + ], + "experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "gpu": 0, + "layer": 15, + "physical_experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "physical_tensor_count": 6912, + "schema": "glm53-brandon-style-low-bpw-tail-part-v1", + "selected_k": { + "0": 2, + "10": 2, + "100": 2, + "102": 2, + "104": 2, + "106": 2, + "108": 2, + "110": 2, + "112": 2, + "114": 2, + "116": 2, + "118": 2, + "12": 2, + "120": 2, + "122": 2, + "124": 2, + "126": 2, + "128": 2, + "130": 2, + "132": 2, + "134": 2, + "136": 2, + "138": 2, + "14": 2, + "140": 2, + "142": 2, + "144": 2, + "146": 2, + "148": 2, + "150": 2, + "152": 2, + "154": 2, + "156": 2, + "158": 2, + "16": 2, + "160": 2, + "162": 2, + "164": 2, + "166": 2, + "168": 2, + "170": 2, + "172": 2, + "174": 2, + "176": 2, + "178": 2, + "18": 2, + "180": 2, + "182": 2, + "184": 2, + "186": 2, + "188": 2, + "190": 2, + "192": 2, + "194": 2, + "196": 2, + "198": 2, + "2": 2, + "20": 2, + "200": 2, + "202": 2, + "204": 2, + "206": 2, + "208": 2, + "210": 2, + "212": 2, + "214": 2, + "216": 2, + "218": 2, + "22": 2, + "220": 2, + "222": 2, + "224": 2, + "226": 2, + "228": 2, + "230": 2, + "232": 2, + "234": 2, + "236": 2, + "238": 2, + "24": 2, + "240": 2, + "242": 2, + "244": 2, + "246": 2, + "248": 2, + "250": 2, + "252": 2, + "254": 2, + "256": 2, + "258": 2, + "26": 2, + "260": 2, + "262": 2, + "264": 2, + "266": 2, + "268": 2, + "270": 2, + "272": 2, + "274": 2, + "276": 2, + "278": 2, + "28": 2, + "280": 2, + "282": 2, + "284": 2, + "286": 2, + "30": 2, + "32": 2, + "34": 2, + "36": 2, + "38": 2, + "4": 2, + "40": 2, + "42": 2, + "44": 2, + "46": 2, + "48": 2, + "50": 2, + "52": 2, + "54": 2, + "56": 2, + "58": 2, + "6": 2, + "60": 2, + "62": 2, + "64": 2, + "66": 2, + "68": 2, + "70": 2, + "72": 2, + "74": 2, + "76": 2, + "78": 2, + "8": 2, + "80": 2, + "82": 2, + "84": 2, + "86": 2, + "88": 2, + "90": 2, + "92": 2, + "94": 2, + "96": 2, + "98": 2 + }, + "sha256": "ba8406e1938d337b46e7fa997e8f224cb45cfaf20185e3c5b1e8255b7add8ad8", + "shadow_experts_ignored": [], + "source_sidecar_sha256": "777ad714f42334af1b0a2d13f67298c4b1c1c41078be58938944678b473b5e56", + "state": "COMPLETE", + "tensor_count": 6912, + "tier": "k2-k3-tail", + "variant": "2.0" +}