diff --git "a/layer-33-part-tail-0.json" "b/layer-33-part-tail-0.json" new file mode 100644--- /dev/null +++ "b/layer-33-part-tail-0.json" @@ -0,0 +1,26086 @@ +{ + "bytes": 922818568, + "expert_reports": [ + { + "expert": 0, + "hessian": { + "down_diagonal_mean": 254.08187866210938, + "gate_up_diagonal_mean": 1500.72412109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10654641857141407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021703063362573615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10633048692359748, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022542905769328728, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10628056183666333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022396585585032217, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10634819058869892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022143494630148232, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10627384897954806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02344440350424226, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10638208899481866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02503491416392236, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10621951398469677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025209955122575383, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10611893089209515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024632235687693977, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07986321809190229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05362607677334808, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07906315368275779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0550212074993826, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07862744650676366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05568129982904503, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07968528614660651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05367663116060825, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 2, + "hessian": { + "down_diagonal_mean": 115.43730163574219, + "gate_up_diagonal_mean": 1500.681640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10472468254783249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02491975440730501, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10473941894744454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02504327939484263, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465649849020545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02489201608293178, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10484333967114988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024716907368108592, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047198016748335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03413135365050946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1045837388278744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03372647421201048, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10471949402003225, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03398452835616909, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452947708630714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033536431947500056, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07384963515338681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0621347936390521, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.0738881877825013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.062220180653187025, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07380681254172611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06219805057014621, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07428725962589267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061428461623602625, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 4, + "hessian": { + "down_diagonal_mean": 233.69503784179688, + "gate_up_diagonal_mean": 1500.5618896484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10671581552717115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02212667400963909, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10676688625021469, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022000469192360362, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1069127913976903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02202150323494141, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10676762203459733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022291980888699412, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10685409114050329, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024496979315978242, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10693535299768236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024480192392607313, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683835236734532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024523485898479466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10674130108261715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024496988993390464, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07913875423740274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054175613103743116, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07909209093777408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052931791095738624, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07913903306475303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05454471986803692, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.079281756981085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0528365850935086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 6, + "hessian": { + "down_diagonal_mean": 240.25210571289062, + "gate_up_diagonal_mean": 1500.7347412109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10851434525798709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020177190560784177, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10851317921180091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02015784364615369, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10851030627123578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02021779776256294, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.1084503990926304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02029524934712574, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10838306504984353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02183617925316417, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1085479217685538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022104371785034653, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10831887989929834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0218101627683584, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10861236538067656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022178940957592604, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08107895043060082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050422584729553785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08077821057369801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04952420428079863, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08070948400827746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049331918896055156, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08035244964889683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05066823593613508, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 8, + "hessian": { + "down_diagonal_mean": 224.93753051757812, + "gate_up_diagonal_mean": 1500.71484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10663293584318592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02318294552349907, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1066268574481895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023016756611648852, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069245682598243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02343091572545184, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10646590016103231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023353989347652912, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10675781269317836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024843568152396037, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10661820617125448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02501503849715392, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10661767573241898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02521087230304579, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10657064043347893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024990822091449733, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07958049125791171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05395700765702728, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07973591674938073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053886123726110347, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07897038895633943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055080101940280395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07919447094137344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054457659975120726, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 10, + "hessian": { + "down_diagonal_mean": 160.5687255859375, + "gate_up_diagonal_mean": 1500.372314453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10547490435808107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023902948158477027, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10549580476631408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024162339198825347, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1053509647319128, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023857091540324112, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10518433092935019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02395948543017064, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10524405595417839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02725870243819385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10530519787763189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027494973373941467, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10525322980701826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02732249742841792, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10520807212001794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02737469326217568, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641782941569544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057470542069633486, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07618291868370398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05752259069027382, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641127767755268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05683849220681816, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07604335407694579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057390215761061826, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 12, + "hessian": { + "down_diagonal_mean": 221.08004760742188, + "gate_up_diagonal_mean": 1500.646240234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10574039896029169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027201377021641774, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10584575862089826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027318168637452227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10593095343123557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027183418699741165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10584860651584929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02692000531241746, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10585352238448503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028975213331320325, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602583284678842, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029018905340299852, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572666650905967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028678213202125706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10595941610133597, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028968561351912225, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07726825742687224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05773552772783367, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07694414377936726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05835919644537053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0772554499338816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057779797618600845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07726314145881451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057838711964012565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 14, + "hessian": { + "down_diagonal_mean": 169.87857055664062, + "gate_up_diagonal_mean": 1500.582763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10542717545815863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023876942848462594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10519249577317592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02388035789338088, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10513660588640508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023756269068956126, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10520910205167981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023842786001496453, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1054329348036045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02551405345103755, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10546224417604813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026319643040386254, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10521316405499956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025890863448958152, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10516522750118096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025699299435123585, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07733962034993905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0545358091952293, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07705755723555513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05592282700305223, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07738788053763003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05373427549336709, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0772014161126189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05503620672919848, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 16, + "hessian": { + "down_diagonal_mean": 203.58963012695312, + "gate_up_diagonal_mean": 1500.7391357421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10474935380930733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02533621878665135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10458830737810061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02549802178850105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10488600609020632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025785622048734677, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10474555388721517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025599467805620838, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465954754862454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028837984122176454, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10496603991005024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028565161409520753, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10483390865234882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02937068202639954, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10493429629022766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029374601111470902, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07773454940722295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05615733280737811, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.0773206968925284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05707912468898505, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07751252773749413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05780129571255855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07727627746512906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056274718715209976, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 18, + "hessian": { + "down_diagonal_mean": 217.95669555664062, + "gate_up_diagonal_mean": 1500.796630859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10699890304514398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02613734386553827, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10716980380924397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026473182416555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10718496670519989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026700134890338397, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10708534149209784, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026256010935550824, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10702583567750316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027718878180260243, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10711265160678334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02774802633711942, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10695635608141053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027946223524832395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10711747847901536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027404703199535346, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.0775597867393767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05425549551837586, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07754464858904381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05540324755832481, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07747065114309307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054251765361400445, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07775986453509257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05397705027301323, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 20, + "hessian": { + "down_diagonal_mean": 293.1689453125, + "gate_up_diagonal_mean": 1500.777587890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1070917874078168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02187983742544273, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1068920036919383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02194000029709878, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069904850298948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021472735896716322, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10673292383105236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02192158603984212, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10686868639766327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023901673935903537, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10691937600891176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023758317356135605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10699166798719088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023550188273085984, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1069111559102287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02396766771428279, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07988848991549648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05071679141558758, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07951900396479405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05170630401409387, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07953309188202522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052168869634101646, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07925964616839574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051536146271738574, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 22, + "hessian": { + "down_diagonal_mean": 117.42999267578125, + "gate_up_diagonal_mean": 1500.509033203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10324611531013972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02311075497306114, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10326959666455053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02317862946924331, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1032905638672289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023220691108090163, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10315593945591293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023168651310831533, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10323708975452378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03322224644401018, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1029136472568872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0327386225577474, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307808596819341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03290259999573401, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10304444608675127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03287967414623261, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07409829257518386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06151792600264554, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0743809875697618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060491387268091106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07426939384892811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06122048839489489, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07403049382500176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061163332834556355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 24, + "hessian": { + "down_diagonal_mean": 138.07437133789062, + "gate_up_diagonal_mean": 1500.5760498046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10368284517991921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025811798710994323, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10339271264620283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025837344825348902, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10349619396934016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025964245294555346, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10346692369382952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026006394448356788, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10354074041558689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03222505102074173, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10345286205296719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0320710942822993, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10359548028653102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032171234825100724, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10363949348607307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03190848607404779, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0743936558287553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061017764803261036, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07457766279198268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060475505965240865, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07483221537030162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06019960246508081, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0746804131255268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06015002506724567, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 26, + "hessian": { + "down_diagonal_mean": 258.7883605957031, + "gate_up_diagonal_mean": 1500.7191162109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10574091671481094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026187090462388794, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10588705598104942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026175811893749484, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10583985557095939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026083336596889526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1056862571504791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026000306158464274, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1056508813828645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028116178746171384, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10549229317644283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0280358973314097, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10570788913369551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02801794063576794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1057046663947947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02799993061970146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07751022174862134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053516845139320285, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07803230165382728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04724111479557453, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07763502129002185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03851478713270728, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07871371239919608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042713318630850206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 28, + "hessian": { + "down_diagonal_mean": 136.8090057373047, + "gate_up_diagonal_mean": 1500.612060546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1035257000639549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024631356918226835, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10337706912380681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02479779613196534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10334862707878527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024745143228868782, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10342229447522341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02482821662422106, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10342000425655062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032243041047101226, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10349028799673088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032180999811661275, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1032674725189256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03171701626821448, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10335124286043557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03258740354987144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07524913825929186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059776317587453, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07524671089112035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05982146738219446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07559011041640859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059195658522373795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07514195559093312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05937575081501481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 30, + "hessian": { + "down_diagonal_mean": 162.57278442382812, + "gate_up_diagonal_mean": 1500.7786865234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10528561689962933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02947591693733856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10533743877902371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029320245395363223, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10540738014209486, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02929225992162802, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10548679521220669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029544447771333537, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10544310927153007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03200353649431712, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10532248124690327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03178558548140302, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10544078188690258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03172277516671869, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055938107010097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032194194576062346, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07412991365517306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06257584463908068, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07439281817521744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06193016456425071, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07450613311377655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061779063603773575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07441940044731366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0620621443866635, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 32, + "hessian": { + "down_diagonal_mean": 200.6963348388672, + "gate_up_diagonal_mean": 1500.8486328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642671068344924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02576512396409708, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10638094231294545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025825507756364326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1063368655757807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025641234929936012, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10650505960197669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025763462483914396, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10638049640305595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027282524487189663, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10640176684374476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02732223263628774, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10635783854326607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0274321396661732, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.1064013651236767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027586355310051436, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07734134688002475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05552997072468561, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07726818170366316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05619229649140112, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0776816546703792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054839332888236184, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07729842145591583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05612050621431254, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 34, + "hessian": { + "down_diagonal_mean": 257.6145935058594, + "gate_up_diagonal_mean": 1500.55126953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10861434845664185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016754622501904598, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10853153444858572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017011468793109073, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10867623065293681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016786740365800492, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1085070438125448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016483711934554644, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10859619865361504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02064216520035298, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10842807694772506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020799317002044505, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10854442296589442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019953890145738055, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10845507399840008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019412525918536965, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0821218320895586, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049429354694017595, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08070454268453255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05137869930969054, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08197251015197013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04924621415815393, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08219585794398557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048235276801868295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 36, + "hessian": { + "down_diagonal_mean": 162.42416381835938, + "gate_up_diagonal_mean": 1500.55615234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10326821622379487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0284177183477548, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10321470210824944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027939777997656692, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10319821697650056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028510969024164497, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10325998887583303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02836884784705179, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10331660146426151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03210253686821619, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10325571026723883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032884752516891165, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10334472903436098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03277050419395388, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10318260412197829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03238487181194609, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07503325664187248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05846031683143119, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07482932700564221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06009468805083799, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07443662924842942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061244946416144405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07447220885312479, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060852268349405034, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 38, + "hessian": { + "down_diagonal_mean": 253.51443481445312, + "gate_up_diagonal_mean": 1500.70166015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10715788630400164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021980164570386387, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10700461424936988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022087637163271025, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1069486883452089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021958048955722943, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10718285660674022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021973661444650794, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10700677724278773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023738592744630714, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1071076015073991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023959430459570965, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10721759648244998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023922305302717013, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10704765124911662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023726219367226475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07971988690663839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05339239904392467, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07921274103596827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05427860750611842, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07976729270474216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05353873564996319, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0795280223887741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053824203543205495, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 40, + "hessian": { + "down_diagonal_mean": 129.23828125, + "gate_up_diagonal_mean": 1500.7042236328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1037033312695295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027966532295572718, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10380802992673506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02786823395994933, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10365863518068537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027859795219418955, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10361081695200655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02774423039101434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10350113710429684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03484362351166071, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10352094295115098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03509013890970119, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10347365472803968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0352452480585338, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10353050360888033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.035187121420118886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07400534151365025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061594452577038, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07357569551815156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06273103049687574, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07372238680521187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06260257799747974, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07359158711735286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06259774687426824, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 42, + "hessian": { + "down_diagonal_mean": 188.9534454345703, + "gate_up_diagonal_mean": 1500.7877197265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10720767387627159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023978030277913707, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10735170381367608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0235757590661777, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10724761842635218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023427687358263276, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10729944462307021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02351823313326854, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10730261578307802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026603194783025568, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10737020596636716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026589759950232834, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10718812313321253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02600521050058872, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10719094912727271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02667380780015817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07780905502436913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0550116920377082, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07727207974079438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056467965873949366, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07790824354689886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054167701089129994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07800047373803327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055112437968098636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 44, + "hessian": { + "down_diagonal_mean": 187.62478637695312, + "gate_up_diagonal_mean": 1500.764404296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10473824892071008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028617866399702433, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10464137001658182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028243127206220554, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1047399873495282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028444150294908693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10485395159167755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028674949707690123, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10463582546046059, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031294591176980656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10470082605182116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031188756660533026, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1045860356047693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031026581396529595, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1046437726214671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030896092827421673, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07614038598223849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05854388711610474, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07639863572693348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0575088820746861, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07624650431586305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05858948876337516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07609862682198694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057019078918225885, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 46, + "hessian": { + "down_diagonal_mean": 135.094482421875, + "gate_up_diagonal_mean": 1500.731689453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10489040710975304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026117758725838283, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10484227148038588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026239313529483582, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10486252859129928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026444579362339196, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10500103243112531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026250634343580312, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10493908180801122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032483724550407655, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10477202203883805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03247170073653167, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048780132260794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03253742435708368, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10512388240178477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03237548328546145, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07454872968744285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06106302069011442, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07504600507113021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059071687107562165, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07453123005111095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06107637127264076, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07506284002888713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060018013485810075, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 48, + "hessian": { + "down_diagonal_mean": 241.51478576660156, + "gate_up_diagonal_mean": 1500.4476318359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10590163078236559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024450401470568074, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10597021721895762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02521441692386623, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10585469862765791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024886124795147794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10595009407867802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02464175288717845, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10593952539410113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02610953457577773, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10566535496071588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02729818572158793, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10585311742736392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027282601291006642, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10573441442986524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025886617064636228, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08067898784123867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051083489815574416, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08010892270674601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052566334225049416, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07961875999747299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05262050840556777, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08065131307382961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051545738353696434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 50, + "hessian": { + "down_diagonal_mean": 99.09275817871094, + "gate_up_diagonal_mean": 1500.5379638671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10375629072125547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024384987514069444, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367249986402248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024858801204456937, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10374443973874252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024541688651516767, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10370598169453435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024586425494369884, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10347615283349969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03486696484454887, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1036621727994096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03477320575384569, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1034651387427096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.034906559352303045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10368071962249767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03482059415325492, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07288792383603719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06359605303998477, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07276198590383673, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06385735146586598, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07290284495116633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06378526731007679, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07292269614666406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06355073482765607, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 52, + "hessian": { + "down_diagonal_mean": 109.99957275390625, + "gate_up_diagonal_mean": 1499.700439453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10282743365283953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015039758009273587, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1028183593894426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014832375907697704, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10278324479795986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015606833544736683, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10261637227983922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015514011052499343, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10277032714198117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020786198726564068, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10262171887011107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02007943194991272, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10289448511592071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019826249018587264, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10284183246280397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019926439657424706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07879037114575849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04654053381419492, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07839882340544096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04627797448799769, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07843350672188454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046864690349423706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.078789184569383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04648716186208063, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 54, + "hessian": { + "down_diagonal_mean": 1817.8458251953125, + "gate_up_diagonal_mean": 1500.0850830078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047594990778925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027818705225059893, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10499008388424821, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02797825867465295, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10492367793378338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027944831727188594, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10485736528699595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028212065134262023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498122694730279, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03108134781051163, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10483217431400192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031061512092089617, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10486230671729453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03155239800593091, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10495627534347378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0316597482582302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07452203106201473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0022965285147197407, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0762491679505301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0070414971875379035, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0742614683873121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.003683742109227917, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.075310336145746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.006012041132213612, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 56, + "hessian": { + "down_diagonal_mean": 351.8634033203125, + "gate_up_diagonal_mean": 1500.68359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10636048481784387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020115403340606495, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10638135221478769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020452311398120027, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10640702567451962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02011412474533694, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10643384101880879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020510456338272122, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632456891648592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022007231465541566, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1063549614514349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021039161945413727, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10633642353421749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021608065258976048, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10634087028414047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021582386007522374, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08440041308374652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04120187815767214, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0839109286502078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03870584485759023, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08428214252660869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04144712169535155, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08379618207392854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0404957802731168, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 58, + "hessian": { + "down_diagonal_mean": 215.39749145507812, + "gate_up_diagonal_mean": 1500.664306640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10633360954448991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027692710191631, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10640299675597228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027602482861540757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10645620787888893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027420073065773435, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10641665396345656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027907845305187914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10643895228388099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02926669777999822, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10617460231700367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029161909843535817, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.106342640602626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029269422584729837, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1061507149185332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029362753545206195, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07602515332533186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05962654322730625, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07583248083265617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05984851267112182, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0762543999947793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05946465581534846, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07613411846440861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905775796241458, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 60, + "hessian": { + "down_diagonal_mean": 171.17977905273438, + "gate_up_diagonal_mean": 1500.7330322265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10681667150979605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02577641726677834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10699293676619374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025805679088830413, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10697977973656606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02556729494909654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10671027570048737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02590801495981113, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10680906619909947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028352330208214385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10690666763460749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028474157805703237, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10693280835147283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02843106006250739, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10675140999036505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02814914495180147, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.0761925478781852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05865988744568971, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07620193826453588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05842429163571257, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07610859946318448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05841029427912784, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07605630487737072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05904125079436283, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 62, + "hessian": { + "down_diagonal_mean": 157.805908203125, + "gate_up_diagonal_mean": 1500.68310546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500742650720418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027525579171173657, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10499268843523121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027493804105719135, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051177609764384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027416488667704263, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10477085515764932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027383863436952125, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1050081320141862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032161112670408984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483881652106068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032159162819859616, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10511350754736842, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0319601099561405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10494935534727043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032167382903376646, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0752514561666744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059946157484328896, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07537216839748181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059794714873450006, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07509727049783048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06001354271132431, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07510386961880611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06017323607417199, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 64, + "hessian": { + "down_diagonal_mean": 170.56048583984375, + "gate_up_diagonal_mean": 1500.6556396484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1057252046239836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026160609714967965, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1060292570138995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026174563001713537, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10602096917616377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026035373528571274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10607561177362795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0256807065172615, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1059133490281979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029534398905375882, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10584303545285766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029083055334402454, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10604146104182692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02964146952293306, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10604166247393405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029400602716399202, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0759571994424398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059128792240463274, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07631594778188358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057406029809275685, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07607281397640181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05914788345848277, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07618690317795936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05806983274815348, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 66, + "hessian": { + "down_diagonal_mean": 257.65557861328125, + "gate_up_diagonal_mean": 1500.681396484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1070173475348943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02218452915391954, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10724990286710447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02235584420236071, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10718798135063659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022067963756158472, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10725746124793835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022419965477549146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10706151707647786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023881775809993615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1070515615320003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02380371610825606, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10723865857680419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024027975725296094, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10715593922771084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023938300786825167, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07962742301119531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05274474658915931, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08031099402313256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050791028649104085, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07982390417470574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05280812868377436, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.0797297063788763, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050504765255454144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 68, + "hessian": { + "down_diagonal_mean": 180.43740844726562, + "gate_up_diagonal_mean": 1500.6082763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10603515627677919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025657007106810282, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10559870284882261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0258906701043936, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10593643854690182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026160920440886706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10595949698205263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025928978810816836, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10586092981978347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02872924160007602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1060009005205163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029186591311500157, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10588446943875277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029277423716715247, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10593700795501644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02946016073622149, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07817425186477421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05489800261628499, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07762827410265104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05531924170212293, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07714205706321492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05708058919298833, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07696315289894148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05713770071401491, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 70, + "hessian": { + "down_diagonal_mean": 144.60507202148438, + "gate_up_diagonal_mean": 1500.4957275390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10408746474861592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022821597367623236, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10411917011890612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023067754289660562, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10404503739901048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023009901857335677, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430286083419148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02267359429829215, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10423069377444215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029981903408700408, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10394158845828602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029796305577159673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10402325611144983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030225970353346312, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10420472621683922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030160343741479783, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07687998300310249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05615546895527545, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07736653958459147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05559455607471807, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07670573975114228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05709179318779471, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07678450577991439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056615123124453076, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 72, + "hessian": { + "down_diagonal_mean": 376.4422302246094, + "gate_up_diagonal_mean": 1500.42431640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10715653821361985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019202644781187325, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1070619847250222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01850922045115523, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10721397936883227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019017829809289337, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10738463120438424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019008873047778915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10715774854994807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0202140138765754, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10732777180262713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01934619105890805, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10732606850542675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02034874270615183, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10714251972587549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02018605952699478, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08430332653393546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046545020101064115, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08460311873238678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046343343961131434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08463468438015925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046639076764747066, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08420434943678672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04726398499777278, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 74, + "hessian": { + "down_diagonal_mean": 311.0207824707031, + "gate_up_diagonal_mean": 1500.652099609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10671681223759416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0197407835706271, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10665442379016613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01994140418134862, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065387305829835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020060659031993642, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10664209588505652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020059448088627543, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10670350896187107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021939365883210417, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10658928560318641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02127490695811658, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10655027747342839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022155225082175312, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.106691431116977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021760304423134636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08200622613327783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04798159965957245, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0820883155786416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04702578308009506, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08151663778240116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04965139737015754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08184351976827582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04855553742591207, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 76, + "hessian": { + "down_diagonal_mean": 277.6690368652344, + "gate_up_diagonal_mean": 1500.69384765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10696607227892507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022176689756093007, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10689601004310371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02177797167914608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10688470366943938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022068715062121047, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10689075538350062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022119828584460714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10670771997474034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02446018655756581, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1068645619093769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023292122118017428, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10670106600029391, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02458971596613391, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10710510122577155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02422519818511231, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08104221996523892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05032252379439605, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08269817932133859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047322489829458574, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08207736029117874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04951076394008404, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08244621279855979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04806326796646366, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 78, + "hessian": { + "down_diagonal_mean": 123.8992919921875, + "gate_up_diagonal_mean": 1500.840576171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10575027750908138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02302218577032623, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10576102984461011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022821522096990416, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10574307752790149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022995193902290163, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10567080843312149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02274528012755228, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10575395400269799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0285603016984565, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10579379515598604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028435427352502124, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10582538175565033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02951937150774913, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10589230271516521, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02935793857667035, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07555661613748182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905946027661068, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07565793895335164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058466252221824624, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07522453799505426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060016151147253116, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07549256483631726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059835100197516936, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 80, + "hessian": { + "down_diagonal_mean": 172.62384033203125, + "gate_up_diagonal_mean": 1500.412353515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430009244387867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024823677649644167, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10427231180093828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025216281115888202, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10435302974629594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024975261556425585, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10437538874808813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024069194160346762, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10426185492377185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028598135961985995, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440033990986704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027946599391390443, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1045446103486881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027713145635645253, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10442725722720621, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027595637023192527, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07650326290718489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0563713178650427, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07687032962911039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055425645971693735, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07706778737906446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054837827793132884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07735367168675261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05353254383680546, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 82, + "hessian": { + "down_diagonal_mean": 178.65036010742188, + "gate_up_diagonal_mean": 1500.440185546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1048497552723575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024874669225030246, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10458456974998098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025128995297117093, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10456261018480918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024946960592898318, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10464800636458602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024717903373611724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10468270777682108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029107107164482247, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10459546035361222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029286514950855585, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448349548672795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028716929645375422, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1045183264004205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028766196057858868, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07690276908861682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05594582723961153, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07644185587906176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05629165873532026, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07740029717639584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05471808462890743, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07677282130063211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056251097810685144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 84, + "hessian": { + "down_diagonal_mean": 108.78034973144531, + "gate_up_diagonal_mean": 1500.575927734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10404587763915137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02400617211884108, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10397714067021514, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024155236525580163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10375696131678527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023796998232449823, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10390399525151747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02386426599739999, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10391231946034377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033602166796753576, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10383675771465167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03361920895697989, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10404343983623204, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03345273300436587, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10390754372778534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03344100875368997, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07377664579062193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06243101075077602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07378281255328598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06209497085180327, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07385677796148177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06211751122871891, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0737805050279145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06209912870101432, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 86, + "hessian": { + "down_diagonal_mean": 123.57357788085938, + "gate_up_diagonal_mean": 1500.354736328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1023014786545914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02474932065785769, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1024719477341391, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025123145062287897, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10245656669154429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02482484441321718, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10217862236076772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024627139809410892, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10237105108686378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03273285741106349, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1021538331623592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032409957632344936, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10242033120062556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03239114049810877, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10228316749522137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03219691031777288, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07465649587915237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059312955971685365, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07486210981846471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05891943680470649, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07505310904575749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057302909859769435, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07517079272099127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05799542173827477, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 88, + "hessian": { + "down_diagonal_mean": 191.77914428710938, + "gate_up_diagonal_mean": 1500.67138671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10629046363797352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024887131209001306, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10633659505374855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024776894481278226, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10612522788843497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025021251541860554, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.106321488140127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024768276858462883, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1060913890561759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027129408861538678, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10626895467326557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027067644706041637, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10624631924967441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026910005608555637, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1064883931552617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02749554852516281, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.07818834381464018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053779501127557645, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07760622403560567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05585347438320857, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07772872226053411, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05417037844076778, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07745248860404878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611119091989267, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 90, + "hessian": { + "down_diagonal_mean": 202.57078552246094, + "gate_up_diagonal_mean": 1500.529052734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10577440650725596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023989720504515904, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10578362307056599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023848392370537857, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10579765273669586, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023808660094861347, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10566654577845894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02380584627330074, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10571734080152528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026841050256828175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10563195832346879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026705837599050684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10567347437137885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02706350755098326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560697437552755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026168086941179138, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0777154024940413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056070292542040025, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07794149716668894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05600655768959173, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0777416315320773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05625864830286165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07794496717999011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055454114569225624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 92, + "hessian": { + "down_diagonal_mean": 212.96243286132812, + "gate_up_diagonal_mean": 1500.547119140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10569759721419715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024169655661619778, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10568528236634292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02393897692661555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572244430915362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02397000756818317, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536947125546052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024191405342987072, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10533210255553721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02660993239340191, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1052911185190067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026617705533927706, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551107780710986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02677581056124166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10546041772325856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02670247671415736, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07777087695519891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05510650338743616, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07807107154303788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0532299324612302, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0780420549853141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05167239390273741, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07761872394552834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05489436584223698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 94, + "hessian": { + "down_diagonal_mean": 187.2491455078125, + "gate_up_diagonal_mean": 1500.72314453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10686794374693989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026652541568874297, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10698731552600006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026621708150015605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10687410410966412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02649811750740059, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10696312170357754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026569204834778915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10675178899890446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02854247294310226, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10674463049813263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02870480955868938, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10677660722388482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02876498229759604, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10682553817067023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028230843823899822, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0760083102577957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058615498334226225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07625800771598548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05879261710027274, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07618259314914938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05897219272180413, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07633283964923011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058272682213636476, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 96, + "hessian": { + "down_diagonal_mean": 169.93353271484375, + "gate_up_diagonal_mean": 1500.748291015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10561741085781745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026769131060328934, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10559859096488029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026619099534991402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572076972447038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02665982212183294, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1054913167856112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02690506916760901, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10569139736263682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03016115590092906, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10543004344776054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03041325072905253, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10553721597647955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0301655234610086, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10550160996981235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030517096485088973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07609172688045668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0586645171750136, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07591680372324812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05901738340730004, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0759215926053432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05855274364208113, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07552642787726838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0595705585735102, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 98, + "hessian": { + "down_diagonal_mean": 140.14590454101562, + "gate_up_diagonal_mean": 1500.5927734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10465776997642041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026156438263010468, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10461890728986102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026043891086926466, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10465268594235407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02591899699172916, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10468533915416835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026043240029767278, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10448791052517699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03170860578880128, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467391743059097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031832984951009786, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1048396281344068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03157731502305202, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10466225937068097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03195033690025943, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0749666078483239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06069321981110573, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07496264195292655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060493804058504914, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0752132421001556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05944501142529273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07493796937535648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06018628967885951, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 100, + "hessian": { + "down_diagonal_mean": 151.05682373046875, + "gate_up_diagonal_mean": 1500.553955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10493825825612546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02060529161049941, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10515227797301503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02033554783853734, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10504743953991075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020370386367861523, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10516402455609256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02046112550479812, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10483070497451347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025832127490427313, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051175256113813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025594560381260418, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10520196212382041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025833770424144076, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10503045848306058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025492454036042698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0784509770975039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052354226466728986, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07856976358747245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05273041586343421, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07801408315767855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054416258758372166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0789747642417983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0519797088975036, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 102, + "hessian": { + "down_diagonal_mean": 245.92813110351562, + "gate_up_diagonal_mean": 1500.65771484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10600982385105348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02169742697992187, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10622951611336831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02192638444331175, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10611631930303667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021568895444447483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602362223747154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022456373269558693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10622878618686891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025310849273869566, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059512548542076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02511383076107103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10608513916606382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023784227018196553, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602392870199337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02519635598889001, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07920044816557609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052988095019364384, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07882482380052622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055518389242986084, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0788518644566888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054288641511324995, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07796556977090931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611464908672082, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 104, + "hessian": { + "down_diagonal_mean": 211.95281982421875, + "gate_up_diagonal_mean": 1500.8316650390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10822568810646774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020252518027466106, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10833273099400387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020155619004798408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1080501855419674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020227263927419057, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10799047693023335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020310845032443934, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10799737094144049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022659214939325375, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1080805206829086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02251419127026587, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10824103169504373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022269611086856312, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10824168909909489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02219724460062509, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0803908319710952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05171264747216878, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08060683899961457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05078301478151619, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08011247824360192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05177945705367472, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08020562903563122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051603286360487825, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 106, + "hessian": { + "down_diagonal_mean": 363.1668701171875, + "gate_up_diagonal_mean": 1500.663330078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10681690800558792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018567777623578866, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10712263406349229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01878376930760543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10711814052102417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01830366477403051, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10694874085574675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018380357155778283, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10716225873069216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020530960431288502, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10702267115020053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020856768954820464, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10694087013611651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02047554329645218, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10710397384266834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020002502563998062, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0831555674828005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043465204662151036, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08274174795288858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0458106782848052, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08357933919367182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044467724748097084, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08413540957584212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043371000305480606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 108, + "hessian": { + "down_diagonal_mean": 357.3666687011719, + "gate_up_diagonal_mean": 1500.326416015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10757004326123477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014095984702867307, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10775085516639255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014409316730457002, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1077893908187586, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013836754847907959, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10790023158703477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014848103309532465, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1077734769436762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017178037890747358, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10778198021206084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01789485660423678, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10763339300810508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01689775396026132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10765595314628773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017846279215628813, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08363407338467753, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04447718554192222, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08420359102560952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04631536159277691, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08486135073443275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0438607199008232, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08319081228454536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047034639338019275, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 110, + "hessian": { + "down_diagonal_mean": 222.55645751953125, + "gate_up_diagonal_mean": 1500.560302734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1065545672802503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022750372322790015, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10649054542111935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023148963196816846, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10662324179646185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02302895398216532, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10630513497900161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02285046499740021, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10643081786436778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026575276629616205, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10652413830106909, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027093263697092952, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10630568027107387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026928292249108584, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10646091094945272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026922839476000902, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08045137289702335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052544313966016616, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.0792984314969991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05567676723407408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07961204914652655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05434235333805389, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07922481348450354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05496016764159641, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 112, + "hessian": { + "down_diagonal_mean": 129.73272705078125, + "gate_up_diagonal_mean": 1500.677734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10359351448028413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02824983492566681, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10385549872214654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02776207007386865, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10374087946253836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028218095329696852, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10375211673525309, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027987421630766358, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10362329136993761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03443954519400799, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1038943226797134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03459456888098094, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10392254189729822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.034507050788822484, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10392700414553027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03449485806231217, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07328011518344957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06321819035339686, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07344557254858733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06190771029424288, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07348363393225957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06270047735038516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07321788292016418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06337702810963981, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 114, + "hessian": { + "down_diagonal_mean": 232.77923583984375, + "gate_up_diagonal_mean": 1500.6671142578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11067736157973308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018399358385402787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11054198200359762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018209051823056033, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11060168117746653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01847592972681238, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.11072045629942542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01854873505095295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11047401085016459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020028884775040258, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11061308098480212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02026158819968052, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11063914382126248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02013941442763731, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11025820570828797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020392030337184334, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0815688038392119, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049535286588609465, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08164139085419837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050189393782367094, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08140282380819482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05129291146718176, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08120588627464811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05052505922815678, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 116, + "hessian": { + "down_diagonal_mean": 295.8703308105469, + "gate_up_diagonal_mean": 1500.67626953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1064109868604124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019289599117007318, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10635607371920822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019352562573597593, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.106256936486178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019588944763271797, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10658596851428725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01927104997341357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10646989893802693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020877935191414676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10652772860049506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020593704490901046, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10647241138900347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021334708084703138, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10644278479310061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021211082473086645, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08120396336390648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04814792906341654, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08162689122442808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04685601739556322, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08131053648536544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047931436985685395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08167964075711082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0459288005179049, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 118, + "hessian": { + "down_diagonal_mean": 262.51654052734375, + "gate_up_diagonal_mean": 1500.8040771484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10651566986341464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022986539271589095, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1064538878062335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02300178679730591, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10656547400203803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022649121036887498, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10652213049242491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022910109949809448, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10631880229437916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02474436061549296, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10663917732525245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024633015253566055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1063241267291962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02477297328246724, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065098462719513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02498470779997787, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07975822572199263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05264135242943344, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07874959179585324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05464647722741521, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07949375171525193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05334513999303747, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07944046484030379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05378504742354321, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 120, + "hessian": { + "down_diagonal_mean": 267.71527099609375, + "gate_up_diagonal_mean": 1500.695068359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10919399040225566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02031175585771047, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10910864712584464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019961372375075975, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1091594761307919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020100967775482502, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1093644283251387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020072636916714834, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1090958898036428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022466304030513133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10923790653809899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022425265713888948, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10907078947420941, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022412041559413114, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10917482304597616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022209336438923758, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08150517676535615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05163040243755599, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08218700171147104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05122375947718915, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08185445604330073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05157909903470433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08244434795917813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05085605965352706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 122, + "hessian": { + "down_diagonal_mean": 255.05789184570312, + "gate_up_diagonal_mean": 1500.6458740234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10724778517351578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022296476059343453, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10718750411299385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022324047222689065, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10707503276888267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022340315338834044, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10713112894960235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022099618409669425, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10723605351225585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026399640685938464, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10704477312465921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0258039215633223, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1071065139753558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026137694862394315, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10697501938110228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02603148861345296, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07736605589320074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05796595700031731, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07737991634101525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057976851107597656, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07733527787842814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05782231234583097, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07735641787753095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05792408099275488, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 124, + "hessian": { + "down_diagonal_mean": 95.88993835449219, + "gate_up_diagonal_mean": 1500.496337890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536142992723818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020882843833535367, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10529206388316072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021041732705107733, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521104106511366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02069945640222694, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10545475758478236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020908420464296303, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10537921766542184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03171232787399207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10546113134510626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031843820937277625, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1051593530680919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03182588203943713, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10531981559513794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03193198427565606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07411539674556408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061707377123502, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07386959212394525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06197972812564318, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07394573134863865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061755950252146044, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0738868543242179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061995616244512664, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 126, + "hessian": { + "down_diagonal_mean": 209.5216827392578, + "gate_up_diagonal_mean": 1500.719482421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10551897040176712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023898628820252066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1056863495422155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024276156769134968, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10570328429485612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023875924897556824, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10558598749830561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023774374663556617, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10545757240731349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027241483938632564, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10561945747865355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027355993037191127, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10578834955806987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02684845372907898, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560952957123607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027355956674774112, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07782546623675012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05556926472482303, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07776205875490023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056904626200368784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0777441484331203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056784041376115105, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07770657965264466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671319398377911, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 128, + "hessian": { + "down_diagonal_mean": 211.95388793945312, + "gate_up_diagonal_mean": 1500.5003662109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10469962689623578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020946981633615372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10461730372136444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02081278261779015, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10475858386055367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02047310284075898, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10473865440129734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02063211907571669, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10473352930226953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02706518102053967, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467528730364352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02602555013491281, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1047546418109158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026310447520832014, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10463402272129649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026184443710282227, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07961757571707567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05133476853292763, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07959034968510226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05144439485219631, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07952216600840735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04996083248096188, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08040581138194236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04777641809418263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 130, + "hessian": { + "down_diagonal_mean": 177.39439392089844, + "gate_up_diagonal_mean": 1500.478271484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10513415950814409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024169047862693836, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1051981154006227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02437003082929795, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10491088334284525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02401209895814225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10512064609389353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023995565243592573, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10496459228401045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025503608096845425, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10511229875146354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025828624124690516, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10522397334751385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025747517021190037, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10514491874671128, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025314223732945104, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07823011048234003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053809827791035346, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07755677329338131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05578292927389088, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07781574289903848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054876166998557914, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07824871994720713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05433786317778919, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 132, + "hessian": { + "down_diagonal_mean": 272.2606201171875, + "gate_up_diagonal_mean": 1500.633544921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10636784467271314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02352870933245859, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10640680346652465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023305951438602864, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10636540742764511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023432645389306273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10648143267280658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02328100129128618, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10624463751161048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025721990619868726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10624163606914701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02537283865403518, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10660866103090054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02538843578306395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10623366598100774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025138534296519348, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0781962175165708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05539711742190548, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0785535667615557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055338392129432604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07809646911814085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055181199054733676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07880090693196401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054211917217878915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 134, + "hessian": { + "down_diagonal_mean": 183.140869140625, + "gate_up_diagonal_mean": 1500.5633544921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10451606478425701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02654336764487652, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044951303636802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02636824659469526, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10461059558492947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026626665686226528, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10462329974598887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0261689128906504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10445346504704954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02973354901714243, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1043743620583408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029374177378092035, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10446256522387182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029826520187991264, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10456641081315135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029529765611823005, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07541048453136512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05997532648186176, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07554856753676772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059858404555366576, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07560704205350724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059896226992343274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0756994424691713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05950439156896002, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 136, + "hessian": { + "down_diagonal_mean": 197.1099090576172, + "gate_up_diagonal_mean": 1500.739501953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10587099278809885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027003162725227636, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10585783440002734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026528274034824096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10577906094466427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026759645923549517, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10593276451725309, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026434338652400007, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10585478144954488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030129658111453184, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10575444037806833, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030205546437045414, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10591925462246297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03015350014106641, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10557743281836297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029960009782808616, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0757069836241715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06008595505595645, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07610051513831796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059492626673347905, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07566509890490805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05967626369802113, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07616214326541786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05909971760618441, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 138, + "hessian": { + "down_diagonal_mean": 184.78372192382812, + "gate_up_diagonal_mean": 1500.703369140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10632759386340654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02559412522402427, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10635835165659101, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0253527293437788, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.106186104898887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025470187404400105, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10639344231245308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025716137996144327, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632232636506862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028732292745249237, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10627054245999724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028510542695465543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1063764949443772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02866854449084689, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10635476047433343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028483339412707614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07667730830603867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05860581747269591, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07729791662465854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057833318213448004, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07715213720126142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05800664282670084, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07685111616052942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05841918261863771, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 140, + "hessian": { + "down_diagonal_mean": 333.9516296386719, + "gate_up_diagonal_mean": 1500.5927734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10718355036089106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01884005767721514, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10727415085294029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018650344168463583, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10724218160696342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01913029195505049, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10712716345680705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019269033294035346, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10710478517103454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021363853604269845, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10717080607346859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021268373975615876, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10718772818041358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02175522909686673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10691059834319708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021958357753518315, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0838509615291279, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03010197799603026, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08443250412496606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03297019568980242, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08262817234054234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03668385166648386, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0821832853209478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042713359977489034, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 142, + "hessian": { + "down_diagonal_mean": 103.98186492919922, + "gate_up_diagonal_mean": 1500.511474609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10323876676216472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02480310652617562, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10343870955988063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0244636263972644, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1032565328878977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02459302274798627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10338903909555062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02458346848870978, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10333806695759058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03456371534368432, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10331866071339234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03510286746227609, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10335417296167153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.035392371297323436, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10328104823880263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03535348558653052, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0736714565181199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061162023123401923, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07353108136153529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.062100945829434255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07335997429269699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06233100137686761, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07372507969630115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06151408824120288, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 144, + "hessian": { + "down_diagonal_mean": 151.36190795898438, + "gate_up_diagonal_mean": 1500.612548828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1047695608463034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02555785078642457, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10506527993143994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025471587862383998, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10504350929702856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025519122210476208, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498976323479302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025784459761306836, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1048283196721879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03066831385168283, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482989147866631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03080090038487654, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10475835439483858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031185708556019348, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10492047047260229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031068156600823092, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07621071581287524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05768031195064017, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07591424257515379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05844662200661635, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07564963624535512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05857364438916185, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07566922252468676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05880416717805273, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 146, + "hessian": { + "down_diagonal_mean": 213.73228454589844, + "gate_up_diagonal_mean": 1500.6552734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10586125967718042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023094748067420946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10589427340730861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02302585901632641, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10605499246587023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022991580964777512, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10585323297685209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02294139788087616, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10581736941070227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02679638759814808, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10578791624999112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026749885922292386, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1059070962773242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027003577462347772, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10577273666517015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02704827155988346, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07819252606965076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05472493593949607, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07858511148501705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053874140249530786, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0782258767031813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05413015204716036, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07791555471651687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055258064895978426, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 148, + "hessian": { + "down_diagonal_mean": 124.9146728515625, + "gate_up_diagonal_mean": 1500.666748046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662163831046935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025634811796613408, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1066947059938793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025633716882451293, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10695697772201421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025636407991334584, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10679639520557441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025688545019564357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672568436683423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03240780927730499, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.106736660538847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03271630957843917, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10687370589106708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03236050773774145, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10654101812703312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032396268896680916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07516330061741656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06021402237676769, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07479888154027635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06022298519686294, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07498591025605456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05976019977323204, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07483238970579442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0607494926677535, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 150, + "hessian": { + "down_diagonal_mean": 244.71160888671875, + "gate_up_diagonal_mean": 1500.6405029296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10321105528947133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024639965442751353, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10323835910510261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025041731685956792, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10342311673790006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02537482821880809, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1031879232196816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02508079126092477, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10336182985737094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029729710556596255, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10317688861256971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029646669615348155, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10338024049607604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029784773563396486, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10319150275393545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029776879478640904, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0824592745179791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04198526460505602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.08163981624977829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04676684427175452, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0814207144772695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04657498987840132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08228765523046125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04451865391973995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 152, + "hessian": { + "down_diagonal_mean": 224.85894775390625, + "gate_up_diagonal_mean": 1500.2364501953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10542513797879074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018916293740112323, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10536067076233342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019657355162732766, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10544285453529574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019578506541611237, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10540604544646058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019383017659230534, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10526469332284753, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02021165900494946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10516463409375304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020828535739963454, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10529018414225459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020257752718982707, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10537096565847742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02037249886835481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08108234386670732, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04637655478642739, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08083766659603511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04936494542001062, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0812447957492736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047757782465131245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08153491213770228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04686700046584068, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 154, + "hessian": { + "down_diagonal_mean": 306.49444580078125, + "gate_up_diagonal_mean": 1500.751953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10820277791086873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020918040463764033, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10823911323246214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02075164242284744, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10840989702343455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020878601352658826, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10830921027635548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02101608573876626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10819067027607802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022701077972724203, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10810237675984698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02253438551144334, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10826375397938372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022675326047786267, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10829102988298396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023068930477311345, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0813182152743174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05224792795792602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08130191640939227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0520071931069826, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08101231578061201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05232493208686048, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08066744130170969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05358550865762725, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 156, + "hessian": { + "down_diagonal_mean": 193.938720703125, + "gate_up_diagonal_mean": 1500.76416015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10616196936418619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024143969649578097, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10613786092258515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023891057135383458, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10612542831350752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024186322453130543, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10621641664222503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024019364336580983, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10614292053978236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027186218865226233, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1062101665117087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02658211501612838, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10613241483956709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026828160854873877, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10603615530070509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026570912969681688, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0775631691748929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056003555206028816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07808734070252683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05484971597167033, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07799986821659109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05484450803037884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07854680548987372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05400312304416727, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 158, + "hessian": { + "down_diagonal_mean": 226.1881103515625, + "gate_up_diagonal_mean": 1500.620849609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10557463131446686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025422712508986155, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528727612196771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02523556855742109, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1056465059307046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025385490619657874, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10550639174386263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02542905223309519, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10538271088855708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02683185557929991, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10529134232065392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026972271297807236, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10538033087086802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02709925795564613, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536389500305701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026845383677603717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07809873837169275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05487483264734287, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07846435646843716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054030186268896135, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07775261247750793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05563850728655025, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07834303699555413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05485369737292049, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 160, + "hessian": { + "down_diagonal_mean": 122.68082427978516, + "gate_up_diagonal_mean": 1500.361328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10400292774362437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023441837242686343, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10381053177567547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023642607522683053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10410990375146355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024053036560800566, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038363855556984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023891315334679048, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10395792561736991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030373805563429328, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10397036885906272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03037838846273303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10394317279480124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030311300136328365, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10402276544281755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030541194007119084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0753170859606345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05955213447471285, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07534643474370302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05897669040641409, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07504512340385859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05976519391711132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0749547094411011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06009324336504571, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 162, + "hessian": { + "down_diagonal_mean": 188.4763946533203, + "gate_up_diagonal_mean": 1500.457275390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10538108135674257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02496727813376561, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10523258160916742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024912408658866105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10548983548558327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02461177982455644, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10553128071385691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025291859381291915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528366419226617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028199225136307647, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536154480459572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02807675666411038, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10523295703791684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028009274264616153, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10531404550764871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028212256864558156, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.077118015990614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057879064861015066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0768924780535857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05847792466404702, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07733859012328663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05742131619175088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07695534184267316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05834043451468259, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 164, + "hessian": { + "down_diagonal_mean": 103.4560546875, + "gate_up_diagonal_mean": 1500.570556640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1033652937681524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024262478488387605, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10322424630943107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023737791528327484, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10342379458030718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02440643484070697, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10350205227067262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024318030608441836, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10342171919802762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.034451893378849886, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10330917272960123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03427787121702056, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10320962909426731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03442648634541825, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10328545456172018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03426054262833712, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0737496984096834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06193983954245978, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07378743418977744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061730002913803116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07364837867507658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.062092374428893204, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07380657116749334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06173380148098062, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 166, + "hessian": { + "down_diagonal_mean": 246.6381072998047, + "gate_up_diagonal_mean": 1500.5457763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10474123525438071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02537335196872421, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10460486126193712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024924510335226774, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1046203630135908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025026513866638844, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10467195232362346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025320187434586548, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10460747352559767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02967623752999194, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10448477937055341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02988668184154797, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10475974781170402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029142328753893137, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10481395152783729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03000135674713181, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08056755697852151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044693780312461864, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08164612725996803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04314533997013615, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08352342887906876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03677800145992848, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08101461246315217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04371334556056261, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 168, + "hessian": { + "down_diagonal_mean": 273.23577880859375, + "gate_up_diagonal_mean": 1500.723388671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10650923717150834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019615519830098307, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10656352530766146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019542772723249252, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10649397763478514, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019585836804072446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1063819811396039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01969887295065921, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10648655860500156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022077824417983945, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10648054427508795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021356714763089923, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10656210463533235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022213022903676233, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10645887770826051, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022358143689907767, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08200908893652488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.040508468423238804, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0818791106372345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04511592443617879, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08232739107585411, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04387560851084524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08202237320162459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04553831030265298, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 170, + "hessian": { + "down_diagonal_mean": 320.758056640625, + "gate_up_diagonal_mean": 1500.7415771484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10612671162564359, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024409956306029004, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10617856996559648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024357425105521753, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10591735107326712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024337194475130136, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10603117602374569, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024155810837268588, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10609144964595969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02665267132303427, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10624247886799029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026241511498110708, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1060009674895111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026473481640369584, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1060735472437936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02588154829134983, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.08149500067147425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04791922626468928, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08201642941182884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04841285005837837, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08143388537928788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0478922325435595, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08239178209022034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046519245468610054, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 172, + "hessian": { + "down_diagonal_mean": 351.72979736328125, + "gate_up_diagonal_mean": 1500.6824951171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10998429860105678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016593884161453767, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10998588252793239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016731538052202972, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10996393757259565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016750615645815533, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11000432992108429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016975773380136356, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10981321673694365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018167207403575393, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10955060850323155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018813803111364824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10987203292415533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01908138412233005, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10977628256741169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019133227761418133, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08622763967195844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04149905426529856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08507320728419462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041911994791994006, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08553359266491468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04134063581899157, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08584672055525074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04244255017982338, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 174, + "hessian": { + "down_diagonal_mean": 265.4698486328125, + "gate_up_diagonal_mean": 1500.70068359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1061703456570688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021111474448054245, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1061672953797873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020708507032473115, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10591108613741985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020728581874302803, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10611904174905076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020887598828218053, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10605498179094072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02469423226228867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10599247727920448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02467374926628729, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1060372743591707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02423023669710253, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10591662700504668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0246036373873927, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0828580202808573, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04985882526505118, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08393877128108004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049194230716785185, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08355007548197323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049049687972342326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.08392694113101844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04964218781153587, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 176, + "hessian": { + "down_diagonal_mean": 165.01446533203125, + "gate_up_diagonal_mean": 1500.5762939453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10477977856988635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026731100674931334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10466915588993579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027067059881553928, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10457609669562445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026878880532271332, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10456740638035167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027272387792929235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10464743863878788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029145227061863385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10465500688465135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029357673696471662, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474364867147991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02892681175634752, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10462254214548537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028773488073749902, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0756815681369886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05862272966059434, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07529480078741685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05944137764120166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07611082799407186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057951019788171285, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07555527014156758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05879731126838868, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 178, + "hessian": { + "down_diagonal_mean": 233.97918701171875, + "gate_up_diagonal_mean": 1500.756103515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10536954773707008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025295967070486713, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1050137617547068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02533339187089123, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1052838426905523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02538563525894576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10516036468383602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025424720686244066, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10528801214354218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02774329456569878, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10516255843755169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02760119634592183, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10508384480961157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028036796830539945, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10506319985605592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027903174444689153, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07920582682155441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05232237417001508, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07904137193754576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051910956691025585, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0787424218772253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05333844776159431, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0788446589759256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05061244399334556, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 180, + "hessian": { + "down_diagonal_mean": 166.5069580078125, + "gate_up_diagonal_mean": 1500.73974609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10636398020080338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028793770488459797, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10617089414065345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028771144069485284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10624584477457223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028750125731666645, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10634603985829365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028647759952814704, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10617885764285606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031089211920765872, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10605219741339039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03129687056667358, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10643180171541015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03153091535903815, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10628223141598775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031400264619110094, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07566881103174215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05877650683143017, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07547497592607168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05932315970957255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07526012039202636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06013965424661696, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07548732371528181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05889701899154621, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 182, + "hessian": { + "down_diagonal_mean": 262.308837890625, + "gate_up_diagonal_mean": 1500.713134765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10585561230028895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02385005112478361, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10595305426973198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023596492013076082, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10600324549161097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0236693458100955, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10579344916726272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023644722054772336, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1056752450045549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02523037269578328, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10580107525440566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025145904151966315, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10599186201388812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025203011859410998, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10582686867063419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025031317935224968, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07942280959953354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052616298915496455, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07983046489325492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05212202662315769, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.0801117400022613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052279015398296255, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07975742385185759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05237065406145916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 184, + "hessian": { + "down_diagonal_mean": 102.79817199707031, + "gate_up_diagonal_mean": 1500.395263671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10492622826595238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018103278628049918, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10493110867952861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01846268595698801, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10477313846811283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01824460243964747, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10486059779701129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018536189154415152, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10484575445978216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029090136745963415, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10484294519034039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0296415059311486, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10504761211629009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029529519001156034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10473823536871417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02970976378983405, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07502795887826759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058730850634377975, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07495882743331671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05935124318304459, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07510057798513609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05882726668075372, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07478392900321013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059384037012185226, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 186, + "hessian": { + "down_diagonal_mean": 180.6925048828125, + "gate_up_diagonal_mean": 1500.5418701171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10667179547283534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019886445167276952, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10666199191935896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02063080306724885, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10680281159953374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02023716588382784, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10683087419060058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020072980036854402, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1068001411496156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026041620674545574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10667968374530702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026122708842500487, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10677174295469223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026644720296608, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10664417899611431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0264667836332419, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07995210954562004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045208763374708624, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07924306954835594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04591050852822173, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07878136353412081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050190011033898814, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07978865520977982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04622628978111532, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 188, + "hessian": { + "down_diagonal_mean": 125.63077545166016, + "gate_up_diagonal_mean": 1500.61669921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10518694086601288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024879751836146886, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502697252549686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024940119714440916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1049355943438855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02463754073632112, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10509859317709543, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024650972075507035, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493842024690997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030807040187723095, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502204254900208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03073001843878439, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10491013720304135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03071038198480015, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517098406083719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030716427156370955, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07464446386088314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05997319604371633, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0746602030465275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060038461417958704, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07495771326422558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059328167712621754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07495732736680857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05922055651625298, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 190, + "hessian": { + "down_diagonal_mean": 256.19427490234375, + "gate_up_diagonal_mean": 1500.658447265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10723266394874573, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021860468539766385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10709662136771048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02207307207832217, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10740648576393319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02182805935490625, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1071833162329754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022145706947441616, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10711069360860807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0241849028094736, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10712365516312837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024623242041409688, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10709493172669637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024147276473724017, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1071373955006925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02437068014960084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08047778317955481, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05252579397502212, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07997000017003765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052979770474190856, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.080848036282041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05233889975096027, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08044483469878425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052399298755952936, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 192, + "hessian": { + "down_diagonal_mean": 185.65475463867188, + "gate_up_diagonal_mean": 1500.40234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10673983468866716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020180078301027266, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10701021096278304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020383402523519906, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10686716910639282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020455854152187605, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10685721762240248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020047588138295384, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10680941184237172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024206687265424107, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10705977056306092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02505763895447648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10662595352638712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02483166281372322, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1067965870574658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024954057209003833, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07832533148508897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05430638724534424, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07797527666903702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056064334133739756, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07806368226174658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05573907907962225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.07829707792110373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05287454982224951, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 194, + "hessian": { + "down_diagonal_mean": 165.894287109375, + "gate_up_diagonal_mean": 1500.744873046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1050862045538252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025369090678507806, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1052210923189659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025287334771420488, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10539297493870231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025510816453781202, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10533222099374505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025593204396173255, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10524669556739819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029990281241646637, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10519725933251045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02984210880768108, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10532813172580789, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030271452473502627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10523132984629809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030456724929423033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07627122493742833, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056239231075303375, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07677570177619522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0538424668443814, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07615561438838302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05651378425485677, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07645183887568437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055691295834379026, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 196, + "hessian": { + "down_diagonal_mean": 166.5162353515625, + "gate_up_diagonal_mean": 1500.457275390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10375844440467297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030198731042599052, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1037604945729567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030023811095674324, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10378199075479849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.03014358556988015, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10393753313516778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.03044652029603738, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10370482370929168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033171207234023156, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10363429405865028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03278238854901815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10351309370821966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033075730844049946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10372514547654822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03293629667499504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07379643348591577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0623723407496381, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0741505643073533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06175389995155789, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07393658561946151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06202528575181572, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07366026592372694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06257483386148366, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 198, + "hessian": { + "down_diagonal_mean": 202.4113006591797, + "gate_up_diagonal_mean": 1500.697265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10748737453176743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024168644414245575, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10762222894746849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024040520092061163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10776855545483287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02367119837425842, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10790464545930809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0239804676107533, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10777617377143371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028251342503639778, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10749904713774028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028427691879155812, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10761627103428083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02876023990350571, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10755658106105401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0283936236661503, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07924992136629255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05574958428774008, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0795977171503319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055079981668256284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07945449702231583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05568319479155822, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07915068521446761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05629481418091768, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 200, + "hessian": { + "down_diagonal_mean": 176.5468292236328, + "gate_up_diagonal_mean": 1500.84130859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10978271115320759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023897467307268376, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10989538142728902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023846371829096272, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10964561241580142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023919617896535975, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10945522232807485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02377228342437492, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10941219516504241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02591414747013112, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10972303414704711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025562960989626207, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10976124189079932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025476336246867837, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10963911919379107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025529229793601917, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07750166226296525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05620660899108403, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07774159805346063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05609221044669074, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07806484247812806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055120577478619356, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07781842472812737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05536150793795072, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 202, + "hessian": { + "down_diagonal_mean": 271.36712646484375, + "gate_up_diagonal_mean": 1500.730224609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10721141725646678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023218171707437642, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10717736040171602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02317034067707033, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10741317685994352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02287436880580508, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10736752710433864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02331127461012327, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10715294584070677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025258983986783654, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10733786038564312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024865579176875336, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10743100564756644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02517376237182821, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10730247335161777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025074604253660683, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08036986653514612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05363927072943572, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08021093528195541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053333294332398105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08059959800507292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05282959038823961, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07960192631647109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05386402414529114, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 204, + "hessian": { + "down_diagonal_mean": 139.52133178710938, + "gate_up_diagonal_mean": 1500.0252685546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10318717645470885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026789019198342357, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10331540494975183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026271692609230513, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10339550874095932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026365960227085043, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10343519540863999, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0264566275283109, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10326638475613245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032552478006709386, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10325618340777508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0327971602260621, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10315804289599602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032825988162640714, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10328854181436585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03271881865857565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0754696467486268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05898784914949265, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07530748525038786, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05903142931990556, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07492796174993317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06021490270601462, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07474692539838888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06038476563661233, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 206, + "hessian": { + "down_diagonal_mean": 339.91534423828125, + "gate_up_diagonal_mean": 1500.459228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11016576006396565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01902096073040726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11008232231290588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01911854510213777, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11032343134627903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018590873496131286, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10990806523557999, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01887569480662193, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11001482054420586, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021424583614993515, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10992720887254558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021106927173967047, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10983862847907658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021340960734716787, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10991969487406948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021730404606885474, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08229576323685489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049448152702825245, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08168975004341633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0495445501070184, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08232074858060798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04920183484157778, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08196021169955105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04996120053535926, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 208, + "hessian": { + "down_diagonal_mean": 301.5806579589844, + "gate_up_diagonal_mean": 1500.728515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10754894491339687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020474261349284432, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10794149110255388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020384291984574563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10760519892503403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020638819717442473, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10763075919566374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020625757754553407, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10764602909923246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024063468611912796, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10754093532984293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024185039162487137, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10749053832739121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025004100403250268, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10751147512672043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024278786266275994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0830477220728085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051926776664421984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08219232859958168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0535280072139109, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08219953630592504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05277929002892435, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08297774386716215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05197491927791062, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 210, + "hessian": { + "down_diagonal_mean": 159.98048400878906, + "gate_up_diagonal_mean": 1500.681884765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10572795479953208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026590790680465777, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10593330114888963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026547780877996616, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10568558904279109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02671405943013391, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10584761944681245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026627171292866306, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10572493445768631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03059268341184988, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10576568404896881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03024091486417325, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1057475971266858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03059792502617567, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10577472283797641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030410210438806477, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07583973157954381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058164838717700014, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07546946588060095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05877579878507871, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07580345485682001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057791597833135304, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07593964963049676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05704108707856547, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 212, + "hessian": { + "down_diagonal_mean": 202.99533081054688, + "gate_up_diagonal_mean": 1500.771484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10690843041297401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02307440315341718, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10714765592293525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022999239102466672, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10697142364118067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022993533383814463, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10689840770751666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023000616901041677, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10695587363151438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025170597329460965, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10701675241943436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0252922677830588, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10701645109084183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02518184153024868, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10708815108298309, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02538953787321466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07781240314773012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05625518802217959, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07746309852066506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056826456365610255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07782938333836852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055577519376157195, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07767369161962047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056140187880102714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 214, + "hessian": { + "down_diagonal_mean": 160.54885864257812, + "gate_up_diagonal_mean": 1500.7894287109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10706655158012203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022851153221929495, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10697912221027878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022813374016858134, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10717555322497985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023070823171976236, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10706722648603946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02296651979450633, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10710068446882422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02651204564730928, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10724626983410301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026159792091612618, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10701639078750087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026161498925603464, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10696362688832463, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02661965567348825, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07698199199355527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05680213907836834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07714452333941169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05653556572188939, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07718663174578645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05607926325308263, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07715563208541595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056719510936518475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 216, + "hessian": { + "down_diagonal_mean": 241.0811004638672, + "gate_up_diagonal_mean": 1500.686279296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10683696701368181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021723984415069657, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10669541646233587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021826417560162733, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065782031960527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02185211929354721, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10670901826742402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021795600865245398, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10671453418354884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024839582512809783, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10668706674577531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024639474375552094, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10659465021826575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024940552627970018, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1065688570630242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024704553537027805, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07900102161226476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0488534493165878, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07946017405614329, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04221852837863232, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07931969331229723, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048169313467014654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07865399725588346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050004504272844565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 218, + "hessian": { + "down_diagonal_mean": 169.29922485351562, + "gate_up_diagonal_mean": 1500.754638671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10415130718540394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02723760224767656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10407630772972973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02809328792486799, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10430371387522758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028201267558473464, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1040958682441185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027869644840282586, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416745788198914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032759031400346555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10408594165087395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03406632607236017, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10399137787266331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03377797741955023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1041373295489806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033741537283660944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07597221077011769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057559508331011026, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07554987578567068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05868382032570453, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07533278629869575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059308731105751245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07546105688854664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05812560090099481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 220, + "hessian": { + "down_diagonal_mean": 152.4496307373047, + "gate_up_diagonal_mean": 1500.6988525390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10634196631352069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02346433654460132, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10625907909189809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02278622087636481, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10614184259355518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023684830539424055, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10605212347449697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02332031031388897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10605339276302805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028942224586940877, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10614823195057137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0283267360013527, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10594633380505532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028976610119888526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10637090377822879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028813451402758383, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0763584369237675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057863990275023106, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07600338323943281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05771947817874529, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07625963234070351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05818790109108703, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07571728822665559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059426210769350916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 222, + "hessian": { + "down_diagonal_mean": 145.73245239257812, + "gate_up_diagonal_mean": 1500.55029296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10417941020169019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024479552307662952, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1043384902610353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024447686886819198, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10427129850573538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02408452926394197, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10423569941507126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024410902026422522, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10426714180718177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027435559350761703, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435363065167147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027315502201731227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10437055745512097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027541469344152063, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416857904686418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027345425466329107, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07646347417182374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05585684483093107, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07626928137113745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055876914070467354, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07630892263399386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05666813516635499, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07619056344370823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05627338290322555, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 224, + "hessian": { + "down_diagonal_mean": 196.52743530273438, + "gate_up_diagonal_mean": 1500.6904296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10473143186724672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024309475450741747, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482163880666234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024751860404454974, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10492147819681484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02429224427846479, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10478039533507727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02485210127828309, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10497212808266101, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028066230582381904, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10472700083768592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028524575529926075, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10497311243785369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02873457935432731, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10515828020901737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028557183460668, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07743177747252857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05588928311942482, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0765693946825487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0565653483030317, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07688827694378285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05705309276612008, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07672051774556028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0575219707827348, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 226, + "hessian": { + "down_diagonal_mean": 143.13348388671875, + "gate_up_diagonal_mean": 1500.7784423828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10766710255827361, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025612636729252108, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10739891304387665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02554397038761921, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10746510800194149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025782905311952032, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1072545642521915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025669018290650145, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10754777642714747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028633225681527073, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.107552780705033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02873634694048438, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10732391732729288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02827669124612401, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10735065854953353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02876449669915403, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07551393423395783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06012001802089096, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07566706204091568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05927620651792592, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07558727400275106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05975949224965222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07556619249287948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059899922076352294, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 228, + "hessian": { + "down_diagonal_mean": 103.30442810058594, + "gate_up_diagonal_mean": 1500.201171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435115307405361, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023544051913284847, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430284742058055, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023414106741458607, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10440177454897812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023518309598804788, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10440303751142524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02321149765705479, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10418247740110677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03255114867761637, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10431477233102461, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03270883314841045, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10417837335973294, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032755994519981485, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10437270092904308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032360110848857676, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07403793170256548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060867157846571554, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07430026376790554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06041341483600095, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07418208538385665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06044323608993372, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.07417127281565424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06059531045777895, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 230, + "hessian": { + "down_diagonal_mean": 224.93795776367188, + "gate_up_diagonal_mean": 1500.61572265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10601267200215567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023984606826286517, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10616354419961233, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02360156248629575, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632277832735479, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023425899254454655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10614403149591987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024160725665118593, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10634551055236367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027182778642254473, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10624374599894287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026928146179014203, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10610031257103317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026966483980619394, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10628583663327204, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027049949896061822, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07776957511418447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05707127703578092, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07841231120624387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055974268502936554, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07832169554053242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05655664341021172, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07828182209316992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05623527335751074, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 232, + "hessian": { + "down_diagonal_mean": 309.4068603515625, + "gate_up_diagonal_mean": 1500.70166015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10931425855444721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018713502344217273, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10939425563911154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018586729769792424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10916173477834984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01864764736584483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10928790903759691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0185669952343848, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.109257506642876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02034998314151322, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10914449201499961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020385143803945717, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10890128697483278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020185888931470858, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10915840457008309, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020303485609819743, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08426025735299471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047949060265151805, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08439519977090963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04597213551117465, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08458443128195947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046101659647172236, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08395262297143048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046806478158078084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 234, + "hessian": { + "down_diagonal_mean": 199.3809814453125, + "gate_up_diagonal_mean": 1499.8349609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10449627096944893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01389362014700735, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10447047861637547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014184841708113119, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10450198978317539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014155071423350936, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10443533757856792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01414352520424915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10442536860960971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018296361204583618, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10462141883335829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01795012170193419, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10444503757845688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018179939170266394, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10442134787704505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01753951943794744, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08627915443117999, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.036450611567225545, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08628345692961811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.039679534085293004, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08535947164145816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04025730775274994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08530858191709016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04020222804495374, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 236, + "hessian": { + "down_diagonal_mean": 140.08450317382812, + "gate_up_diagonal_mean": 1500.716552734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10324485919074523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02879350163146826, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1032003619630864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02847729133142735, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.103125872402516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028331438104124546, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10310075983323386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02850084398131388, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10302696072693654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03295456797102879, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10323129414091418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03370960215057604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10320829595196186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031861444146645726, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307958670232341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03363682982193447, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07451812274190885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05922535875930711, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07429746903094014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0610152773373586, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07428963069455748, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05996942317867688, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07422925600558623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060856385801378574, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 238, + "hessian": { + "down_diagonal_mean": 114.21964263916016, + "gate_up_diagonal_mean": 1500.5406494140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10425881291406025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022860540060060386, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10454036406076783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022788203652819073, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434270861571587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023202014304272555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10435611550247167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023107965806448142, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10434248717093698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03184014719911619, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10429799062381125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03171679629178742, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10409684323360772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03171369715012516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1042109840372893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031637412582390924, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07431723396613515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06163811555850433, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07426660868549367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061519401348000255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07438155523973985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06134174508151546, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07436426214116039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06120260106201587, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 240, + "hessian": { + "down_diagonal_mean": 174.0144805908203, + "gate_up_diagonal_mean": 1500.4716796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10428983889065649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0282244337342752, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10417724279642676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027878519271052164, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.104186947393818, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028260935824654953, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416860500297477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028179616159517196, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10418389719502541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030713413235945047, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10405977382401005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03040092366562907, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1041854503914877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030790898462612352, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10428187658738994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030243161717031977, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07536066189177633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059905888318748594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0756441211208649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905924580752234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07569062788300045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05934610852962557, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07560865987327747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06013077837597384, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 242, + "hessian": { + "down_diagonal_mean": 192.49075317382812, + "gate_up_diagonal_mean": 1500.80810546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10782325987410236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026659813598447792, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10795033223767761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026817909643275722, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10801475644402196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026724651591721573, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10794453983641741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026615647669091823, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10781465062979351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029135360617517137, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10789117437425136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028989222792980246, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1078770996645688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02911938035490482, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10780428184432227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029043797207839877, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07659624830499444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05725138699635685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07693866022105961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05689231951342223, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07626110787539817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0578248719642882, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0768743546040172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057807923100728756, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 244, + "hessian": { + "down_diagonal_mean": 157.38511657714844, + "gate_up_diagonal_mean": 1500.604248046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10534952529110082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027569067386008877, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10517572316865531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027557503355880607, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10536426743006876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027351695489281087, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10532713189774094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02763521276374723, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1052930139824024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031089196288247817, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10512927728710214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031047660552869536, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10534820855944584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030957337745792676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1054333934634949, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03133453952000683, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0753559154686318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06011116636988529, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07542028283779359, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060315162354226584, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07524088633909577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05943736477386194, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07489257268868992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060476490048878714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 246, + "hessian": { + "down_diagonal_mean": 178.18775939941406, + "gate_up_diagonal_mean": 1500.7049560546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1056970283617073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029372297937208032, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10559469929667722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029513307804564008, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10582382363907077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0295557608357217, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10586588546573493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029524213498925627, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10576169515544016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030606870207330876, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10579249575388562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030624803216198985, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10570535556323662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030183946504256296, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10590159031102882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030327550169463917, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0755687317208652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060515657154841335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0755737549141666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06029814750038866, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0756397973825552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059925792356547034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07592234261477712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059252584163510355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 248, + "hessian": { + "down_diagonal_mean": 162.32516479492188, + "gate_up_diagonal_mean": 1500.71484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10396679990095943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029590908249808416, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10388228407824877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029791399980909086, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10417483866728124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029744706999440138, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1040111148329606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0296934541462086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10388577769562932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03388982276914742, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10394100337756075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033744739255282395, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10403640610486313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033868534217498476, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10385438941448175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03395977968494905, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07361688630322141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06290759300266009, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07374408266300866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06166238387792946, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07354618564297406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06308512383730625, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07373804133825326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06241449039567584, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 250, + "hessian": { + "down_diagonal_mean": 164.46571350097656, + "gate_up_diagonal_mean": 1500.7518310546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10679937775750117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025322136316793375, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676027523449685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02522859001088402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10686475369541913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025216897367064922, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10678055164447284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025105287996218268, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683647971196175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029386514615104112, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065847718227717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02926577845715717, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1067232658838945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029506787669587448, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10686435516073439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029291131302081937, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07584959143558015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059299390221877996, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07573674665086876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05970461011037402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07553112783855156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05999056259039884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07589072649664057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05940948254568481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 252, + "hessian": { + "down_diagonal_mean": 179.5007781982422, + "gate_up_diagonal_mean": 1500.7650146484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10861914074717186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022018107598298486, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10845777300065372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022522901526239304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10843822627758258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021980105478372952, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10856072577848618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022168660179379594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10883386313393173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023633982780072647, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10853799577236613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024025306043240425, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10872216064938577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02385491747663885, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10854979893919667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02329366902161881, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07979980233012639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05196856633420833, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07898595780279702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05282148252463627, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0796590621752538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05226905983322048, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07951268205532427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05006976383642709, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 254, + "hessian": { + "down_diagonal_mean": 281.35345458984375, + "gate_up_diagonal_mean": 1500.739990234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10617459570621708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020557627071187676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10647059906670271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020786279438868595, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10633053436021436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020688477840062586, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10643630408872959, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02079007027842438, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10645803617727768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0217046461897929, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10640235707917134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02308416266617399, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1063266789954336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022582358303382736, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133597, + "packed_nmse_vs_bf16": 0.10641557961300213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022269382743878685, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0825248767260035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04315480356062594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0823383788817248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04492984989203495, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.08258847388551993, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04483086681261848, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08142308512226508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04728781474617706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 256, + "hessian": { + "down_diagonal_mean": 276.22943115234375, + "gate_up_diagonal_mean": 1500.62548828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10677420068397672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020435728365107687, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10659794319971232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02064807537995251, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1067581074291777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020483657460407347, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10653241554202834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020910988287269987, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1065493073783602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023956091743704406, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10647301059847397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0248360967307984, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10664357471881868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023782185746204346, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10655942637235732, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024150525395418967, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08133353530477488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05088061924021726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08193428954577349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04874352703573759, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08106122752577406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047061450551421816, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08080706449018972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048844214100969556, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 258, + "hessian": { + "down_diagonal_mean": 250.01409912109375, + "gate_up_diagonal_mean": 1500.763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1101887242006021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020783043942378215, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11010755591157076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020621201213529176, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11020632828629692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020566542075796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11014677406102656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020605570805128186, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1101365030633886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02289462501103436, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11018870558862513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022716242495860368, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11032223307661872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022670787067483846, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11019754588446155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02279266931937926, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08061586481412623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05103761792195478, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08092979980984771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0501809933868262, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08093010318927163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05033456100183602, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08047051945577074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051352202148233966, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 260, + "hessian": { + "down_diagonal_mean": 230.4366455078125, + "gate_up_diagonal_mean": 1500.7242431640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10709119279497775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022495379685596374, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1068579709471531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022412954972902622, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10693435907795103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022594771188032832, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10694476991013983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022479349096674355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10721572515098318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02504426653493969, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10707276278610911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025280547687348016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10690349127279568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025085323355974545, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10714769786637957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0252705254286816, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07959773973726085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0502901488941131, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.0794292478833087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052085611709449676, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07946757286383722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050717007083079864, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07914406263324182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05201226745640313, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 262, + "hessian": { + "down_diagonal_mean": 140.31272888183594, + "gate_up_diagonal_mean": 1500.498291015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10475852614978742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026813653501757764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047540732605022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026621415803852938, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10445878161783499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026520811017498355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465277350529267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026517972499123004, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454652820467031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030496065999249297, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10455443368652981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030199075468197382, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10455841001380105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030163846467291436, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1046136193629078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03052011157522091, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07499135526954787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057271913076245844, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07491901040640811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05678428290938673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07499115661110967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671764331360405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07461210872135277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05810109165235361, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 264, + "hessian": { + "down_diagonal_mean": 204.12728881835938, + "gate_up_diagonal_mean": 1500.72998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10764214307661789, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02266345121499339, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10765537227054892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02250519315715769, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10783026341344357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02251024469873866, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10782429240393954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022918226477198655, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10761669087786069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02444780217641184, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1078323894249087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02459606723845114, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10770208877081483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024269918162977865, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10779403895189618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024597747257671578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07854847803872471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05395089735884999, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07796057247706942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05599417581697071, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07780040743194432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05583863154156984, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0776826746648137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056280181138138596, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 266, + "hessian": { + "down_diagonal_mean": 202.97007751464844, + "gate_up_diagonal_mean": 1500.5902099609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10769610234556029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0220264038452551, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10772324019551215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021755656801251595, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10772457120797173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021958399862835473, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10777322884673102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02213766877998651, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10766053535520109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025978001950243225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10783751111012116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0250522567622491, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.107663024640839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02544520872948124, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10772301088552903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025690616771089456, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08077693296774306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05207504699070941, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08193687270233276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050915798253293544, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08133499181488384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05121667897770067, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08139636769776636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051676863153342086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 268, + "hessian": { + "down_diagonal_mean": 238.2435302734375, + "gate_up_diagonal_mean": 1500.59423828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10487357545863854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022285222155829013, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10491881307406765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021787096301360723, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10510913991074264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022131077573576893, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10513631523067578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022193817093831555, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502283174268348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026506192729141682, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1049677879413676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025918989621577605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10498231251286114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026229244128311124, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503851812947847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026644307646976315, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07965564111246153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054212683876458674, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07919389439000202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05453228752697741, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07952690394530819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0539821741390248, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07922462006351669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05473470319612191, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 270, + "hessian": { + "down_diagonal_mean": 169.44223022460938, + "gate_up_diagonal_mean": 1500.804931640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10628313143282071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0279876487825726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10613124720868014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028014711387628597, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1063074678070859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027832805479867422, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10627588963896546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027993606513896097, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.106256177329827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030222515255628024, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10619644765467413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030318393377505234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10618164659867624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030549312124469825, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10617770515123053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030217539934470204, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07521397445563323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06060228975936354, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07549387562624159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060171373678978304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07503836773926452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06068684745171378, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07497613068266244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06101088133599515, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 272, + "hessian": { + "down_diagonal_mean": 133.696533203125, + "gate_up_diagonal_mean": 1500.383056640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10383758747589412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022347355640670728, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10407938244439849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022261685348279655, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1039409150234296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022418611893712442, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038518584281218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022627689338391023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1039155080898138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029379629810744955, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1038889164086761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02926050374313734, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10405348835997992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029484951254966855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10385385270462869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02932180955815151, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0765869715733404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05486019292203373, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07631130269126236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05535156322226653, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07586755810543554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05716218033804619, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07630545393333132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05603475573422498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 274, + "hessian": { + "down_diagonal_mean": 114.93665313720703, + "gate_up_diagonal_mean": 1500.624755859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10382764485117506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02488007463423058, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10385677064034685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024935891085131716, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10412385456858689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02480168042579735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1040189764929001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02496157397248897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10375146992341942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03330574082804102, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10387016464872938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03354398339773983, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10375718683337061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033341946879646686, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10418185209621984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03352985625530862, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07393410065780236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06215419517505991, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07394927450355368, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06235582533248418, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07386982172741208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06234683729755527, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07364650197397166, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0627355114656214, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 276, + "hessian": { + "down_diagonal_mean": 136.59991455078125, + "gate_up_diagonal_mean": 1500.7156982421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10700616091655916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025297325105044705, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10713586949562895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025165648562723044, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069532946813171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024941538249992637, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1069397835822777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02518480292953, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10690969297723345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030930496934334705, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10694238095990924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03112411788832927, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10699195258076581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03085047391098318, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10696086690158174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030289561970043018, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07514435922990155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06032201333466903, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07479432736334148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0611244882820534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07541083381613435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059025708133286024, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07564756694149977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05740269312659538, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 278, + "hessian": { + "down_diagonal_mean": 168.77212524414062, + "gate_up_diagonal_mean": 1500.7554931640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10563700779867727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024339886961597557, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1054335638887698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024668906118854073, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1058013419959454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02436653554014554, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10560917732885462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024677297622436263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1057841912362682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02735493701059194, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10573289617533979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02828715044672406, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10567198851119264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027718549674363397, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10560737053465037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027932470067061527, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07736766427742789, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05457730564405006, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07671446097210016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056829348696916526, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0776552656693872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054214401599161764, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07736786997400914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05600826062237578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 280, + "hessian": { + "down_diagonal_mean": 105.25205993652344, + "gate_up_diagonal_mean": 1500.7166748046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10703475645443017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021148387258436787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10705885760403185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020453832612879, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10688947882026079, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020909398030225845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10702385958289155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02110456294189304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1067923958833526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03075189516155009, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10696476147070248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02920268053508867, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1068265496070551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02914383648713434, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10672842305056668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03046138509911193, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07468437984673651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06017271411811388, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07549703399471162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057184571834738004, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07484787348486822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058801639265511414, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0747641413059313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059444127991174336, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 282, + "hessian": { + "down_diagonal_mean": 170.20880126953125, + "gate_up_diagonal_mean": 1500.6900634765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10593274491822742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025837668294145768, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1061305185633017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025609593301088994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10582824699784912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025963810662139902, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10588282539235323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02572249722771659, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10595427654331807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02996132860988759, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10585758797370129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030002850928015273, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10575721513796223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0295596644380221, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10606854895444985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02962600960441475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07575613145421187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059345555064945904, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07601015511710169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05803632423079173, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0759367114610307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05882562363162515, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.07638461891211923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05814932589259174, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 284, + "hessian": { + "down_diagonal_mean": 209.651123046875, + "gate_up_diagonal_mean": 1500.592529296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1069587153070797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02212505091856303, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10692949633660809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02237735147347551, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10706677224764845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022080895112947693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10692753997300532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02219523395040705, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10686417027924222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026024915168461886, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10672734810815698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026410319119719056, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10715325958507564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02637144648619622, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10695028235387743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026476075020786524, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08029508975078849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053601658771183024, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08037999243853963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0533272957269464, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07980180186310429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053929343768762336, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08048269510483913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05271765331273344, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 286, + "hessian": { + "down_diagonal_mean": 242.937744140625, + "gate_up_diagonal_mean": 1500.489013671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055695711452554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02362960827891451, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10568593018632039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023920879088702992, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10571159099912737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02371689826343526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10558306891848251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02395327942953154, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.1056546956607147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025583108865998787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10574802625903537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025460114279884552, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10582352562437684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02583845877070797, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10557812323018989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02575769344736739, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07865406003593779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05457167777002055, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0780522521011994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05478093974838217, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07822703997101205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05530705197350216, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07843114881682518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054716736066283696, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + } + ], + "experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "gpu": 0, + "layer": 33, + "physical_experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "physical_tensor_count": 6912, + "schema": "glm53-brandon-style-low-bpw-tail-part-v1", + "selected_k": { + "0": 2, + "10": 2, + "100": 2, + "102": 2, + "104": 2, + "106": 2, + "108": 2, + "110": 2, + "112": 2, + "114": 2, + "116": 2, + "118": 2, + "12": 2, + "120": 2, + "122": 2, + "124": 2, + "126": 2, + "128": 2, + "130": 2, + "132": 2, + "134": 2, + "136": 2, + "138": 2, + "14": 2, + "140": 2, + "142": 2, + "144": 2, + "146": 2, + "148": 2, + "150": 2, + "152": 2, + "154": 2, + "156": 2, + "158": 2, + "16": 2, + "160": 2, + "162": 2, + "164": 2, + "166": 2, + "168": 2, + "170": 2, + "172": 2, + "174": 2, + "176": 2, + "178": 2, + "18": 2, + "180": 2, + "182": 2, + "184": 2, + "186": 2, + "188": 2, + "190": 2, + "192": 2, + "194": 2, + "196": 2, + "198": 2, + "2": 2, + "20": 2, + "200": 2, + "202": 2, + "204": 2, + "206": 2, + "208": 2, + "210": 2, + "212": 2, + "214": 2, + "216": 2, + "218": 2, + "22": 2, + "220": 2, + "222": 2, + "224": 2, + "226": 2, + "228": 2, + "230": 2, + "232": 2, + "234": 2, + "236": 2, + "238": 2, + "24": 2, + "240": 2, + "242": 2, + "244": 2, + "246": 2, + "248": 2, + "250": 2, + "252": 2, + "254": 2, + "256": 2, + "258": 2, + "26": 2, + "260": 2, + "262": 2, + "264": 2, + "266": 2, + "268": 2, + "270": 2, + "272": 2, + "274": 2, + "276": 2, + "278": 2, + "28": 2, + "280": 2, + "282": 2, + "284": 2, + "286": 2, + "30": 2, + "32": 2, + "34": 2, + "36": 2, + "38": 2, + "4": 2, + "40": 2, + "42": 2, + "44": 2, + "46": 2, + "48": 2, + "50": 2, + "52": 2, + "54": 2, + "56": 2, + "58": 2, + "6": 2, + "60": 2, + "62": 2, + "64": 2, + "66": 2, + "68": 2, + "70": 2, + "72": 2, + "74": 2, + "76": 2, + "78": 2, + "8": 2, + "80": 2, + "82": 2, + "84": 2, + "86": 2, + "88": 2, + "90": 2, + "92": 2, + "94": 2, + "96": 2, + "98": 2 + }, + "sha256": "719b04073ed5f3a88945edaac85c412249350d9afd7bcce96cce077e522510d6", + "shadow_experts_ignored": [], + "source_sidecar_sha256": "24596a1e6351bba5b0bcb5d08ed4423447c22c947a717651060d546fe77de8f0", + "state": "COMPLETE", + "tensor_count": 6912, + "tier": "k2-k3-tail", + "variant": "2.0" +}