{ "base_objective": 5659.535350379767, "bf16": "/Users/litwein/.omlx/models/Qwen3.8-27B", "bits": 4, "choices": { "lloyd": 55352164, "lloyd_gptq": 69193418, "lloyd_gptq_act_order": 79523367, "oqe": 2721430, "oqe_gptq": 10569443, "oqe_gptq_act_order": 12425778 }, "damp": 0.01, "format": "omlx-m4q-initializer-report", "group_size": 64, "hessian_cache": "/Users/litwein/GithubControl/dwq/m4q_hessian_qwen38_q4g64.safetensors", "hessian_identity": "618c11aee7ab24a762e0baac4102e5954f03adc165a97a078b7b07cab48f96e8", "hessian_spec": { "bits": 4, "calibration": "5790d72b58b96b3ecb9a39c9c6fcc44249ef24d47a05d9dd7bb8eba0c0d94bcb", "format": "omlx-m4q-group-hessian", "group_size": 64, "num_samples": 256, "seed": 123, "selection_samples": 128, "sequence_length": 512, "student": "e18b6a19f1816f2639ab1d0b10a8c8830d35e843c6d4ef65f9c5b6d836c7a797", "tokens_per_batch": 128, "version": 3 }, "module_count": 165, "modules": { "language_model.model.layers.0.mlp.gate_proj": { "base_objective": 0.4595101249869913, "choices": { "lloyd": 164726, "lloyd_gptq": 429836, "lloyd_gptq_act_order": 651124, "oqe": 7599, "oqe_gptq": 52006, "oqe_gptq_act_order": 87349 }, "objective_split": "held_out_selection", "relative_reduction": 0.1887429693731657, "selected_objective": 0.37278081953991205, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.0.mlp.gate_proj.weight" }, "language_model.model.layers.0.mlp.up_proj": { "base_objective": 0.4266967566218227, "choices": { "lloyd": 164989, "lloyd_gptq": 429936, "lloyd_gptq_act_order": 650903, "oqe": 7769, "oqe_gptq": 51737, "oqe_gptq_act_order": 87306 }, "objective_split": "held_out_selection", "relative_reduction": 0.18936618408979622, "selected_objective": 0.3458948200568557, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.0.mlp.up_proj.weight" }, "language_model.model.layers.1.mlp.gate_proj": { "base_objective": 0.841862749774009, "choices": { "lloyd": 192495, "lloyd_gptq": 439414, "lloyd_gptq_act_order": 606075, "oqe": 9064, "oqe_gptq": 58416, "oqe_gptq_act_order": 87176 }, "objective_split": "held_out_selection", "relative_reduction": 0.16566517744466647, "selected_objective": 0.702395407948643, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.1.mlp.gate_proj.weight" }, "language_model.model.layers.1.mlp.up_proj": { "base_objective": 0.8054950113873929, "choices": { "lloyd": 192230, "lloyd_gptq": 439436, "lloyd_gptq_act_order": 605823, "oqe": 8940, "oqe_gptq": 58466, "oqe_gptq_act_order": 87745 }, "objective_split": "held_out_selection", "relative_reduction": 0.1664205706675339, "selected_objective": 0.6714440719224513, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.1.mlp.up_proj.weight" }, "language_model.model.layers.10.mlp.gate_proj": { "base_objective": 4.020561162382364, "choices": { "lloyd": 264217, "lloyd_gptq": 444758, "lloyd_gptq_act_order": 499503, "oqe": 15114, "oqe_gptq": 77648, "oqe_gptq_act_order": 91400 }, "objective_split": "held_out_selection", "relative_reduction": 0.1391670140316148, "selected_objective": 3.4610316706821322, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.10.mlp.gate_proj.weight" }, "language_model.model.layers.10.mlp.up_proj": { "base_objective": 3.83932751044631, "choices": { "lloyd": 263582, "lloyd_gptq": 444974, "lloyd_gptq_act_order": 499638, "oqe": 15256, "oqe_gptq": 77218, "oqe_gptq_act_order": 91972 }, "objective_split": "held_out_selection", "relative_reduction": 0.14040432292579086, "selected_objective": 3.3002693308517337, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.10.mlp.up_proj.weight" }, "language_model.model.layers.11.mlp.gate_proj": { "base_objective": 4.344947325065732, "choices": { "lloyd": 275718, "lloyd_gptq": 443370, "lloyd_gptq_act_order": 489095, "oqe": 15751, "oqe_gptq": 78806, "oqe_gptq_act_order": 89900 }, "objective_split": "held_out_selection", "relative_reduction": 0.1433372446226222, "selected_objective": 3.722154547460377, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.11.mlp.gate_proj.weight" }, "language_model.model.layers.11.mlp.up_proj": { "base_objective": 4.170417866669595, "choices": { "lloyd": 276193, "lloyd_gptq": 444767, "lloyd_gptq_act_order": 489688, "oqe": 15571, "oqe_gptq": 77410, "oqe_gptq_act_order": 89011 }, "objective_split": "held_out_selection", "relative_reduction": 0.14611898012476432, "selected_objective": 3.5610406612977386, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.11.mlp.up_proj.weight" }, "language_model.model.layers.12.mlp.gate_proj": { "base_objective": 4.649569490924478, "choices": { "lloyd": 269597, "lloyd_gptq": 447335, "lloyd_gptq_act_order": 490516, "oqe": 15483, "oqe_gptq": 79385, "oqe_gptq_act_order": 90324 }, "objective_split": "held_out_selection", "relative_reduction": 0.14032431501623455, "selected_objective": 3.997121836990118, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.12.mlp.gate_proj.weight" }, "language_model.model.layers.12.mlp.up_proj": { "base_objective": 4.452637831680477, "choices": { "lloyd": 270118, "lloyd_gptq": 447614, "lloyd_gptq_act_order": 490374, "oqe": 15696, "oqe_gptq": 78653, "oqe_gptq_act_order": 90185 }, "objective_split": "held_out_selection", "relative_reduction": 0.14236809700308461, "selected_objective": 3.818724256940186, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.12.mlp.up_proj.weight" }, "language_model.model.layers.13.mlp.gate_proj": { "base_objective": 4.998010698705912, "choices": { "lloyd": 295743, "lloyd_gptq": 447892, "lloyd_gptq_act_order": 468766, "oqe": 16158, "oqe_gptq": 78253, "oqe_gptq_act_order": 85828 }, "objective_split": "held_out_selection", "relative_reduction": 0.13406185295091122, "selected_objective": 4.327968123368919, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.13.mlp.gate_proj.weight" }, "language_model.model.layers.13.mlp.up_proj": { "base_objective": 4.78683852404356, "choices": { "lloyd": 295818, "lloyd_gptq": 448134, "lloyd_gptq_act_order": 468729, "oqe": 15992, "oqe_gptq": 78147, "oqe_gptq_act_order": 85820 }, "objective_split": "held_out_selection", "relative_reduction": 0.1356358343619889, "selected_objective": 4.1375716868788, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.13.mlp.up_proj.weight" }, "language_model.model.layers.14.mlp.gate_proj": { "base_objective": 5.284640742465854, "choices": { "lloyd": 301950, "lloyd_gptq": 449201, "lloyd_gptq_act_order": 460660, "oqe": 16408, "oqe_gptq": 79123, "oqe_gptq_act_order": 85298 }, "objective_split": "held_out_selection", "relative_reduction": 0.13059158640903312, "selected_objective": 4.594511124305427, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.14.mlp.gate_proj.weight" }, "language_model.model.layers.14.mlp.up_proj": { "base_objective": 5.076406957581639, "choices": { "lloyd": 301542, "lloyd_gptq": 448259, "lloyd_gptq_act_order": 461965, "oqe": 16326, "oqe_gptq": 78690, "oqe_gptq_act_order": 85858 }, "objective_split": "held_out_selection", "relative_reduction": 0.13181251819939765, "selected_objective": 4.407272973097861, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.14.mlp.up_proj.weight" }, "language_model.model.layers.15.mlp.gate_proj": { "base_objective": 5.961888471618295, "choices": { "lloyd": 323394, "lloyd_gptq": 442245, "lloyd_gptq_act_order": 456135, "oqe": 16342, "oqe_gptq": 74046, "oqe_gptq_act_order": 80478 }, "objective_split": "held_out_selection", "relative_reduction": 0.13216531733835302, "selected_objective": 5.173933589830995, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.15.mlp.gate_proj.weight" }, "language_model.model.layers.15.mlp.up_proj": { "base_objective": 5.777877099812031, "choices": { "lloyd": 323561, "lloyd_gptq": 441768, "lloyd_gptq_act_order": 456305, "oqe": 16106, "oqe_gptq": 74135, "oqe_gptq_act_order": 80765 }, "objective_split": "held_out_selection", "relative_reduction": 0.13364877017666232, "selected_objective": 5.005670931190252, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.15.mlp.up_proj.weight" }, "language_model.model.layers.16.mlp.gate_proj": { "base_objective": 6.680432364344597, "choices": { "lloyd": 311781, "lloyd_gptq": 439933, "lloyd_gptq_act_order": 472484, "oqe": 15630, "oqe_gptq": 71826, "oqe_gptq_act_order": 80986 }, "objective_split": "held_out_selection", "relative_reduction": 0.13830257670998825, "selected_objective": 5.75651135481894, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.16.mlp.gate_proj.weight" }, "language_model.model.layers.16.mlp.up_proj": { "base_objective": 6.445050187408924, "choices": { "lloyd": 310346, "lloyd_gptq": 441363, "lloyd_gptq_act_order": 473019, "oqe": 15531, "oqe_gptq": 71408, "oqe_gptq_act_order": 80973 }, "objective_split": "held_out_selection", "relative_reduction": 0.1400478847571618, "selected_objective": 5.542434541508555, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.16.mlp.up_proj.weight" }, "language_model.model.layers.17.mlp.gate_proj": { "base_objective": 7.740207539871335, "choices": { "lloyd": 305700, "lloyd_gptq": 436915, "lloyd_gptq_act_order": 480335, "oqe": 15804, "oqe_gptq": 71010, "oqe_gptq_act_order": 82876 }, "objective_split": "held_out_selection", "relative_reduction": 0.14088131451398545, "selected_objective": 6.6497569270431995, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.17.mlp.gate_proj.weight" }, "language_model.model.layers.17.mlp.up_proj": { "base_objective": 7.469485396519303, "choices": { "lloyd": 304366, "lloyd_gptq": 437274, "lloyd_gptq_act_order": 482036, "oqe": 15371, "oqe_gptq": 70772, "oqe_gptq_act_order": 82821 }, "objective_split": "held_out_selection", "relative_reduction": 0.14319540930364583, "selected_objective": 6.399889377877116, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.17.mlp.up_proj.weight" }, "language_model.model.layers.18.mlp.gate_proj": { "base_objective": 9.746447626501322, "choices": { "lloyd": 262457, "lloyd_gptq": 434145, "lloyd_gptq_act_order": 525253, "oqe": 14426, "oqe_gptq": 68630, "oqe_gptq_act_order": 87729 }, "objective_split": "held_out_selection", "relative_reduction": 0.16229036959987198, "selected_objective": 8.164693038910627, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.18.mlp.gate_proj.weight" }, "language_model.model.layers.18.mlp.up_proj": { "base_objective": 9.25420866906643, "choices": { "lloyd": 261174, "lloyd_gptq": 434464, "lloyd_gptq_act_order": 526519, "oqe": 14026, "oqe_gptq": 68222, "oqe_gptq_act_order": 88235 }, "objective_split": "held_out_selection", "relative_reduction": 0.1661334167765589, "selected_objective": 7.7167753633111715, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.18.mlp.up_proj.weight" }, "language_model.model.layers.19.mlp.gate_proj": { "base_objective": 12.375938680022955, "choices": { "lloyd": 302442, "lloyd_gptq": 435953, "lloyd_gptq_act_order": 480977, "oqe": 16263, "oqe_gptq": 73003, "oqe_gptq_act_order": 84002 }, "objective_split": "held_out_selection", "relative_reduction": 0.14686997865099183, "selected_objective": 10.558284830302, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.19.mlp.gate_proj.weight" }, "language_model.model.layers.19.mlp.up_proj": { "base_objective": 11.813799154013395, "choices": { "lloyd": 301316, "lloyd_gptq": 436566, "lloyd_gptq_act_order": 481721, "oqe": 16065, "oqe_gptq": 72542, "oqe_gptq_act_order": 84430 }, "objective_split": "held_out_selection", "relative_reduction": 0.1527666330706977, "selected_objective": 10.009044833481312, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.19.mlp.up_proj.weight" }, "language_model.model.layers.2.mlp.gate_proj": { "base_objective": 1.2813364495523274, "choices": { "lloyd": 258023, "lloyd_gptq": 447063, "lloyd_gptq_act_order": 503744, "oqe": 14439, "oqe_gptq": 76878, "oqe_gptq_act_order": 92493 }, "objective_split": "held_out_selection", "relative_reduction": 0.1382072220733359, "selected_objective": 1.104246498318389, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.2.mlp.gate_proj.weight" }, "language_model.model.layers.2.mlp.up_proj": { "base_objective": 1.2072026170790195, "choices": { "lloyd": 257523, "lloyd_gptq": 446138, "lloyd_gptq_act_order": 506039, "oqe": 14168, "oqe_gptq": 76761, "oqe_gptq_act_order": 92011 }, "objective_split": "held_out_selection", "relative_reduction": 0.1387283762865852, "selected_objective": 1.039729358162731, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.2.mlp.up_proj.weight" }, "language_model.model.layers.20.mlp.gate_proj": { "base_objective": 14.964200031012297, "choices": { "lloyd": 314388, "lloyd_gptq": 443164, "lloyd_gptq_act_order": 453221, "oqe": 18016, "oqe_gptq": 79359, "oqe_gptq_act_order": 84492 }, "objective_split": "held_out_selection", "relative_reduction": 0.136173876216794, "selected_objective": 12.926466908305883, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.20.mlp.gate_proj.weight" }, "language_model.model.layers.20.mlp.up_proj": { "base_objective": 13.992726922035217, "choices": { "lloyd": 313577, "lloyd_gptq": 443295, "lloyd_gptq_act_order": 454291, "oqe": 17955, "oqe_gptq": 78776, "oqe_gptq_act_order": 84746 }, "objective_split": "held_out_selection", "relative_reduction": 0.13983141767871377, "selected_objective": 12.036104079335928, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.20.mlp.up_proj.weight" }, "language_model.model.layers.21.mlp.gate_proj": { "base_objective": 16.276506315916777, "choices": { "lloyd": 333716, "lloyd_gptq": 443093, "lloyd_gptq_act_order": 433538, "oqe": 19247, "oqe_gptq": 79980, "oqe_gptq_act_order": 83066 }, "objective_split": "held_out_selection", "relative_reduction": 0.12927882711807825, "selected_objective": 14.172298669815063, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.21.mlp.gate_proj.weight" }, "language_model.model.layers.21.mlp.up_proj": { "base_objective": 15.299354836344719, "choices": { "lloyd": 333021, "lloyd_gptq": 442885, "lloyd_gptq_act_order": 434889, "oqe": 19089, "oqe_gptq": 80332, "oqe_gptq_act_order": 82424 }, "objective_split": "held_out_selection", "relative_reduction": 0.13234915731933494, "selected_objective": 13.274498116225004, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.21.mlp.up_proj.weight" }, "language_model.model.layers.22.mlp.gate_proj": { "base_objective": 19.04569412022829, "choices": { "lloyd": 351210, "lloyd_gptq": 439436, "lloyd_gptq_act_order": 423849, "oqe": 19874, "oqe_gptq": 78418, "oqe_gptq_act_order": 79853 }, "objective_split": "held_out_selection", "relative_reduction": 0.13698001379152547, "selected_objective": 16.436814676970243, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.22.mlp.gate_proj.weight" }, "language_model.model.layers.22.mlp.up_proj": { "base_objective": 17.05502561107278, "choices": { "lloyd": 349386, "lloyd_gptq": 440151, "lloyd_gptq_act_order": 425158, "oqe": 19583, "oqe_gptq": 78708, "oqe_gptq_act_order": 79654 }, "objective_split": "held_out_selection", "relative_reduction": 0.14332622932789252, "selected_objective": 14.610593099147081, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.22.mlp.up_proj.weight" }, "language_model.model.layers.23.mlp.gate_proj": { "base_objective": 19.95293838530779, "choices": { "lloyd": 382733, "lloyd_gptq": 434152, "lloyd_gptq_act_order": 400737, "oqe": 21451, "oqe_gptq": 77537, "oqe_gptq_act_order": 76030 }, "objective_split": "held_out_selection", "relative_reduction": 0.1309050478516034, "selected_objective": 17.34099803119898, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.23.mlp.gate_proj.weight" }, "language_model.model.layers.23.mlp.up_proj": { "base_objective": 18.652442552149296, "choices": { "lloyd": 381744, "lloyd_gptq": 434273, "lloyd_gptq_act_order": 403677, "oqe": 21029, "oqe_gptq": 77005, "oqe_gptq_act_order": 74912 }, "objective_split": "held_out_selection", "relative_reduction": 0.13753608069855605, "selected_objective": 16.08705870807171, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.23.mlp.up_proj.weight" }, "language_model.model.layers.24.mlp.gate_proj": { "base_objective": 22.211520396173, "choices": { "lloyd": 379235, "lloyd_gptq": 433632, "lloyd_gptq_act_order": 404096, "oqe": 21649, "oqe_gptq": 77587, "oqe_gptq_act_order": 76441 }, "objective_split": "held_out_selection", "relative_reduction": 0.1374424768300364, "selected_objective": 19.15871401876211, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.24.mlp.gate_proj.weight" }, "language_model.model.layers.24.mlp.up_proj": { "base_objective": 20.787153027951717, "choices": { "lloyd": 378752, "lloyd_gptq": 432648, "lloyd_gptq_act_order": 406889, "oqe": 21107, "oqe_gptq": 77025, "oqe_gptq_act_order": 76219 }, "objective_split": "held_out_selection", "relative_reduction": 0.1463075092420345, "selected_objective": 17.745836444199085, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.24.mlp.up_proj.weight" }, "language_model.model.layers.25.mlp.gate_proj": { "base_objective": 21.91891623288393, "choices": { "lloyd": 359005, "lloyd_gptq": 437238, "lloyd_gptq_act_order": 417437, "oqe": 21052, "oqe_gptq": 78468, "oqe_gptq_act_order": 79440 }, "objective_split": "held_out_selection", "relative_reduction": 0.14312815782125077, "selected_objective": 18.781702131032944, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.25.mlp.gate_proj.weight" }, "language_model.model.layers.25.mlp.up_proj": { "base_objective": 21.58106104284525, "choices": { "lloyd": 358762, "lloyd_gptq": 436851, "lloyd_gptq_act_order": 419302, "oqe": 20763, "oqe_gptq": 78055, "oqe_gptq_act_order": 78907 }, "objective_split": "held_out_selection", "relative_reduction": 0.15254526277829472, "selected_objective": 18.288972415030003, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.25.mlp.up_proj.weight" }, "language_model.model.layers.26.mlp.gate_proj": { "base_objective": 21.256208345294, "choices": { "lloyd": 331483, "lloyd_gptq": 440845, "lloyd_gptq_act_order": 442824, "oqe": 19114, "oqe_gptq": 76276, "oqe_gptq_act_order": 82098 }, "objective_split": "held_out_selection", "relative_reduction": 0.16287326833489868, "selected_objective": 17.794140219688416, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.26.mlp.gate_proj.weight" }, "language_model.model.layers.26.mlp.up_proj": { "base_objective": 22.052930414676666, "choices": { "lloyd": 331248, "lloyd_gptq": 439833, "lloyd_gptq_act_order": 444635, "oqe": 19303, "oqe_gptq": 76180, "oqe_gptq_act_order": 81441 }, "objective_split": "held_out_selection", "relative_reduction": 0.1740963823012518, "selected_objective": 18.213595010340214, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.26.mlp.up_proj.weight" }, "language_model.model.layers.27.mlp.down_proj": { "base_objective": 1.1910142879933119, "choices": { "lloyd": 613797, "lloyd_gptq": 352077, "lloyd_gptq_act_order": 326682, "oqe": 24645, "oqe_gptq": 40206, "oqe_gptq_act_order": 35233 }, "objective_split": "held_out_selection", "relative_reduction": 0.1224658135094161, "selected_objective": 1.0451557543128729, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.27.mlp.down_proj.weight" }, "language_model.model.layers.27.mlp.gate_proj": { "base_objective": 22.219318941235542, "choices": { "lloyd": 354990, "lloyd_gptq": 438149, "lloyd_gptq_act_order": 426810, "oqe": 19733, "oqe_gptq": 75382, "oqe_gptq_act_order": 77576 }, "objective_split": "held_out_selection", "relative_reduction": 0.15553862125052142, "selected_objective": 18.76335670799017, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.27.mlp.gate_proj.weight" }, "language_model.model.layers.27.mlp.up_proj": { "base_objective": 23.88298212736845, "choices": { "lloyd": 354644, "lloyd_gptq": 437299, "lloyd_gptq_act_order": 428611, "oqe": 19284, "oqe_gptq": 74697, "oqe_gptq_act_order": 78105 }, "objective_split": "held_out_selection", "relative_reduction": 0.16969182193416998, "selected_objective": 19.83023537695408, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.27.mlp.up_proj.weight" }, "language_model.model.layers.28.mlp.down_proj": { "base_objective": 1.230525995604694, "choices": { "lloyd": 629098, "lloyd_gptq": 349765, "lloyd_gptq_act_order": 321149, "oqe": 21225, "oqe_gptq": 38295, "oqe_gptq_act_order": 33108 }, "objective_split": "held_out_selection", "relative_reduction": 0.11325940653184763, "selected_objective": 1.0911573516204953, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.28.mlp.down_proj.weight" }, "language_model.model.layers.28.mlp.gate_proj": { "base_objective": 21.818252608180046, "choices": { "lloyd": 352735, "lloyd_gptq": 439206, "lloyd_gptq_act_order": 432549, "oqe": 18521, "oqe_gptq": 73588, "oqe_gptq_act_order": 76041 }, "objective_split": "held_out_selection", "relative_reduction": 0.1546844076152215, "selected_objective": 18.443309128284454, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.28.mlp.gate_proj.weight" }, "language_model.model.layers.28.mlp.up_proj": { "base_objective": 23.920535132288933, "choices": { "lloyd": 352060, "lloyd_gptq": 438887, "lloyd_gptq_act_order": 434413, "oqe": 18241, "oqe_gptq": 72805, "oqe_gptq_act_order": 76234 }, "objective_split": "held_out_selection", "relative_reduction": 0.16939516958605683, "selected_objective": 19.868512026965618, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.28.mlp.up_proj.weight" }, "language_model.model.layers.29.mlp.down_proj": { "base_objective": 1.2084681987762451, "choices": { "lloyd": 673921, "lloyd_gptq": 337885, "lloyd_gptq_act_order": 295405, "oqe": 19199, "oqe_gptq": 36296, "oqe_gptq_act_order": 29934 }, "objective_split": "held_out_selection", "relative_reduction": 0.10327145600494247, "selected_objective": 1.083667928352952, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.29.mlp.down_proj.weight" }, "language_model.model.layers.29.mlp.gate_proj": { "base_objective": 21.86723231524229, "choices": { "lloyd": 369246, "lloyd_gptq": 434225, "lloyd_gptq_act_order": 429472, "oqe": 17821, "oqe_gptq": 69077, "oqe_gptq_act_order": 72799 }, "objective_split": "held_out_selection", "relative_reduction": 0.1502764232969791, "selected_objective": 18.58110285550356, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.29.mlp.gate_proj.weight" }, "language_model.model.layers.29.mlp.up_proj": { "base_objective": 24.105517700314522, "choices": { "lloyd": 369151, "lloyd_gptq": 434682, "lloyd_gptq_act_order": 430792, "oqe": 17685, "oqe_gptq": 68657, "oqe_gptq_act_order": 71673 }, "objective_split": "held_out_selection", "relative_reduction": 0.16485610937662254, "selected_objective": 20.13157583773136, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.29.mlp.up_proj.weight" }, "language_model.model.layers.3.mlp.gate_proj": { "base_objective": 1.3963809171691537, "choices": { "lloyd": 242745, "lloyd_gptq": 438509, "lloyd_gptq_act_order": 518021, "oqe": 15102, "oqe_gptq": 79217, "oqe_gptq_act_order": 99046 }, "objective_split": "held_out_selection", "relative_reduction": 0.14377922811915933, "selected_objective": 1.195610346738249, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.3.mlp.gate_proj.weight" }, "language_model.model.layers.3.mlp.up_proj": { "base_objective": 1.3457481269724667, "choices": { "lloyd": 242972, "lloyd_gptq": 437906, "lloyd_gptq_act_order": 519766, "oqe": 14863, "oqe_gptq": 78765, "oqe_gptq_act_order": 98368 }, "objective_split": "held_out_selection", "relative_reduction": 0.14446164218449897, "selected_objective": 1.1513391425833106, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.3.mlp.up_proj.weight" }, "language_model.model.layers.30.mlp.down_proj": { "base_objective": 1.27770890109241, "choices": { "lloyd": 652854, "lloyd_gptq": 342753, "lloyd_gptq_act_order": 312396, "oqe": 17614, "oqe_gptq": 36022, "oqe_gptq_act_order": 31001 }, "objective_split": "held_out_selection", "relative_reduction": 0.10643247593265881, "selected_objective": 1.1417191792279482, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.30.mlp.down_proj.weight" }, "language_model.model.layers.30.mlp.gate_proj": { "base_objective": 21.67806761711836, "choices": { "lloyd": 359368, "lloyd_gptq": 435064, "lloyd_gptq_act_order": 442584, "oqe": 16604, "oqe_gptq": 67246, "oqe_gptq_act_order": 71774 }, "objective_split": "held_out_selection", "relative_reduction": 0.16078853234654922, "selected_objective": 18.192482940852642, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.30.mlp.gate_proj.weight" }, "language_model.model.layers.30.mlp.up_proj": { "base_objective": 24.552422113716602, "choices": { "lloyd": 357898, "lloyd_gptq": 435638, "lloyd_gptq_act_order": 443751, "oqe": 16541, "oqe_gptq": 66919, "oqe_gptq_act_order": 71893 }, "objective_split": "held_out_selection", "relative_reduction": 0.17675321061860882, "selected_objective": 20.212702676653862, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.30.mlp.up_proj.weight" }, "language_model.model.layers.31.mlp.down_proj": { "base_objective": 1.3800626397132874, "choices": { "lloyd": 708088, "lloyd_gptq": 323862, "lloyd_gptq_act_order": 282701, "oqe": 17377, "oqe_gptq": 33337, "oqe_gptq_act_order": 27275 }, "objective_split": "held_out_selection", "relative_reduction": 0.10280925454213208, "selected_objective": 1.2381794285029173, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.31.mlp.down_proj.weight" }, "language_model.model.layers.31.mlp.gate_proj": { "base_objective": 22.694474689662457, "choices": { "lloyd": 388188, "lloyd_gptq": 429605, "lloyd_gptq_act_order": 424349, "oqe": 17462, "oqe_gptq": 65257, "oqe_gptq_act_order": 67779 }, "objective_split": "held_out_selection", "relative_reduction": 0.15547891162082217, "selected_objective": 19.16596246510744, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.31.mlp.gate_proj.weight" }, "language_model.model.layers.31.mlp.up_proj": { "base_objective": 26.26206049323082, "choices": { "lloyd": 387471, "lloyd_gptq": 429256, "lloyd_gptq_act_order": 425916, "oqe": 17091, "oqe_gptq": 64961, "oqe_gptq_act_order": 67945 }, "objective_split": "held_out_selection", "relative_reduction": 0.16849196572058123, "selected_objective": 21.837114296853542, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.31.mlp.up_proj.weight" }, "language_model.model.layers.32.mlp.down_proj": { "base_objective": 1.4781511276960373, "choices": { "lloyd": 680041, "lloyd_gptq": 334385, "lloyd_gptq_act_order": 297076, "oqe": 16733, "oqe_gptq": 35272, "oqe_gptq_act_order": 29133 }, "objective_split": "held_out_selection", "relative_reduction": 0.10172237379596083, "selected_objective": 1.32779008615762, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.32.mlp.down_proj.weight" }, "language_model.model.layers.32.mlp.gate_proj": { "base_objective": 24.034780979156494, "choices": { "lloyd": 392560, "lloyd_gptq": 428311, "lloyd_gptq_act_order": 424138, "oqe": 17145, "oqe_gptq": 63924, "oqe_gptq_act_order": 66562 }, "objective_split": "held_out_selection", "relative_reduction": 0.15584195059600625, "selected_objective": 20.289153829216957, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.32.mlp.gate_proj.weight" }, "language_model.model.layers.32.mlp.up_proj": { "base_objective": 28.094232492148876, "choices": { "lloyd": 391159, "lloyd_gptq": 428660, "lloyd_gptq_act_order": 425658, "oqe": 17007, "oqe_gptq": 63615, "oqe_gptq_act_order": 66541 }, "objective_split": "held_out_selection", "relative_reduction": 0.169225372683111, "selected_objective": 23.339975528419018, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.32.mlp.up_proj.weight" }, "language_model.model.layers.33.mlp.down_proj": { "base_objective": 1.555372977629304, "choices": { "lloyd": 627898, "lloyd_gptq": 353367, "lloyd_gptq_act_order": 326822, "oqe": 17353, "oqe_gptq": 35907, "oqe_gptq_act_order": 31293 }, "objective_split": "held_out_selection", "relative_reduction": 0.10924738680470514, "selected_objective": 1.3854525443166494, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.33.mlp.down_proj.weight" }, "language_model.model.layers.33.mlp.gate_proj": { "base_objective": 25.025221325457096, "choices": { "lloyd": 389161, "lloyd_gptq": 430372, "lloyd_gptq_act_order": 425582, "oqe": 17095, "oqe_gptq": 63571, "oqe_gptq_act_order": 66859 }, "objective_split": "held_out_selection", "relative_reduction": 0.15675674589228616, "selected_objective": 21.102349065244198, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.33.mlp.gate_proj.weight" }, "language_model.model.layers.33.mlp.up_proj": { "base_objective": 29.569145411252975, "choices": { "lloyd": 386976, "lloyd_gptq": 430615, "lloyd_gptq_act_order": 427439, "oqe": 16755, "oqe_gptq": 63559, "oqe_gptq_act_order": 67296 }, "objective_split": "held_out_selection", "relative_reduction": 0.16861615469690328, "selected_objective": 24.583309814333916, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.33.mlp.up_proj.weight" }, "language_model.model.layers.34.mlp.down_proj": { "base_objective": 2.135206799954176, "choices": { "lloyd": 520081, "lloyd_gptq": 379880, "lloyd_gptq_act_order": 404206, "oqe": 15578, "oqe_gptq": 35685, "oqe_gptq_act_order": 37210 }, "objective_split": "held_out_selection", "relative_reduction": 0.12737587452773713, "selected_objective": 1.8632329665124416, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.34.mlp.down_proj.weight" }, "language_model.model.layers.34.mlp.gate_proj": { "base_objective": 27.784273855388165, "choices": { "lloyd": 335827, "lloyd_gptq": 429010, "lloyd_gptq_act_order": 476832, "oqe": 15445, "oqe_gptq": 61877, "oqe_gptq_act_order": 73649 }, "objective_split": "held_out_selection", "relative_reduction": 0.17389840925802147, "selected_objective": 22.95263282954693, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.34.mlp.gate_proj.weight" }, "language_model.model.layers.34.mlp.up_proj": { "base_objective": 32.971353612840176, "choices": { "lloyd": 334942, "lloyd_gptq": 430144, "lloyd_gptq_act_order": 477986, "oqe": 15149, "oqe_gptq": 61389, "oqe_gptq_act_order": 73030 }, "objective_split": "held_out_selection", "relative_reduction": 0.18702454749664488, "selected_objective": 26.804901123046875, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.34.mlp.up_proj.weight" }, "language_model.model.layers.35.mlp.down_proj": { "base_objective": 2.329005878418684, "choices": { "lloyd": 507673, "lloyd_gptq": 385788, "lloyd_gptq_act_order": 392466, "oqe": 23882, "oqe_gptq": 41527, "oqe_gptq_act_order": 41304 }, "objective_split": "held_out_selection", "relative_reduction": 0.14086547888321965, "selected_objective": 2.0009293500334024, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.35.mlp.down_proj.weight" }, "language_model.model.layers.35.mlp.gate_proj": { "base_objective": 30.473417460918427, "choices": { "lloyd": 351188, "lloyd_gptq": 428967, "lloyd_gptq_act_order": 449089, "oqe": 18025, "oqe_gptq": 68652, "oqe_gptq_act_order": 76719 }, "objective_split": "held_out_selection", "relative_reduction": 0.15978252002071425, "selected_objective": 25.604298025369644, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.35.mlp.gate_proj.weight" }, "language_model.model.layers.35.mlp.up_proj": { "base_objective": 35.13678327202797, "choices": { "lloyd": 349469, "lloyd_gptq": 429367, "lloyd_gptq_act_order": 451534, "oqe": 17357, "oqe_gptq": 68260, "oqe_gptq_act_order": 76653 }, "objective_split": "held_out_selection", "relative_reduction": 0.17206275370314295, "selected_objective": 29.09105158597231, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.35.mlp.up_proj.weight" }, "language_model.model.layers.36.mlp.down_proj": { "base_objective": 2.0046358834952116, "choices": { "lloyd": 513673, "lloyd_gptq": 381797, "lloyd_gptq_act_order": 391680, "oqe": 24495, "oqe_gptq": 40722, "oqe_gptq_act_order": 40273 }, "objective_split": "held_out_selection", "relative_reduction": 0.14153800268631067, "selected_objective": 1.7209037244319916, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.36.mlp.down_proj.weight" }, "language_model.model.layers.36.mlp.gate_proj": { "base_objective": 33.707527972757816, "choices": { "lloyd": 324352, "lloyd_gptq": 435929, "lloyd_gptq_act_order": 454522, "oqe": 18610, "oqe_gptq": 75388, "oqe_gptq_act_order": 83839 }, "objective_split": "held_out_selection", "relative_reduction": 0.15106647440377696, "selected_objective": 28.6154505610466, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.36.mlp.gate_proj.weight" }, "language_model.model.layers.36.mlp.up_proj": { "base_objective": 34.296021580696106, "choices": { "lloyd": 323329, "lloyd_gptq": 435450, "lloyd_gptq_act_order": 456574, "oqe": 18083, "oqe_gptq": 74566, "oqe_gptq_act_order": 84638 }, "objective_split": "held_out_selection", "relative_reduction": 0.16108249369583427, "selected_objective": 28.771532900631428, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.36.mlp.up_proj.weight" }, "language_model.model.layers.37.mlp.down_proj": { "base_objective": 1.842599593102932, "choices": { "lloyd": 569070, "lloyd_gptq": 367362, "lloyd_gptq_act_order": 350047, "oqe": 26841, "oqe_gptq": 41448, "oqe_gptq_act_order": 37872 }, "objective_split": "held_out_selection", "relative_reduction": 0.12772102382095418, "selected_objective": 1.607260886579752, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.37.mlp.down_proj.weight" }, "language_model.model.layers.37.mlp.gate_proj": { "base_objective": 33.86310436576605, "choices": { "lloyd": 324206, "lloyd_gptq": 436990, "lloyd_gptq_act_order": 450356, "oqe": 18812, "oqe_gptq": 77656, "oqe_gptq_act_order": 84620 }, "objective_split": "held_out_selection", "relative_reduction": 0.14229342801074463, "selected_objective": 29.044607162475586, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.37.mlp.gate_proj.weight" }, "language_model.model.layers.37.mlp.up_proj": { "base_objective": 33.53317854553461, "choices": { "lloyd": 322311, "lloyd_gptq": 438059, "lloyd_gptq_act_order": 452725, "oqe": 18182, "oqe_gptq": 76463, "oqe_gptq_act_order": 84900 }, "objective_split": "held_out_selection", "relative_reduction": 0.15070946898898108, "selected_objective": 28.479411013424397, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.37.mlp.up_proj.weight" }, "language_model.model.layers.38.mlp.down_proj": { "base_objective": 1.8677259180694818, "choices": { "lloyd": 546165, "lloyd_gptq": 372471, "lloyd_gptq_act_order": 362524, "oqe": 27677, "oqe_gptq": 43263, "oqe_gptq_act_order": 40540 }, "objective_split": "held_out_selection", "relative_reduction": 0.13442221723287617, "selected_objective": 1.6166620589792728, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.38.mlp.down_proj.weight" }, "language_model.model.layers.38.mlp.gate_proj": { "base_objective": 42.240163028240204, "choices": { "lloyd": 339314, "lloyd_gptq": 434966, "lloyd_gptq_act_order": 445686, "oqe": 18427, "oqe_gptq": 73803, "oqe_gptq_act_order": 80444 }, "objective_split": "held_out_selection", "relative_reduction": 0.1524942902773576, "selected_objective": 35.79877934604883, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.38.mlp.gate_proj.weight" }, "language_model.model.layers.38.mlp.up_proj": { "base_objective": 36.46470972895622, "choices": { "lloyd": 337182, "lloyd_gptq": 435318, "lloyd_gptq_act_order": 448195, "oqe": 17960, "oqe_gptq": 73195, "oqe_gptq_act_order": 80790 }, "objective_split": "held_out_selection", "relative_reduction": 0.16386011458045802, "selected_objective": 30.489598214626312, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.38.mlp.up_proj.weight" }, "language_model.model.layers.39.mlp.down_proj": { "base_objective": 1.7803108133375645, "choices": { "lloyd": 594876, "lloyd_gptq": 360018, "lloyd_gptq_act_order": 319424, "oqe": 32549, "oqe_gptq": 46056, "oqe_gptq_act_order": 39717 }, "objective_split": "held_out_selection", "relative_reduction": 0.11855254654520864, "selected_objective": 1.5692504327744246, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.39.mlp.down_proj.weight" }, "language_model.model.layers.39.mlp.gate_proj": { "base_objective": 38.34022521972656, "choices": { "lloyd": 357198, "lloyd_gptq": 429700, "lloyd_gptq_act_order": 432379, "oqe": 19274, "oqe_gptq": 74560, "oqe_gptq_act_order": 79529 }, "objective_split": "held_out_selection", "relative_reduction": 0.14212475419835613, "selected_objective": 32.89113013446331, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.39.mlp.gate_proj.weight" }, "language_model.model.layers.39.mlp.up_proj": { "base_objective": 35.02458843588829, "choices": { "lloyd": 355005, "lloyd_gptq": 429920, "lloyd_gptq_act_order": 435694, "oqe": 18737, "oqe_gptq": 73788, "oqe_gptq_act_order": 79496 }, "objective_split": "held_out_selection", "relative_reduction": 0.15092117665689164, "selected_objective": 29.73863633722067, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.39.mlp.up_proj.weight" }, "language_model.model.layers.4.mlp.gate_proj": { "base_objective": 1.7296784687787294, "choices": { "lloyd": 243401, "lloyd_gptq": 442026, "lloyd_gptq_act_order": 513089, "oqe": 15140, "oqe_gptq": 80560, "oqe_gptq_act_order": 98424 }, "objective_split": "held_out_selection", "relative_reduction": 0.14096403157291426, "selected_objective": 1.4858560184948146, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.4.mlp.gate_proj.weight" }, "language_model.model.layers.4.mlp.up_proj": { "base_objective": 1.672181366942823, "choices": { "lloyd": 244113, "lloyd_gptq": 442210, "lloyd_gptq_act_order": 512216, "oqe": 15271, "oqe_gptq": 80668, "oqe_gptq_act_order": 98162 }, "objective_split": "held_out_selection", "relative_reduction": 0.1415271493711594, "selected_objective": 1.4355223048478365, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.4.mlp.up_proj.weight" }, "language_model.model.layers.40.mlp.down_proj": { "base_objective": 1.9411130268126726, "choices": { "lloyd": 564654, "lloyd_gptq": 362765, "lloyd_gptq_act_order": 333695, "oqe": 37648, "oqe_gptq": 49778, "oqe_gptq_act_order": 44100 }, "objective_split": "held_out_selection", "relative_reduction": 0.12914715612703065, "selected_objective": 1.6904237996786833, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.40.mlp.down_proj.weight" }, "language_model.model.layers.40.mlp.gate_proj": { "base_objective": 41.67084486782551, "choices": { "lloyd": 377490, "lloyd_gptq": 428459, "lloyd_gptq_act_order": 420460, "oqe": 19263, "oqe_gptq": 71652, "oqe_gptq_act_order": 75316 }, "objective_split": "held_out_selection", "relative_reduction": 0.1470317540035334, "selected_objective": 35.54390745609999, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.40.mlp.gate_proj.weight" }, "language_model.model.layers.40.mlp.up_proj": { "base_objective": 37.35861913859844, "choices": { "lloyd": 374287, "lloyd_gptq": 429141, "lloyd_gptq_act_order": 423446, "oqe": 19145, "oqe_gptq": 71386, "oqe_gptq_act_order": 75235 }, "objective_split": "held_out_selection", "relative_reduction": 0.1553540657913719, "selected_objective": 31.554805763065815, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.40.mlp.up_proj.weight" }, "language_model.model.layers.41.mlp.down_proj": { "base_objective": 1.8380074631422758, "choices": { "lloyd": 602974, "lloyd_gptq": 356761, "lloyd_gptq_act_order": 331374, "oqe": 25846, "oqe_gptq": 40257, "oqe_gptq_act_order": 35428 }, "objective_split": "held_out_selection", "relative_reduction": 0.12291819961630213, "selected_objective": 1.6120828948915005, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.41.mlp.down_proj.weight" }, "language_model.model.layers.41.mlp.gate_proj": { "base_objective": 36.13913205265999, "choices": { "lloyd": 375931, "lloyd_gptq": 431569, "lloyd_gptq_act_order": 422926, "oqe": 18510, "oqe_gptq": 70196, "oqe_gptq_act_order": 73508 }, "objective_split": "held_out_selection", "relative_reduction": 0.14333051354315163, "selected_objective": 30.959291696548462, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.41.mlp.gate_proj.weight" }, "language_model.model.layers.41.mlp.up_proj": { "base_objective": 35.98599983751774, "choices": { "lloyd": 374450, "lloyd_gptq": 430522, "lloyd_gptq_act_order": 425914, "oqe": 18170, "oqe_gptq": 69620, "oqe_gptq_act_order": 73964 }, "objective_split": "held_out_selection", "relative_reduction": 0.1526605824051176, "selected_objective": 30.49235614389181, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.41.mlp.up_proj.weight" }, "language_model.model.layers.42.mlp.down_proj": { "base_objective": 2.096575539559126, "choices": { "lloyd": 577515, "lloyd_gptq": 364542, "lloyd_gptq_act_order": 364633, "oqe": 17114, "oqe_gptq": 35279, "oqe_gptq_act_order": 33557 }, "objective_split": "held_out_selection", "relative_reduction": 0.12723661662430175, "selected_objective": 1.8298143614083529, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.42.mlp.down_proj.weight" }, "language_model.model.layers.42.mlp.gate_proj": { "base_objective": 34.16330433636904, "choices": { "lloyd": 346605, "lloyd_gptq": 435670, "lloyd_gptq_act_order": 452167, "oqe": 16800, "oqe_gptq": 67159, "oqe_gptq_act_order": 74239 }, "objective_split": "held_out_selection", "relative_reduction": 0.15648892172924989, "selected_objective": 28.81712567806244, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.42.mlp.gate_proj.weight" }, "language_model.model.layers.42.mlp.up_proj": { "base_objective": 36.51925007998943, "choices": { "lloyd": 346531, "lloyd_gptq": 436041, "lloyd_gptq_act_order": 451821, "oqe": 16361, "oqe_gptq": 67158, "oqe_gptq_act_order": 74728 }, "objective_split": "held_out_selection", "relative_reduction": 0.1668188957085185, "selected_objective": 30.42714910954237, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.42.mlp.up_proj.weight" }, "language_model.model.layers.43.mlp.down_proj": { "base_objective": 2.2589961793273687, "choices": { "lloyd": 608718, "lloyd_gptq": 358623, "lloyd_gptq_act_order": 338414, "oqe": 18387, "oqe_gptq": 35987, "oqe_gptq_act_order": 32511 }, "objective_split": "held_out_selection", "relative_reduction": 0.11490291967350486, "selected_objective": 1.9994309227913618, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.43.mlp.down_proj.weight" }, "language_model.model.layers.43.mlp.gate_proj": { "base_objective": 34.270878829061985, "choices": { "lloyd": 359705, "lloyd_gptq": 434706, "lloyd_gptq_act_order": 439186, "oqe": 17383, "oqe_gptq": 68159, "oqe_gptq_act_order": 73501 }, "objective_split": "held_out_selection", "relative_reduction": 0.15123568396535725, "selected_objective": 29.087899029254913, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.43.mlp.gate_proj.weight" }, "language_model.model.layers.43.mlp.up_proj": { "base_objective": 38.18004992604256, "choices": { "lloyd": 359009, "lloyd_gptq": 433557, "lloyd_gptq_act_order": 440863, "oqe": 16988, "oqe_gptq": 68069, "oqe_gptq_act_order": 74154 }, "objective_split": "held_out_selection", "relative_reduction": 0.16112083711345082, "selected_objective": 32.028448320925236, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.43.mlp.up_proj.weight" }, "language_model.model.layers.44.mlp.down_proj": { "base_objective": 2.554251851513982, "choices": { "lloyd": 599746, "lloyd_gptq": 363819, "lloyd_gptq_act_order": 347889, "oqe": 16315, "oqe_gptq": 34034, "oqe_gptq_act_order": 30837 }, "objective_split": "held_out_selection", "relative_reduction": 0.11871484431199097, "selected_objective": 2.251024240627885, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.44.mlp.down_proj.weight" }, "language_model.model.layers.44.mlp.gate_proj": { "base_objective": 35.07694565504789, "choices": { "lloyd": 352418, "lloyd_gptq": 434780, "lloyd_gptq_act_order": 448141, "oqe": 16843, "oqe_gptq": 66945, "oqe_gptq_act_order": 73513 }, "objective_split": "held_out_selection", "relative_reduction": 0.1530101949931264, "selected_objective": 29.709815360605717, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.44.mlp.gate_proj.weight" }, "language_model.model.layers.44.mlp.up_proj": { "base_objective": 39.52928169071674, "choices": { "lloyd": 350467, "lloyd_gptq": 434300, "lloyd_gptq_act_order": 451161, "oqe": 16380, "oqe_gptq": 66325, "oqe_gptq_act_order": 74007 }, "objective_split": "held_out_selection", "relative_reduction": 0.1622033430566101, "selected_objective": 33.11750005185604, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.44.mlp.up_proj.weight" }, "language_model.model.layers.45.mlp.down_proj": { "base_objective": 2.6509535014629364, "choices": { "lloyd": 620095, "lloyd_gptq": 359980, "lloyd_gptq_act_order": 333311, "oqe": 15839, "oqe_gptq": 33593, "oqe_gptq_act_order": 29822 }, "objective_split": "held_out_selection", "relative_reduction": 0.11463350302517994, "selected_objective": 2.3470654152333736, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.45.mlp.down_proj.weight" }, "language_model.model.layers.45.mlp.gate_proj": { "base_objective": 35.404625222086906, "choices": { "lloyd": 350091, "lloyd_gptq": 434837, "lloyd_gptq_act_order": 450237, "oqe": 16471, "oqe_gptq": 67193, "oqe_gptq_act_order": 73811 }, "objective_split": "held_out_selection", "relative_reduction": 0.15102366364421527, "selected_objective": 30.057689011096954, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.45.mlp.gate_proj.weight" }, "language_model.model.layers.45.mlp.up_proj": { "base_objective": 40.0932025462389, "choices": { "lloyd": 348361, "lloyd_gptq": 434745, "lloyd_gptq_act_order": 452788, "oqe": 15890, "oqe_gptq": 66802, "oqe_gptq_act_order": 74054 }, "objective_split": "held_out_selection", "relative_reduction": 0.1606201623900816, "selected_objective": 33.653425842523575, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.45.mlp.up_proj.weight" }, "language_model.model.layers.46.mlp.down_proj": { "base_objective": 3.013966754078865, "choices": { "lloyd": 576657, "lloyd_gptq": 368583, "lloyd_gptq_act_order": 370099, "oqe": 13961, "oqe_gptq": 32033, "oqe_gptq_act_order": 31307 }, "objective_split": "held_out_selection", "relative_reduction": 0.12459623909132367, "selected_objective": 2.638437831774354, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.46.mlp.down_proj.weight" }, "language_model.model.layers.46.mlp.gate_proj": { "base_objective": 35.475910902023315, "choices": { "lloyd": 325269, "lloyd_gptq": 437876, "lloyd_gptq_act_order": 472842, "oqe": 15372, "oqe_gptq": 66013, "oqe_gptq_act_order": 75268 }, "objective_split": "held_out_selection", "relative_reduction": 0.15988931428645736, "selected_objective": 29.80369183421135, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.46.mlp.gate_proj.weight" }, "language_model.model.layers.46.mlp.up_proj": { "base_objective": 41.10296951234341, "choices": { "lloyd": 324332, "lloyd_gptq": 438860, "lloyd_gptq_act_order": 473730, "oqe": 15181, "oqe_gptq": 64832, "oqe_gptq_act_order": 75705 }, "objective_split": "held_out_selection", "relative_reduction": 0.16973105130776323, "selected_objective": 34.12651928514242, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.46.mlp.up_proj.weight" }, "language_model.model.layers.47.mlp.down_proj": { "base_objective": 3.729852069169283, "choices": { "lloyd": 566597, "lloyd_gptq": 369281, "lloyd_gptq_act_order": 380141, "oqe": 14847, "oqe_gptq": 31225, "oqe_gptq_act_order": 30549 }, "objective_split": "held_out_selection", "relative_reduction": 0.12691591126310597, "selected_objective": 3.256474494934082, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.47.mlp.down_proj.weight" }, "language_model.model.layers.47.mlp.gate_proj": { "base_objective": 39.14089739322662, "choices": { "lloyd": 351160, "lloyd_gptq": 433539, "lloyd_gptq_act_order": 455719, "oqe": 15709, "oqe_gptq": 64315, "oqe_gptq_act_order": 72198 }, "objective_split": "held_out_selection", "relative_reduction": 0.15696031024709334, "selected_objective": 32.997329995036125, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.47.mlp.gate_proj.weight" }, "language_model.model.layers.47.mlp.up_proj": { "base_objective": 45.30843698978424, "choices": { "lloyd": 349551, "lloyd_gptq": 432553, "lloyd_gptq_act_order": 459338, "oqe": 15756, "oqe_gptq": 63093, "oqe_gptq_act_order": 72349 }, "objective_split": "held_out_selection", "relative_reduction": 0.16592031495217818, "selected_objective": 37.79084685444832, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.47.mlp.up_proj.weight" }, "language_model.model.layers.48.mlp.down_proj": { "base_objective": 4.594788264483213, "choices": { "lloyd": 499244, "lloyd_gptq": 384241, "lloyd_gptq_act_order": 430977, "oqe": 12776, "oqe_gptq": 31417, "oqe_gptq_act_order": 33985 }, "objective_split": "held_out_selection", "relative_reduction": 0.13075363639768334, "selected_objective": 3.994002990424633, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.48.mlp.down_proj.weight" }, "language_model.model.layers.48.mlp.gate_proj": { "base_objective": 43.235496282577515, "choices": { "lloyd": 311651, "lloyd_gptq": 436274, "lloyd_gptq_act_order": 488833, "oqe": 14730, "oqe_gptq": 64395, "oqe_gptq_act_order": 76757 }, "objective_split": "held_out_selection", "relative_reduction": 0.16084609021454754, "selected_objective": 36.28123574703932, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.48.mlp.gate_proj.weight" }, "language_model.model.layers.48.mlp.up_proj": { "base_objective": 50.1230186522007, "choices": { "lloyd": 310559, "lloyd_gptq": 437277, "lloyd_gptq_act_order": 489236, "oqe": 14480, "oqe_gptq": 63979, "oqe_gptq_act_order": 77109 }, "objective_split": "held_out_selection", "relative_reduction": 0.1690722007865174, "selected_objective": 41.64860957860947, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.48.mlp.up_proj.weight" }, "language_model.model.layers.49.mlp.down_proj": { "base_objective": 5.9500672817230225, "choices": { "lloyd": 449780, "lloyd_gptq": 390904, "lloyd_gptq_act_order": 474625, "oqe": 12428, "oqe_gptq": 29619, "oqe_gptq_act_order": 35284 }, "objective_split": "held_out_selection", "relative_reduction": 0.14279070652182846, "selected_objective": 5.100452970713377, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.49.mlp.down_proj.weight" }, "language_model.model.layers.49.mlp.gate_proj": { "base_objective": 46.954423651099205, "choices": { "lloyd": 296579, "lloyd_gptq": 434572, "lloyd_gptq_act_order": 505986, "oqe": 14253, "oqe_gptq": 62964, "oqe_gptq_act_order": 78286 }, "objective_split": "held_out_selection", "relative_reduction": 0.1631219428341909, "selected_objective": 39.29512684047222, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.49.mlp.gate_proj.weight" }, "language_model.model.layers.49.mlp.up_proj": { "base_objective": 54.474848076701164, "choices": { "lloyd": 294612, "lloyd_gptq": 434088, "lloyd_gptq_act_order": 509082, "oqe": 13473, "oqe_gptq": 62516, "oqe_gptq_act_order": 78869 }, "objective_split": "held_out_selection", "relative_reduction": 0.17016972652659768, "selected_objective": 45.20487807691097, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.49.mlp.up_proj.weight" }, "language_model.model.layers.5.mlp.gate_proj": { "base_objective": 2.1441610823385417, "choices": { "lloyd": 263886, "lloyd_gptq": 437747, "lloyd_gptq_act_order": 506225, "oqe": 15063, "oqe_gptq": 76538, "oqe_gptq_act_order": 93181 }, "objective_split": "held_out_selection", "relative_reduction": 0.13555316589408695, "selected_objective": 1.8535132594406605, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.5.mlp.gate_proj.weight" }, "language_model.model.layers.5.mlp.up_proj": { "base_objective": 2.023600204847753, "choices": { "lloyd": 264713, "lloyd_gptq": 438329, "lloyd_gptq_act_order": 505379, "oqe": 14731, "oqe_gptq": 76791, "oqe_gptq_act_order": 92697 }, "objective_split": "held_out_selection", "relative_reduction": 0.13610618931004848, "selected_objective": 1.7481756922788918, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.5.mlp.up_proj.weight" }, "language_model.model.layers.50.mlp.down_proj": { "base_objective": 10.223594278097153, "choices": { "lloyd": 317289, "lloyd_gptq": 388393, "lloyd_gptq_act_order": 603393, "oqe": 9753, "oqe_gptq": 28713, "oqe_gptq_act_order": 45099 }, "objective_split": "held_out_selection", "relative_reduction": 0.1585315350715355, "selected_objective": 8.602832183241844, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.50.mlp.down_proj.weight" }, "language_model.model.layers.50.mlp.gate_proj": { "base_objective": 57.470287680625916, "choices": { "lloyd": 239137, "lloyd_gptq": 430999, "lloyd_gptq_act_order": 562156, "oqe": 12142, "oqe_gptq": 61519, "oqe_gptq_act_order": 86687 }, "objective_split": "held_out_selection", "relative_reduction": 0.18065236329677814, "selected_objective": 47.08814439177513, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.50.mlp.gate_proj.weight" }, "language_model.model.layers.50.mlp.up_proj": { "base_objective": 64.5475409924984, "choices": { "lloyd": 237359, "lloyd_gptq": 432574, "lloyd_gptq_act_order": 562636, "oqe": 11941, "oqe_gptq": 61431, "oqe_gptq_act_order": 86699 }, "objective_split": "held_out_selection", "relative_reduction": 0.1876215248974935, "selected_objective": 52.43703292310238, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.50.mlp.up_proj.weight" }, "language_model.model.layers.51.mlp.down_proj": { "base_objective": 12.75507465004921, "choices": { "lloyd": 295831, "lloyd_gptq": 389741, "lloyd_gptq_act_order": 620322, "oqe": 11311, "oqe_gptq": 28980, "oqe_gptq_act_order": 46455 }, "objective_split": "held_out_selection", "relative_reduction": 0.17503146486482712, "selected_objective": 10.522535249590874, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.51.mlp.down_proj.weight" }, "language_model.model.layers.51.mlp.gate_proj": { "base_objective": 65.7016863077879, "choices": { "lloyd": 272854, "lloyd_gptq": 427445, "lloyd_gptq_act_order": 532168, "oqe": 13988, "oqe_gptq": 62829, "oqe_gptq_act_order": 83356 }, "objective_split": "held_out_selection", "relative_reduction": 0.16352966563187077, "selected_objective": 54.95751151442528, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.51.mlp.gate_proj.weight" }, "language_model.model.layers.51.mlp.up_proj": { "base_objective": 75.4831803292036, "choices": { "lloyd": 272337, "lloyd_gptq": 426689, "lloyd_gptq_act_order": 534346, "oqe": 13450, "oqe_gptq": 62100, "oqe_gptq_act_order": 83718 }, "objective_split": "held_out_selection", "relative_reduction": 0.16866211567383027, "selected_objective": 62.752027437090874, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.51.mlp.up_proj.weight" }, "language_model.model.layers.52.mlp.down_proj": { "base_objective": 12.249535948038101, "choices": { "lloyd": 296094, "lloyd_gptq": 390663, "lloyd_gptq_act_order": 611897, "oqe": 13257, "oqe_gptq": 31474, "oqe_gptq_act_order": 49255 }, "objective_split": "held_out_selection", "relative_reduction": 0.18418409430380356, "selected_objective": 9.99336626380682, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.52.mlp.down_proj.weight" }, "language_model.model.layers.52.mlp.gate_proj": { "base_objective": 81.82565066218376, "choices": { "lloyd": 233871, "lloyd_gptq": 436389, "lloyd_gptq_act_order": 544083, "oqe": 13895, "oqe_gptq": 70307, "oqe_gptq_act_order": 94095 }, "objective_split": "held_out_selection", "relative_reduction": 0.16614068939122184, "selected_objective": 68.23108065128326, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.52.mlp.gate_proj.weight" }, "language_model.model.layers.52.mlp.up_proj": { "base_objective": 82.10960099101067, "choices": { "lloyd": 232337, "lloyd_gptq": 435801, "lloyd_gptq_act_order": 546689, "oqe": 13188, "oqe_gptq": 70241, "oqe_gptq_act_order": 94384 }, "objective_split": "held_out_selection", "relative_reduction": 0.17089702671092633, "selected_objective": 68.07731431722641, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.52.mlp.up_proj.weight" }, "language_model.model.layers.53.mlp.down_proj": { "base_objective": 13.190371006727219, "choices": { "lloyd": 311080, "lloyd_gptq": 386596, "lloyd_gptq_act_order": 594578, "oqe": 15057, "oqe_gptq": 33899, "oqe_gptq_act_order": 51430 }, "objective_split": "held_out_selection", "relative_reduction": 0.17457245438608904, "selected_objective": 10.88769556581974, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.53.mlp.down_proj.weight" }, "language_model.model.layers.53.mlp.gate_proj": { "base_objective": 90.54390305280685, "choices": { "lloyd": 230528, "lloyd_gptq": 438978, "lloyd_gptq_act_order": 539797, "oqe": 13978, "oqe_gptq": 73263, "oqe_gptq_act_order": 96096 }, "objective_split": "held_out_selection", "relative_reduction": 0.16376960001725893, "selected_objective": 75.7155642658472, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.53.mlp.gate_proj.weight" }, "language_model.model.layers.53.mlp.up_proj": { "base_objective": 85.9647676050663, "choices": { "lloyd": 230535, "lloyd_gptq": 438609, "lloyd_gptq_act_order": 541100, "oqe": 13733, "oqe_gptq": 72121, "oqe_gptq_act_order": 96542 }, "objective_split": "held_out_selection", "relative_reduction": 0.16762560917819913, "selected_objective": 71.55487106740475, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.53.mlp.up_proj.weight" }, "language_model.model.layers.54.mlp.down_proj": { "base_objective": 17.27578128874302, "choices": { "lloyd": 224105, "lloyd_gptq": 373708, "lloyd_gptq_act_order": 689029, "oqe": 12611, "oqe_gptq": 32690, "oqe_gptq_act_order": 60497 }, "objective_split": "held_out_selection", "relative_reduction": 0.22303261226225302, "selected_objective": 13.422718659043312, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.54.mlp.down_proj.weight" }, "language_model.model.layers.54.mlp.gate_proj": { "base_objective": 110.97225946187973, "choices": { "lloyd": 233592, "lloyd_gptq": 439469, "lloyd_gptq_act_order": 537362, "oqe": 14141, "oqe_gptq": 72906, "oqe_gptq_act_order": 95170 }, "objective_split": "held_out_selection", "relative_reduction": 0.1660973143389468, "selected_objective": 92.54006519913673, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.54.mlp.gate_proj.weight" }, "language_model.model.layers.54.mlp.up_proj": { "base_objective": 99.54399907588959, "choices": { "lloyd": 232836, "lloyd_gptq": 438453, "lloyd_gptq_act_order": 539576, "oqe": 13994, "oqe_gptq": 72396, "oqe_gptq_act_order": 95385 }, "objective_split": "held_out_selection", "relative_reduction": 0.17096574172712475, "selected_objective": 82.5253854393959, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.54.mlp.up_proj.weight" }, "language_model.model.layers.55.mlp.down_proj": { "base_objective": 13.30400662124157, "choices": { "lloyd": 297238, "lloyd_gptq": 402468, "lloyd_gptq_act_order": 584452, "oqe": 17950, "oqe_gptq": 36992, "oqe_gptq_act_order": 53540 }, "objective_split": "held_out_selection", "relative_reduction": 0.1977003733949796, "selected_objective": 10.67379954457283, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.55.mlp.down_proj.weight" }, "language_model.model.layers.55.mlp.gate_proj": { "base_objective": 104.49821153283119, "choices": { "lloyd": 264801, "lloyd_gptq": 430970, "lloyd_gptq_act_order": 519713, "oqe": 15598, "oqe_gptq": 70565, "oqe_gptq_act_order": 90993 }, "objective_split": "held_out_selection", "relative_reduction": 0.15734810053927517, "selected_objective": 88.05561643838882, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.55.mlp.gate_proj.weight" }, "language_model.model.layers.55.mlp.up_proj": { "base_objective": 97.40239009261131, "choices": { "lloyd": 262637, "lloyd_gptq": 429701, "lloyd_gptq_act_order": 524171, "oqe": 14933, "oqe_gptq": 69731, "oqe_gptq_act_order": 91467 }, "objective_split": "held_out_selection", "relative_reduction": 0.161711467129805, "selected_objective": 81.65130668878555, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.55.mlp.up_proj.weight" }, "language_model.model.layers.56.mlp.down_proj": { "base_objective": 13.06900504231453, "choices": { "lloyd": 315221, "lloyd_gptq": 410370, "lloyd_gptq_act_order": 559025, "oqe": 19366, "oqe_gptq": 37887, "oqe_gptq_act_order": 50771 }, "objective_split": "held_out_selection", "relative_reduction": 0.21046844776851759, "selected_objective": 10.31839183717966, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.56.mlp.down_proj.weight" }, "language_model.model.layers.56.mlp.gate_proj": { "base_objective": 114.62913510203362, "choices": { "lloyd": 250525, "lloyd_gptq": 432258, "lloyd_gptq_act_order": 523172, "oqe": 16039, "oqe_gptq": 74539, "oqe_gptq_act_order": 96107 }, "objective_split": "held_out_selection", "relative_reduction": 0.16291670272617587, "selected_objective": 95.95413437485695, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.56.mlp.gate_proj.weight" }, "language_model.model.layers.56.mlp.up_proj": { "base_objective": 103.53454461693764, "choices": { "lloyd": 247857, "lloyd_gptq": 433624, "lloyd_gptq_act_order": 527384, "oqe": 15232, "oqe_gptq": 73303, "oqe_gptq_act_order": 95240 }, "objective_split": "held_out_selection", "relative_reduction": 0.16691963433034662, "selected_objective": 86.25259628891945, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.56.mlp.up_proj.weight" }, "language_model.model.layers.57.mlp.down_proj": { "base_objective": 12.922234252095222, "choices": { "lloyd": 359042, "lloyd_gptq": 399131, "lloyd_gptq_act_order": 525660, "oqe": 21654, "oqe_gptq": 38275, "oqe_gptq_act_order": 48878 }, "objective_split": "held_out_selection", "relative_reduction": 0.19755656613547923, "selected_objective": 10.369362026453018, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.57.mlp.down_proj.weight" }, "language_model.model.layers.57.mlp.gate_proj": { "base_objective": 130.72459211945534, "choices": { "lloyd": 259871, "lloyd_gptq": 434427, "lloyd_gptq_act_order": 510134, "oqe": 16502, "oqe_gptq": 76302, "oqe_gptq_act_order": 95404 }, "objective_split": "held_out_selection", "relative_reduction": 0.1607391126863139, "selected_objective": 109.71203717589378, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.57.mlp.gate_proj.weight" }, "language_model.model.layers.57.mlp.up_proj": { "base_objective": 116.06509140133858, "choices": { "lloyd": 257589, "lloyd_gptq": 435525, "lloyd_gptq_act_order": 514014, "oqe": 16188, "oqe_gptq": 74704, "oqe_gptq_act_order": 94620 }, "objective_split": "held_out_selection", "relative_reduction": 0.16613053337032768, "selected_objective": 96.78313586115837, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.57.mlp.up_proj.weight" }, "language_model.model.layers.58.mlp.down_proj": { "base_objective": 16.639235630631447, "choices": { "lloyd": 332336, "lloyd_gptq": 397895, "lloyd_gptq_act_order": 561106, "oqe": 18472, "oqe_gptq": 34597, "oqe_gptq_act_order": 48234 }, "objective_split": "held_out_selection", "relative_reduction": 0.25495150250281917, "selected_objective": 12.397037506103516, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.58.mlp.down_proj.weight" }, "language_model.model.layers.58.mlp.gate_proj": { "base_objective": 148.90509143471718, "choices": { "lloyd": 270745, "lloyd_gptq": 434846, "lloyd_gptq_act_order": 501431, "oqe": 16706, "oqe_gptq": 75908, "oqe_gptq_act_order": 93004 }, "objective_split": "held_out_selection", "relative_reduction": 0.15912056025513666, "selected_objective": 125.21122986078262, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.58.mlp.gate_proj.weight" }, "language_model.model.layers.58.mlp.up_proj": { "base_objective": 132.17830076813698, "choices": { "lloyd": 269267, "lloyd_gptq": 434154, "lloyd_gptq_act_order": 505864, "oqe": 16456, "oqe_gptq": 74582, "oqe_gptq_act_order": 92317 }, "objective_split": "held_out_selection", "relative_reduction": 0.16475080135102127, "selected_objective": 110.4018197953701, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.58.mlp.up_proj.weight" }, "language_model.model.layers.59.mlp.down_proj": { "base_objective": 19.23992644250393, "choices": { "lloyd": 239113, "lloyd_gptq": 375510, "lloyd_gptq_act_order": 680424, "oqe": 13178, "oqe_gptq": 30812, "oqe_gptq_act_order": 53603 }, "objective_split": "held_out_selection", "relative_reduction": 0.28811135421963197, "selected_objective": 13.696685180068016, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.59.mlp.down_proj.weight" }, "language_model.model.layers.59.mlp.gate_proj": { "base_objective": 146.43485543131828, "choices": { "lloyd": 265697, "lloyd_gptq": 429108, "lloyd_gptq_act_order": 507828, "oqe": 17227, "oqe_gptq": 76966, "oqe_gptq_act_order": 95814 }, "objective_split": "held_out_selection", "relative_reduction": 0.16536815061125154, "selected_objective": 122.21919420361519, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.59.mlp.gate_proj.weight" }, "language_model.model.layers.59.mlp.up_proj": { "base_objective": 133.075846016407, "choices": { "lloyd": 264274, "lloyd_gptq": 429400, "lloyd_gptq_act_order": 512085, "oqe": 16498, "oqe_gptq": 75108, "oqe_gptq_act_order": 95275 }, "objective_split": "held_out_selection", "relative_reduction": 0.1702033110309198, "selected_objective": 110.4258964061737, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.59.mlp.up_proj.weight" }, "language_model.model.layers.6.mlp.gate_proj": { "base_objective": 2.906887259334326, "choices": { "lloyd": 256035, "lloyd_gptq": 439384, "lloyd_gptq_act_order": 530046, "oqe": 13459, "oqe_gptq": 67261, "oqe_gptq_act_order": 86455 }, "objective_split": "held_out_selection", "relative_reduction": 0.18500618208797193, "selected_objective": 2.369095145724714, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.6.mlp.gate_proj.weight" }, "language_model.model.layers.6.mlp.up_proj": { "base_objective": 2.7272594599053264, "choices": { "lloyd": 255057, "lloyd_gptq": 438697, "lloyd_gptq_act_order": 531337, "oqe": 13448, "oqe_gptq": 67808, "oqe_gptq_act_order": 86293 }, "objective_split": "held_out_selection", "relative_reduction": 0.18881946802378624, "selected_objective": 2.212299779523164, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.6.mlp.up_proj.weight" }, "language_model.model.layers.60.mlp.down_proj": { "base_objective": 24.469972133636475, "choices": { "lloyd": 166712, "lloyd_gptq": 350421, "lloyd_gptq_act_order": 783120, "oqe": 9164, "oqe_gptq": 25430, "oqe_gptq_act_order": 57793 }, "objective_split": "held_out_selection", "relative_reduction": 0.27967102987453474, "selected_objective": 17.626429826021194, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.60.mlp.down_proj.weight" }, "language_model.model.layers.60.mlp.gate_proj": { "base_objective": 147.64978513121605, "choices": { "lloyd": 246506, "lloyd_gptq": 431954, "lloyd_gptq_act_order": 520406, "oqe": 16495, "oqe_gptq": 77801, "oqe_gptq_act_order": 99478 }, "objective_split": "held_out_selection", "relative_reduction": 0.17067267910809303, "selected_objective": 122.45000073313713, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.60.mlp.gate_proj.weight" }, "language_model.model.layers.60.mlp.up_proj": { "base_objective": 137.2575252354145, "choices": { "lloyd": 244818, "lloyd_gptq": 431720, "lloyd_gptq_act_order": 525558, "oqe": 15899, "oqe_gptq": 76350, "oqe_gptq_act_order": 98295 }, "objective_split": "held_out_selection", "relative_reduction": 0.17680443983367222, "selected_objective": 112.9897853732109, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.60.mlp.up_proj.weight" }, "language_model.model.layers.61.mlp.down_proj": { "base_objective": 32.82849767804146, "choices": { "lloyd": 110420, "lloyd_gptq": 308288, "lloyd_gptq_act_order": 882133, "oqe": 6361, "oqe_gptq": 21999, "oqe_gptq_act_order": 63439 }, "objective_split": "held_out_selection", "relative_reduction": 0.30758900630999597, "selected_objective": 22.730812698602676, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.61.mlp.down_proj.weight" }, "language_model.model.layers.61.mlp.gate_proj": { "base_objective": 159.1120822429657, "choices": { "lloyd": 222414, "lloyd_gptq": 434198, "lloyd_gptq_act_order": 532981, "oqe": 15853, "oqe_gptq": 81204, "oqe_gptq_act_order": 105990 }, "objective_split": "held_out_selection", "relative_reduction": 0.17774123062621727, "selected_objective": 130.8313049376011, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.61.mlp.gate_proj.weight" }, "language_model.model.layers.61.mlp.up_proj": { "base_objective": 149.12146452069283, "choices": { "lloyd": 220397, "lloyd_gptq": 435596, "lloyd_gptq_act_order": 536006, "oqe": 15482, "oqe_gptq": 80324, "oqe_gptq_act_order": 104835 }, "objective_split": "held_out_selection", "relative_reduction": 0.1837944158495215, "selected_objective": 121.71377205848694, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.61.mlp.up_proj.weight" }, "language_model.model.layers.62.mlp.down_proj": { "base_objective": 58.632885813713074, "choices": { "lloyd": 54941, "lloyd_gptq": 238647, "lloyd_gptq_act_order": 1018377, "oqe": 2464, "oqe_gptq": 14243, "oqe_gptq_act_order": 63968 }, "objective_split": "held_out_selection", "relative_reduction": 0.3329566945027226, "selected_objective": 39.11067396402359, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.62.mlp.down_proj.weight" }, "language_model.model.layers.62.mlp.gate_proj": { "base_objective": 154.0579994916916, "choices": { "lloyd": 177883, "lloyd_gptq": 438488, "lloyd_gptq_act_order": 550975, "oqe": 15570, "oqe_gptq": 90017, "oqe_gptq_act_order": 119707 }, "objective_split": "held_out_selection", "relative_reduction": 0.1976323794856063, "selected_objective": 123.61115047335625, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.62.mlp.gate_proj.weight" }, "language_model.model.layers.62.mlp.up_proj": { "base_objective": 144.79678189754486, "choices": { "lloyd": 176845, "lloyd_gptq": 437551, "lloyd_gptq_act_order": 553403, "oqe": 15173, "oqe_gptq": 89805, "oqe_gptq_act_order": 119863 }, "objective_split": "held_out_selection", "relative_reduction": 0.2021813530260831, "selected_objective": 115.52157261967659, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.62.mlp.up_proj.weight" }, "language_model.model.layers.63.mlp.down_proj": { "base_objective": 159.80059432983398, "choices": { "lloyd": 23731, "lloyd_gptq": 174042, "lloyd_gptq_act_order": 1118097, "oqe": 898, "oqe_gptq": 9093, "oqe_gptq_act_order": 66779 }, "objective_split": "held_out_selection", "relative_reduction": 0.4477737421231748, "selected_objective": 88.24608421325684, "shape": [ 5120, 17408 ], "source": "model.language_model.layers.63.mlp.down_proj.weight" }, "language_model.model.layers.63.mlp.gate_proj": { "base_objective": 140.77778726816177, "choices": { "lloyd": 133946, "lloyd_gptq": 430857, "lloyd_gptq_act_order": 573975, "oqe": 14483, "oqe_gptq": 98906, "oqe_gptq_act_order": 140473 }, "objective_split": "held_out_selection", "relative_reduction": 0.23578961917395636, "selected_objective": 107.58384642004967, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.63.mlp.gate_proj.weight" }, "language_model.model.layers.63.mlp.up_proj": { "base_objective": 128.1735941171646, "choices": { "lloyd": 134144, "lloyd_gptq": 430943, "lloyd_gptq_act_order": 573594, "oqe": 14363, "oqe_gptq": 99611, "oqe_gptq_act_order": 139985 }, "objective_split": "held_out_selection", "relative_reduction": 0.24127234967226785, "selected_objective": 97.24884989857674, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.63.mlp.up_proj.weight" }, "language_model.model.layers.7.mlp.gate_proj": { "base_objective": 3.0673272851854563, "choices": { "lloyd": 299325, "lloyd_gptq": 442846, "lloyd_gptq_act_order": 468905, "oqe": 16808, "oqe_gptq": 78290, "oqe_gptq_act_order": 86466 }, "objective_split": "held_out_selection", "relative_reduction": 0.13105607110960527, "selected_objective": 2.6653354223817587, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.7.mlp.gate_proj.weight" }, "language_model.model.layers.7.mlp.up_proj": { "base_objective": 2.8666873248293996, "choices": { "lloyd": 298944, "lloyd_gptq": 442867, "lloyd_gptq_act_order": 469899, "oqe": 16465, "oqe_gptq": 78097, "oqe_gptq_act_order": 86368 }, "objective_split": "held_out_selection", "relative_reduction": 0.1324936193376237, "selected_objective": 2.4868695456534624, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.7.mlp.up_proj.weight" }, "language_model.model.layers.8.mlp.gate_proj": { "base_objective": 3.360856929793954, "choices": { "lloyd": 317456, "lloyd_gptq": 443812, "lloyd_gptq_act_order": 452917, "oqe": 17035, "oqe_gptq": 78158, "oqe_gptq_act_order": 83262 }, "objective_split": "held_out_selection", "relative_reduction": 0.12601554693221106, "selected_objective": 2.937336705625057, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.8.mlp.gate_proj.weight" }, "language_model.model.layers.8.mlp.up_proj": { "base_objective": 3.165446643717587, "choices": { "lloyd": 317727, "lloyd_gptq": 442813, "lloyd_gptq_act_order": 453128, "oqe": 17086, "oqe_gptq": 78208, "oqe_gptq_act_order": 83678 }, "objective_split": "held_out_selection", "relative_reduction": 0.12707590354523968, "selected_objective": 2.7631946513429284, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.8.mlp.up_proj.weight" }, "language_model.model.layers.9.mlp.gate_proj": { "base_objective": 3.7657600846141577, "choices": { "lloyd": 292068, "lloyd_gptq": 445854, "lloyd_gptq_act_order": 471057, "oqe": 16671, "oqe_gptq": 79878, "oqe_gptq_act_order": 87112 }, "objective_split": "held_out_selection", "relative_reduction": 0.13021976624875264, "selected_objective": 3.275383686646819, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.9.mlp.gate_proj.weight" }, "language_model.model.layers.9.mlp.up_proj": { "base_objective": 3.5701426742598414, "choices": { "lloyd": 293503, "lloyd_gptq": 445506, "lloyd_gptq_act_order": 469945, "oqe": 16582, "oqe_gptq": 79500, "oqe_gptq_act_order": 87604 }, "objective_split": "held_out_selection", "relative_reduction": 0.1311897692386199, "selected_objective": 3.1017764806747437, "shape": [ 17408, 5120 ], "source": "model.language_model.layers.9.mlp.up_proj.weight" } }, "objective_split": "held_out_selection", "peak_memory_bytes": 12226340768, "relative_reduction": 0.18088393617309562, "row_chunk": 64, "selected_objective": 4635.816319292295, "student": "/Users/litwein/.omlx/models/Qwen3.8-27B-oQ4e-XL-mtp-backbone", "version": 3 }