| { |
| "base_objective": 5659.535350379767, |
| "bf16": "/Users/litwein/.omlx/models/Qwen3.8-27B", |
| "bits": 4, |
| "choices": { |
| "lloyd": 55352164, |
| "lloyd_gptq": 69193418, |
| "lloyd_gptq_act_order": 79523367, |
| "oqe": 2721430, |
| "oqe_gptq": 10569443, |
| "oqe_gptq_act_order": 12425778 |
| }, |
| "damp": 0.01, |
| "format": "omlx-m4q-initializer-report", |
| "group_size": 64, |
| "hessian_cache": "/Users/litwein/GithubControl/dwq/m4q_hessian_qwen38_q4g64.safetensors", |
| "hessian_identity": "618c11aee7ab24a762e0baac4102e5954f03adc165a97a078b7b07cab48f96e8", |
| "hessian_spec": { |
| "bits": 4, |
| "calibration": "5790d72b58b96b3ecb9a39c9c6fcc44249ef24d47a05d9dd7bb8eba0c0d94bcb", |
| "format": "omlx-m4q-group-hessian", |
| "group_size": 64, |
| "num_samples": 256, |
| "seed": 123, |
| "selection_samples": 128, |
| "sequence_length": 512, |
| "student": "e18b6a19f1816f2639ab1d0b10a8c8830d35e843c6d4ef65f9c5b6d836c7a797", |
| "tokens_per_batch": 128, |
| "version": 3 |
| }, |
| "module_count": 165, |
| "modules": { |
| "language_model.model.layers.0.mlp.gate_proj": { |
| "base_objective": 0.4595101249869913, |
| "choices": { |
| "lloyd": 164726, |
| "lloyd_gptq": 429836, |
| "lloyd_gptq_act_order": 651124, |
| "oqe": 7599, |
| "oqe_gptq": 52006, |
| "oqe_gptq_act_order": 87349 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1887429693731657, |
| "selected_objective": 0.37278081953991205, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.0.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.0.mlp.up_proj": { |
| "base_objective": 0.4266967566218227, |
| "choices": { |
| "lloyd": 164989, |
| "lloyd_gptq": 429936, |
| "lloyd_gptq_act_order": 650903, |
| "oqe": 7769, |
| "oqe_gptq": 51737, |
| "oqe_gptq_act_order": 87306 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18936618408979622, |
| "selected_objective": 0.3458948200568557, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.0.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.1.mlp.gate_proj": { |
| "base_objective": 0.841862749774009, |
| "choices": { |
| "lloyd": 192495, |
| "lloyd_gptq": 439414, |
| "lloyd_gptq_act_order": 606075, |
| "oqe": 9064, |
| "oqe_gptq": 58416, |
| "oqe_gptq_act_order": 87176 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16566517744466647, |
| "selected_objective": 0.702395407948643, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.1.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.1.mlp.up_proj": { |
| "base_objective": 0.8054950113873929, |
| "choices": { |
| "lloyd": 192230, |
| "lloyd_gptq": 439436, |
| "lloyd_gptq_act_order": 605823, |
| "oqe": 8940, |
| "oqe_gptq": 58466, |
| "oqe_gptq_act_order": 87745 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1664205706675339, |
| "selected_objective": 0.6714440719224513, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.1.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.10.mlp.gate_proj": { |
| "base_objective": 4.020561162382364, |
| "choices": { |
| "lloyd": 264217, |
| "lloyd_gptq": 444758, |
| "lloyd_gptq_act_order": 499503, |
| "oqe": 15114, |
| "oqe_gptq": 77648, |
| "oqe_gptq_act_order": 91400 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1391670140316148, |
| "selected_objective": 3.4610316706821322, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.10.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.10.mlp.up_proj": { |
| "base_objective": 3.83932751044631, |
| "choices": { |
| "lloyd": 263582, |
| "lloyd_gptq": 444974, |
| "lloyd_gptq_act_order": 499638, |
| "oqe": 15256, |
| "oqe_gptq": 77218, |
| "oqe_gptq_act_order": 91972 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14040432292579086, |
| "selected_objective": 3.3002693308517337, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.10.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.11.mlp.gate_proj": { |
| "base_objective": 4.344947325065732, |
| "choices": { |
| "lloyd": 275718, |
| "lloyd_gptq": 443370, |
| "lloyd_gptq_act_order": 489095, |
| "oqe": 15751, |
| "oqe_gptq": 78806, |
| "oqe_gptq_act_order": 89900 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1433372446226222, |
| "selected_objective": 3.722154547460377, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.11.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.11.mlp.up_proj": { |
| "base_objective": 4.170417866669595, |
| "choices": { |
| "lloyd": 276193, |
| "lloyd_gptq": 444767, |
| "lloyd_gptq_act_order": 489688, |
| "oqe": 15571, |
| "oqe_gptq": 77410, |
| "oqe_gptq_act_order": 89011 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14611898012476432, |
| "selected_objective": 3.5610406612977386, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.11.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.12.mlp.gate_proj": { |
| "base_objective": 4.649569490924478, |
| "choices": { |
| "lloyd": 269597, |
| "lloyd_gptq": 447335, |
| "lloyd_gptq_act_order": 490516, |
| "oqe": 15483, |
| "oqe_gptq": 79385, |
| "oqe_gptq_act_order": 90324 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14032431501623455, |
| "selected_objective": 3.997121836990118, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.12.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.12.mlp.up_proj": { |
| "base_objective": 4.452637831680477, |
| "choices": { |
| "lloyd": 270118, |
| "lloyd_gptq": 447614, |
| "lloyd_gptq_act_order": 490374, |
| "oqe": 15696, |
| "oqe_gptq": 78653, |
| "oqe_gptq_act_order": 90185 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14236809700308461, |
| "selected_objective": 3.818724256940186, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.12.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.13.mlp.gate_proj": { |
| "base_objective": 4.998010698705912, |
| "choices": { |
| "lloyd": 295743, |
| "lloyd_gptq": 447892, |
| "lloyd_gptq_act_order": 468766, |
| "oqe": 16158, |
| "oqe_gptq": 78253, |
| "oqe_gptq_act_order": 85828 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13406185295091122, |
| "selected_objective": 4.327968123368919, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.13.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.13.mlp.up_proj": { |
| "base_objective": 4.78683852404356, |
| "choices": { |
| "lloyd": 295818, |
| "lloyd_gptq": 448134, |
| "lloyd_gptq_act_order": 468729, |
| "oqe": 15992, |
| "oqe_gptq": 78147, |
| "oqe_gptq_act_order": 85820 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1356358343619889, |
| "selected_objective": 4.1375716868788, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.13.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.14.mlp.gate_proj": { |
| "base_objective": 5.284640742465854, |
| "choices": { |
| "lloyd": 301950, |
| "lloyd_gptq": 449201, |
| "lloyd_gptq_act_order": 460660, |
| "oqe": 16408, |
| "oqe_gptq": 79123, |
| "oqe_gptq_act_order": 85298 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13059158640903312, |
| "selected_objective": 4.594511124305427, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.14.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.14.mlp.up_proj": { |
| "base_objective": 5.076406957581639, |
| "choices": { |
| "lloyd": 301542, |
| "lloyd_gptq": 448259, |
| "lloyd_gptq_act_order": 461965, |
| "oqe": 16326, |
| "oqe_gptq": 78690, |
| "oqe_gptq_act_order": 85858 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13181251819939765, |
| "selected_objective": 4.407272973097861, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.14.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.15.mlp.gate_proj": { |
| "base_objective": 5.961888471618295, |
| "choices": { |
| "lloyd": 323394, |
| "lloyd_gptq": 442245, |
| "lloyd_gptq_act_order": 456135, |
| "oqe": 16342, |
| "oqe_gptq": 74046, |
| "oqe_gptq_act_order": 80478 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13216531733835302, |
| "selected_objective": 5.173933589830995, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.15.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.15.mlp.up_proj": { |
| "base_objective": 5.777877099812031, |
| "choices": { |
| "lloyd": 323561, |
| "lloyd_gptq": 441768, |
| "lloyd_gptq_act_order": 456305, |
| "oqe": 16106, |
| "oqe_gptq": 74135, |
| "oqe_gptq_act_order": 80765 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13364877017666232, |
| "selected_objective": 5.005670931190252, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.15.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.16.mlp.gate_proj": { |
| "base_objective": 6.680432364344597, |
| "choices": { |
| "lloyd": 311781, |
| "lloyd_gptq": 439933, |
| "lloyd_gptq_act_order": 472484, |
| "oqe": 15630, |
| "oqe_gptq": 71826, |
| "oqe_gptq_act_order": 80986 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13830257670998825, |
| "selected_objective": 5.75651135481894, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.16.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.16.mlp.up_proj": { |
| "base_objective": 6.445050187408924, |
| "choices": { |
| "lloyd": 310346, |
| "lloyd_gptq": 441363, |
| "lloyd_gptq_act_order": 473019, |
| "oqe": 15531, |
| "oqe_gptq": 71408, |
| "oqe_gptq_act_order": 80973 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1400478847571618, |
| "selected_objective": 5.542434541508555, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.16.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.17.mlp.gate_proj": { |
| "base_objective": 7.740207539871335, |
| "choices": { |
| "lloyd": 305700, |
| "lloyd_gptq": 436915, |
| "lloyd_gptq_act_order": 480335, |
| "oqe": 15804, |
| "oqe_gptq": 71010, |
| "oqe_gptq_act_order": 82876 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14088131451398545, |
| "selected_objective": 6.6497569270431995, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.17.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.17.mlp.up_proj": { |
| "base_objective": 7.469485396519303, |
| "choices": { |
| "lloyd": 304366, |
| "lloyd_gptq": 437274, |
| "lloyd_gptq_act_order": 482036, |
| "oqe": 15371, |
| "oqe_gptq": 70772, |
| "oqe_gptq_act_order": 82821 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14319540930364583, |
| "selected_objective": 6.399889377877116, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.17.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.18.mlp.gate_proj": { |
| "base_objective": 9.746447626501322, |
| "choices": { |
| "lloyd": 262457, |
| "lloyd_gptq": 434145, |
| "lloyd_gptq_act_order": 525253, |
| "oqe": 14426, |
| "oqe_gptq": 68630, |
| "oqe_gptq_act_order": 87729 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16229036959987198, |
| "selected_objective": 8.164693038910627, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.18.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.18.mlp.up_proj": { |
| "base_objective": 9.25420866906643, |
| "choices": { |
| "lloyd": 261174, |
| "lloyd_gptq": 434464, |
| "lloyd_gptq_act_order": 526519, |
| "oqe": 14026, |
| "oqe_gptq": 68222, |
| "oqe_gptq_act_order": 88235 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1661334167765589, |
| "selected_objective": 7.7167753633111715, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.18.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.19.mlp.gate_proj": { |
| "base_objective": 12.375938680022955, |
| "choices": { |
| "lloyd": 302442, |
| "lloyd_gptq": 435953, |
| "lloyd_gptq_act_order": 480977, |
| "oqe": 16263, |
| "oqe_gptq": 73003, |
| "oqe_gptq_act_order": 84002 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14686997865099183, |
| "selected_objective": 10.558284830302, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.19.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.19.mlp.up_proj": { |
| "base_objective": 11.813799154013395, |
| "choices": { |
| "lloyd": 301316, |
| "lloyd_gptq": 436566, |
| "lloyd_gptq_act_order": 481721, |
| "oqe": 16065, |
| "oqe_gptq": 72542, |
| "oqe_gptq_act_order": 84430 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1527666330706977, |
| "selected_objective": 10.009044833481312, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.19.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.2.mlp.gate_proj": { |
| "base_objective": 1.2813364495523274, |
| "choices": { |
| "lloyd": 258023, |
| "lloyd_gptq": 447063, |
| "lloyd_gptq_act_order": 503744, |
| "oqe": 14439, |
| "oqe_gptq": 76878, |
| "oqe_gptq_act_order": 92493 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1382072220733359, |
| "selected_objective": 1.104246498318389, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.2.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.2.mlp.up_proj": { |
| "base_objective": 1.2072026170790195, |
| "choices": { |
| "lloyd": 257523, |
| "lloyd_gptq": 446138, |
| "lloyd_gptq_act_order": 506039, |
| "oqe": 14168, |
| "oqe_gptq": 76761, |
| "oqe_gptq_act_order": 92011 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1387283762865852, |
| "selected_objective": 1.039729358162731, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.2.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.20.mlp.gate_proj": { |
| "base_objective": 14.964200031012297, |
| "choices": { |
| "lloyd": 314388, |
| "lloyd_gptq": 443164, |
| "lloyd_gptq_act_order": 453221, |
| "oqe": 18016, |
| "oqe_gptq": 79359, |
| "oqe_gptq_act_order": 84492 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.136173876216794, |
| "selected_objective": 12.926466908305883, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.20.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.20.mlp.up_proj": { |
| "base_objective": 13.992726922035217, |
| "choices": { |
| "lloyd": 313577, |
| "lloyd_gptq": 443295, |
| "lloyd_gptq_act_order": 454291, |
| "oqe": 17955, |
| "oqe_gptq": 78776, |
| "oqe_gptq_act_order": 84746 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13983141767871377, |
| "selected_objective": 12.036104079335928, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.20.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.21.mlp.gate_proj": { |
| "base_objective": 16.276506315916777, |
| "choices": { |
| "lloyd": 333716, |
| "lloyd_gptq": 443093, |
| "lloyd_gptq_act_order": 433538, |
| "oqe": 19247, |
| "oqe_gptq": 79980, |
| "oqe_gptq_act_order": 83066 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12927882711807825, |
| "selected_objective": 14.172298669815063, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.21.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.21.mlp.up_proj": { |
| "base_objective": 15.299354836344719, |
| "choices": { |
| "lloyd": 333021, |
| "lloyd_gptq": 442885, |
| "lloyd_gptq_act_order": 434889, |
| "oqe": 19089, |
| "oqe_gptq": 80332, |
| "oqe_gptq_act_order": 82424 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13234915731933494, |
| "selected_objective": 13.274498116225004, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.21.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.22.mlp.gate_proj": { |
| "base_objective": 19.04569412022829, |
| "choices": { |
| "lloyd": 351210, |
| "lloyd_gptq": 439436, |
| "lloyd_gptq_act_order": 423849, |
| "oqe": 19874, |
| "oqe_gptq": 78418, |
| "oqe_gptq_act_order": 79853 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13698001379152547, |
| "selected_objective": 16.436814676970243, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.22.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.22.mlp.up_proj": { |
| "base_objective": 17.05502561107278, |
| "choices": { |
| "lloyd": 349386, |
| "lloyd_gptq": 440151, |
| "lloyd_gptq_act_order": 425158, |
| "oqe": 19583, |
| "oqe_gptq": 78708, |
| "oqe_gptq_act_order": 79654 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14332622932789252, |
| "selected_objective": 14.610593099147081, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.22.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.23.mlp.gate_proj": { |
| "base_objective": 19.95293838530779, |
| "choices": { |
| "lloyd": 382733, |
| "lloyd_gptq": 434152, |
| "lloyd_gptq_act_order": 400737, |
| "oqe": 21451, |
| "oqe_gptq": 77537, |
| "oqe_gptq_act_order": 76030 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1309050478516034, |
| "selected_objective": 17.34099803119898, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.23.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.23.mlp.up_proj": { |
| "base_objective": 18.652442552149296, |
| "choices": { |
| "lloyd": 381744, |
| "lloyd_gptq": 434273, |
| "lloyd_gptq_act_order": 403677, |
| "oqe": 21029, |
| "oqe_gptq": 77005, |
| "oqe_gptq_act_order": 74912 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13753608069855605, |
| "selected_objective": 16.08705870807171, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.23.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.24.mlp.gate_proj": { |
| "base_objective": 22.211520396173, |
| "choices": { |
| "lloyd": 379235, |
| "lloyd_gptq": 433632, |
| "lloyd_gptq_act_order": 404096, |
| "oqe": 21649, |
| "oqe_gptq": 77587, |
| "oqe_gptq_act_order": 76441 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1374424768300364, |
| "selected_objective": 19.15871401876211, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.24.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.24.mlp.up_proj": { |
| "base_objective": 20.787153027951717, |
| "choices": { |
| "lloyd": 378752, |
| "lloyd_gptq": 432648, |
| "lloyd_gptq_act_order": 406889, |
| "oqe": 21107, |
| "oqe_gptq": 77025, |
| "oqe_gptq_act_order": 76219 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1463075092420345, |
| "selected_objective": 17.745836444199085, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.24.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.25.mlp.gate_proj": { |
| "base_objective": 21.91891623288393, |
| "choices": { |
| "lloyd": 359005, |
| "lloyd_gptq": 437238, |
| "lloyd_gptq_act_order": 417437, |
| "oqe": 21052, |
| "oqe_gptq": 78468, |
| "oqe_gptq_act_order": 79440 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14312815782125077, |
| "selected_objective": 18.781702131032944, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.25.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.25.mlp.up_proj": { |
| "base_objective": 21.58106104284525, |
| "choices": { |
| "lloyd": 358762, |
| "lloyd_gptq": 436851, |
| "lloyd_gptq_act_order": 419302, |
| "oqe": 20763, |
| "oqe_gptq": 78055, |
| "oqe_gptq_act_order": 78907 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15254526277829472, |
| "selected_objective": 18.288972415030003, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.25.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.26.mlp.gate_proj": { |
| "base_objective": 21.256208345294, |
| "choices": { |
| "lloyd": 331483, |
| "lloyd_gptq": 440845, |
| "lloyd_gptq_act_order": 442824, |
| "oqe": 19114, |
| "oqe_gptq": 76276, |
| "oqe_gptq_act_order": 82098 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16287326833489868, |
| "selected_objective": 17.794140219688416, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.26.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.26.mlp.up_proj": { |
| "base_objective": 22.052930414676666, |
| "choices": { |
| "lloyd": 331248, |
| "lloyd_gptq": 439833, |
| "lloyd_gptq_act_order": 444635, |
| "oqe": 19303, |
| "oqe_gptq": 76180, |
| "oqe_gptq_act_order": 81441 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1740963823012518, |
| "selected_objective": 18.213595010340214, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.26.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.27.mlp.down_proj": { |
| "base_objective": 1.1910142879933119, |
| "choices": { |
| "lloyd": 613797, |
| "lloyd_gptq": 352077, |
| "lloyd_gptq_act_order": 326682, |
| "oqe": 24645, |
| "oqe_gptq": 40206, |
| "oqe_gptq_act_order": 35233 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1224658135094161, |
| "selected_objective": 1.0451557543128729, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.27.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.27.mlp.gate_proj": { |
| "base_objective": 22.219318941235542, |
| "choices": { |
| "lloyd": 354990, |
| "lloyd_gptq": 438149, |
| "lloyd_gptq_act_order": 426810, |
| "oqe": 19733, |
| "oqe_gptq": 75382, |
| "oqe_gptq_act_order": 77576 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15553862125052142, |
| "selected_objective": 18.76335670799017, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.27.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.27.mlp.up_proj": { |
| "base_objective": 23.88298212736845, |
| "choices": { |
| "lloyd": 354644, |
| "lloyd_gptq": 437299, |
| "lloyd_gptq_act_order": 428611, |
| "oqe": 19284, |
| "oqe_gptq": 74697, |
| "oqe_gptq_act_order": 78105 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16969182193416998, |
| "selected_objective": 19.83023537695408, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.27.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.28.mlp.down_proj": { |
| "base_objective": 1.230525995604694, |
| "choices": { |
| "lloyd": 629098, |
| "lloyd_gptq": 349765, |
| "lloyd_gptq_act_order": 321149, |
| "oqe": 21225, |
| "oqe_gptq": 38295, |
| "oqe_gptq_act_order": 33108 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.11325940653184763, |
| "selected_objective": 1.0911573516204953, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.28.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.28.mlp.gate_proj": { |
| "base_objective": 21.818252608180046, |
| "choices": { |
| "lloyd": 352735, |
| "lloyd_gptq": 439206, |
| "lloyd_gptq_act_order": 432549, |
| "oqe": 18521, |
| "oqe_gptq": 73588, |
| "oqe_gptq_act_order": 76041 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1546844076152215, |
| "selected_objective": 18.443309128284454, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.28.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.28.mlp.up_proj": { |
| "base_objective": 23.920535132288933, |
| "choices": { |
| "lloyd": 352060, |
| "lloyd_gptq": 438887, |
| "lloyd_gptq_act_order": 434413, |
| "oqe": 18241, |
| "oqe_gptq": 72805, |
| "oqe_gptq_act_order": 76234 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16939516958605683, |
| "selected_objective": 19.868512026965618, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.28.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.29.mlp.down_proj": { |
| "base_objective": 1.2084681987762451, |
| "choices": { |
| "lloyd": 673921, |
| "lloyd_gptq": 337885, |
| "lloyd_gptq_act_order": 295405, |
| "oqe": 19199, |
| "oqe_gptq": 36296, |
| "oqe_gptq_act_order": 29934 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.10327145600494247, |
| "selected_objective": 1.083667928352952, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.29.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.29.mlp.gate_proj": { |
| "base_objective": 21.86723231524229, |
| "choices": { |
| "lloyd": 369246, |
| "lloyd_gptq": 434225, |
| "lloyd_gptq_act_order": 429472, |
| "oqe": 17821, |
| "oqe_gptq": 69077, |
| "oqe_gptq_act_order": 72799 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1502764232969791, |
| "selected_objective": 18.58110285550356, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.29.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.29.mlp.up_proj": { |
| "base_objective": 24.105517700314522, |
| "choices": { |
| "lloyd": 369151, |
| "lloyd_gptq": 434682, |
| "lloyd_gptq_act_order": 430792, |
| "oqe": 17685, |
| "oqe_gptq": 68657, |
| "oqe_gptq_act_order": 71673 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16485610937662254, |
| "selected_objective": 20.13157583773136, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.29.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.3.mlp.gate_proj": { |
| "base_objective": 1.3963809171691537, |
| "choices": { |
| "lloyd": 242745, |
| "lloyd_gptq": 438509, |
| "lloyd_gptq_act_order": 518021, |
| "oqe": 15102, |
| "oqe_gptq": 79217, |
| "oqe_gptq_act_order": 99046 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14377922811915933, |
| "selected_objective": 1.195610346738249, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.3.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.3.mlp.up_proj": { |
| "base_objective": 1.3457481269724667, |
| "choices": { |
| "lloyd": 242972, |
| "lloyd_gptq": 437906, |
| "lloyd_gptq_act_order": 519766, |
| "oqe": 14863, |
| "oqe_gptq": 78765, |
| "oqe_gptq_act_order": 98368 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14446164218449897, |
| "selected_objective": 1.1513391425833106, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.3.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.30.mlp.down_proj": { |
| "base_objective": 1.27770890109241, |
| "choices": { |
| "lloyd": 652854, |
| "lloyd_gptq": 342753, |
| "lloyd_gptq_act_order": 312396, |
| "oqe": 17614, |
| "oqe_gptq": 36022, |
| "oqe_gptq_act_order": 31001 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.10643247593265881, |
| "selected_objective": 1.1417191792279482, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.30.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.30.mlp.gate_proj": { |
| "base_objective": 21.67806761711836, |
| "choices": { |
| "lloyd": 359368, |
| "lloyd_gptq": 435064, |
| "lloyd_gptq_act_order": 442584, |
| "oqe": 16604, |
| "oqe_gptq": 67246, |
| "oqe_gptq_act_order": 71774 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16078853234654922, |
| "selected_objective": 18.192482940852642, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.30.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.30.mlp.up_proj": { |
| "base_objective": 24.552422113716602, |
| "choices": { |
| "lloyd": 357898, |
| "lloyd_gptq": 435638, |
| "lloyd_gptq_act_order": 443751, |
| "oqe": 16541, |
| "oqe_gptq": 66919, |
| "oqe_gptq_act_order": 71893 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17675321061860882, |
| "selected_objective": 20.212702676653862, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.30.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.31.mlp.down_proj": { |
| "base_objective": 1.3800626397132874, |
| "choices": { |
| "lloyd": 708088, |
| "lloyd_gptq": 323862, |
| "lloyd_gptq_act_order": 282701, |
| "oqe": 17377, |
| "oqe_gptq": 33337, |
| "oqe_gptq_act_order": 27275 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.10280925454213208, |
| "selected_objective": 1.2381794285029173, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.31.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.31.mlp.gate_proj": { |
| "base_objective": 22.694474689662457, |
| "choices": { |
| "lloyd": 388188, |
| "lloyd_gptq": 429605, |
| "lloyd_gptq_act_order": 424349, |
| "oqe": 17462, |
| "oqe_gptq": 65257, |
| "oqe_gptq_act_order": 67779 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15547891162082217, |
| "selected_objective": 19.16596246510744, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.31.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.31.mlp.up_proj": { |
| "base_objective": 26.26206049323082, |
| "choices": { |
| "lloyd": 387471, |
| "lloyd_gptq": 429256, |
| "lloyd_gptq_act_order": 425916, |
| "oqe": 17091, |
| "oqe_gptq": 64961, |
| "oqe_gptq_act_order": 67945 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16849196572058123, |
| "selected_objective": 21.837114296853542, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.31.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.32.mlp.down_proj": { |
| "base_objective": 1.4781511276960373, |
| "choices": { |
| "lloyd": 680041, |
| "lloyd_gptq": 334385, |
| "lloyd_gptq_act_order": 297076, |
| "oqe": 16733, |
| "oqe_gptq": 35272, |
| "oqe_gptq_act_order": 29133 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.10172237379596083, |
| "selected_objective": 1.32779008615762, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.32.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.32.mlp.gate_proj": { |
| "base_objective": 24.034780979156494, |
| "choices": { |
| "lloyd": 392560, |
| "lloyd_gptq": 428311, |
| "lloyd_gptq_act_order": 424138, |
| "oqe": 17145, |
| "oqe_gptq": 63924, |
| "oqe_gptq_act_order": 66562 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15584195059600625, |
| "selected_objective": 20.289153829216957, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.32.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.32.mlp.up_proj": { |
| "base_objective": 28.094232492148876, |
| "choices": { |
| "lloyd": 391159, |
| "lloyd_gptq": 428660, |
| "lloyd_gptq_act_order": 425658, |
| "oqe": 17007, |
| "oqe_gptq": 63615, |
| "oqe_gptq_act_order": 66541 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.169225372683111, |
| "selected_objective": 23.339975528419018, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.32.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.33.mlp.down_proj": { |
| "base_objective": 1.555372977629304, |
| "choices": { |
| "lloyd": 627898, |
| "lloyd_gptq": 353367, |
| "lloyd_gptq_act_order": 326822, |
| "oqe": 17353, |
| "oqe_gptq": 35907, |
| "oqe_gptq_act_order": 31293 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.10924738680470514, |
| "selected_objective": 1.3854525443166494, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.33.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.33.mlp.gate_proj": { |
| "base_objective": 25.025221325457096, |
| "choices": { |
| "lloyd": 389161, |
| "lloyd_gptq": 430372, |
| "lloyd_gptq_act_order": 425582, |
| "oqe": 17095, |
| "oqe_gptq": 63571, |
| "oqe_gptq_act_order": 66859 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15675674589228616, |
| "selected_objective": 21.102349065244198, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.33.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.33.mlp.up_proj": { |
| "base_objective": 29.569145411252975, |
| "choices": { |
| "lloyd": 386976, |
| "lloyd_gptq": 430615, |
| "lloyd_gptq_act_order": 427439, |
| "oqe": 16755, |
| "oqe_gptq": 63559, |
| "oqe_gptq_act_order": 67296 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16861615469690328, |
| "selected_objective": 24.583309814333916, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.33.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.34.mlp.down_proj": { |
| "base_objective": 2.135206799954176, |
| "choices": { |
| "lloyd": 520081, |
| "lloyd_gptq": 379880, |
| "lloyd_gptq_act_order": 404206, |
| "oqe": 15578, |
| "oqe_gptq": 35685, |
| "oqe_gptq_act_order": 37210 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12737587452773713, |
| "selected_objective": 1.8632329665124416, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.34.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.34.mlp.gate_proj": { |
| "base_objective": 27.784273855388165, |
| "choices": { |
| "lloyd": 335827, |
| "lloyd_gptq": 429010, |
| "lloyd_gptq_act_order": 476832, |
| "oqe": 15445, |
| "oqe_gptq": 61877, |
| "oqe_gptq_act_order": 73649 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17389840925802147, |
| "selected_objective": 22.95263282954693, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.34.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.34.mlp.up_proj": { |
| "base_objective": 32.971353612840176, |
| "choices": { |
| "lloyd": 334942, |
| "lloyd_gptq": 430144, |
| "lloyd_gptq_act_order": 477986, |
| "oqe": 15149, |
| "oqe_gptq": 61389, |
| "oqe_gptq_act_order": 73030 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18702454749664488, |
| "selected_objective": 26.804901123046875, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.34.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.35.mlp.down_proj": { |
| "base_objective": 2.329005878418684, |
| "choices": { |
| "lloyd": 507673, |
| "lloyd_gptq": 385788, |
| "lloyd_gptq_act_order": 392466, |
| "oqe": 23882, |
| "oqe_gptq": 41527, |
| "oqe_gptq_act_order": 41304 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14086547888321965, |
| "selected_objective": 2.0009293500334024, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.35.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.35.mlp.gate_proj": { |
| "base_objective": 30.473417460918427, |
| "choices": { |
| "lloyd": 351188, |
| "lloyd_gptq": 428967, |
| "lloyd_gptq_act_order": 449089, |
| "oqe": 18025, |
| "oqe_gptq": 68652, |
| "oqe_gptq_act_order": 76719 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15978252002071425, |
| "selected_objective": 25.604298025369644, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.35.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.35.mlp.up_proj": { |
| "base_objective": 35.13678327202797, |
| "choices": { |
| "lloyd": 349469, |
| "lloyd_gptq": 429367, |
| "lloyd_gptq_act_order": 451534, |
| "oqe": 17357, |
| "oqe_gptq": 68260, |
| "oqe_gptq_act_order": 76653 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17206275370314295, |
| "selected_objective": 29.09105158597231, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.35.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.36.mlp.down_proj": { |
| "base_objective": 2.0046358834952116, |
| "choices": { |
| "lloyd": 513673, |
| "lloyd_gptq": 381797, |
| "lloyd_gptq_act_order": 391680, |
| "oqe": 24495, |
| "oqe_gptq": 40722, |
| "oqe_gptq_act_order": 40273 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14153800268631067, |
| "selected_objective": 1.7209037244319916, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.36.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.36.mlp.gate_proj": { |
| "base_objective": 33.707527972757816, |
| "choices": { |
| "lloyd": 324352, |
| "lloyd_gptq": 435929, |
| "lloyd_gptq_act_order": 454522, |
| "oqe": 18610, |
| "oqe_gptq": 75388, |
| "oqe_gptq_act_order": 83839 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15106647440377696, |
| "selected_objective": 28.6154505610466, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.36.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.36.mlp.up_proj": { |
| "base_objective": 34.296021580696106, |
| "choices": { |
| "lloyd": 323329, |
| "lloyd_gptq": 435450, |
| "lloyd_gptq_act_order": 456574, |
| "oqe": 18083, |
| "oqe_gptq": 74566, |
| "oqe_gptq_act_order": 84638 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16108249369583427, |
| "selected_objective": 28.771532900631428, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.36.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.37.mlp.down_proj": { |
| "base_objective": 1.842599593102932, |
| "choices": { |
| "lloyd": 569070, |
| "lloyd_gptq": 367362, |
| "lloyd_gptq_act_order": 350047, |
| "oqe": 26841, |
| "oqe_gptq": 41448, |
| "oqe_gptq_act_order": 37872 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12772102382095418, |
| "selected_objective": 1.607260886579752, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.37.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.37.mlp.gate_proj": { |
| "base_objective": 33.86310436576605, |
| "choices": { |
| "lloyd": 324206, |
| "lloyd_gptq": 436990, |
| "lloyd_gptq_act_order": 450356, |
| "oqe": 18812, |
| "oqe_gptq": 77656, |
| "oqe_gptq_act_order": 84620 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14229342801074463, |
| "selected_objective": 29.044607162475586, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.37.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.37.mlp.up_proj": { |
| "base_objective": 33.53317854553461, |
| "choices": { |
| "lloyd": 322311, |
| "lloyd_gptq": 438059, |
| "lloyd_gptq_act_order": 452725, |
| "oqe": 18182, |
| "oqe_gptq": 76463, |
| "oqe_gptq_act_order": 84900 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15070946898898108, |
| "selected_objective": 28.479411013424397, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.37.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.38.mlp.down_proj": { |
| "base_objective": 1.8677259180694818, |
| "choices": { |
| "lloyd": 546165, |
| "lloyd_gptq": 372471, |
| "lloyd_gptq_act_order": 362524, |
| "oqe": 27677, |
| "oqe_gptq": 43263, |
| "oqe_gptq_act_order": 40540 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13442221723287617, |
| "selected_objective": 1.6166620589792728, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.38.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.38.mlp.gate_proj": { |
| "base_objective": 42.240163028240204, |
| "choices": { |
| "lloyd": 339314, |
| "lloyd_gptq": 434966, |
| "lloyd_gptq_act_order": 445686, |
| "oqe": 18427, |
| "oqe_gptq": 73803, |
| "oqe_gptq_act_order": 80444 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1524942902773576, |
| "selected_objective": 35.79877934604883, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.38.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.38.mlp.up_proj": { |
| "base_objective": 36.46470972895622, |
| "choices": { |
| "lloyd": 337182, |
| "lloyd_gptq": 435318, |
| "lloyd_gptq_act_order": 448195, |
| "oqe": 17960, |
| "oqe_gptq": 73195, |
| "oqe_gptq_act_order": 80790 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16386011458045802, |
| "selected_objective": 30.489598214626312, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.38.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.39.mlp.down_proj": { |
| "base_objective": 1.7803108133375645, |
| "choices": { |
| "lloyd": 594876, |
| "lloyd_gptq": 360018, |
| "lloyd_gptq_act_order": 319424, |
| "oqe": 32549, |
| "oqe_gptq": 46056, |
| "oqe_gptq_act_order": 39717 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.11855254654520864, |
| "selected_objective": 1.5692504327744246, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.39.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.39.mlp.gate_proj": { |
| "base_objective": 38.34022521972656, |
| "choices": { |
| "lloyd": 357198, |
| "lloyd_gptq": 429700, |
| "lloyd_gptq_act_order": 432379, |
| "oqe": 19274, |
| "oqe_gptq": 74560, |
| "oqe_gptq_act_order": 79529 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14212475419835613, |
| "selected_objective": 32.89113013446331, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.39.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.39.mlp.up_proj": { |
| "base_objective": 35.02458843588829, |
| "choices": { |
| "lloyd": 355005, |
| "lloyd_gptq": 429920, |
| "lloyd_gptq_act_order": 435694, |
| "oqe": 18737, |
| "oqe_gptq": 73788, |
| "oqe_gptq_act_order": 79496 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15092117665689164, |
| "selected_objective": 29.73863633722067, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.39.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.4.mlp.gate_proj": { |
| "base_objective": 1.7296784687787294, |
| "choices": { |
| "lloyd": 243401, |
| "lloyd_gptq": 442026, |
| "lloyd_gptq_act_order": 513089, |
| "oqe": 15140, |
| "oqe_gptq": 80560, |
| "oqe_gptq_act_order": 98424 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14096403157291426, |
| "selected_objective": 1.4858560184948146, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.4.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.4.mlp.up_proj": { |
| "base_objective": 1.672181366942823, |
| "choices": { |
| "lloyd": 244113, |
| "lloyd_gptq": 442210, |
| "lloyd_gptq_act_order": 512216, |
| "oqe": 15271, |
| "oqe_gptq": 80668, |
| "oqe_gptq_act_order": 98162 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1415271493711594, |
| "selected_objective": 1.4355223048478365, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.4.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.40.mlp.down_proj": { |
| "base_objective": 1.9411130268126726, |
| "choices": { |
| "lloyd": 564654, |
| "lloyd_gptq": 362765, |
| "lloyd_gptq_act_order": 333695, |
| "oqe": 37648, |
| "oqe_gptq": 49778, |
| "oqe_gptq_act_order": 44100 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12914715612703065, |
| "selected_objective": 1.6904237996786833, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.40.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.40.mlp.gate_proj": { |
| "base_objective": 41.67084486782551, |
| "choices": { |
| "lloyd": 377490, |
| "lloyd_gptq": 428459, |
| "lloyd_gptq_act_order": 420460, |
| "oqe": 19263, |
| "oqe_gptq": 71652, |
| "oqe_gptq_act_order": 75316 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1470317540035334, |
| "selected_objective": 35.54390745609999, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.40.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.40.mlp.up_proj": { |
| "base_objective": 37.35861913859844, |
| "choices": { |
| "lloyd": 374287, |
| "lloyd_gptq": 429141, |
| "lloyd_gptq_act_order": 423446, |
| "oqe": 19145, |
| "oqe_gptq": 71386, |
| "oqe_gptq_act_order": 75235 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1553540657913719, |
| "selected_objective": 31.554805763065815, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.40.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.41.mlp.down_proj": { |
| "base_objective": 1.8380074631422758, |
| "choices": { |
| "lloyd": 602974, |
| "lloyd_gptq": 356761, |
| "lloyd_gptq_act_order": 331374, |
| "oqe": 25846, |
| "oqe_gptq": 40257, |
| "oqe_gptq_act_order": 35428 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12291819961630213, |
| "selected_objective": 1.6120828948915005, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.41.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.41.mlp.gate_proj": { |
| "base_objective": 36.13913205265999, |
| "choices": { |
| "lloyd": 375931, |
| "lloyd_gptq": 431569, |
| "lloyd_gptq_act_order": 422926, |
| "oqe": 18510, |
| "oqe_gptq": 70196, |
| "oqe_gptq_act_order": 73508 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14333051354315163, |
| "selected_objective": 30.959291696548462, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.41.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.41.mlp.up_proj": { |
| "base_objective": 35.98599983751774, |
| "choices": { |
| "lloyd": 374450, |
| "lloyd_gptq": 430522, |
| "lloyd_gptq_act_order": 425914, |
| "oqe": 18170, |
| "oqe_gptq": 69620, |
| "oqe_gptq_act_order": 73964 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1526605824051176, |
| "selected_objective": 30.49235614389181, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.41.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.42.mlp.down_proj": { |
| "base_objective": 2.096575539559126, |
| "choices": { |
| "lloyd": 577515, |
| "lloyd_gptq": 364542, |
| "lloyd_gptq_act_order": 364633, |
| "oqe": 17114, |
| "oqe_gptq": 35279, |
| "oqe_gptq_act_order": 33557 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12723661662430175, |
| "selected_objective": 1.8298143614083529, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.42.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.42.mlp.gate_proj": { |
| "base_objective": 34.16330433636904, |
| "choices": { |
| "lloyd": 346605, |
| "lloyd_gptq": 435670, |
| "lloyd_gptq_act_order": 452167, |
| "oqe": 16800, |
| "oqe_gptq": 67159, |
| "oqe_gptq_act_order": 74239 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15648892172924989, |
| "selected_objective": 28.81712567806244, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.42.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.42.mlp.up_proj": { |
| "base_objective": 36.51925007998943, |
| "choices": { |
| "lloyd": 346531, |
| "lloyd_gptq": 436041, |
| "lloyd_gptq_act_order": 451821, |
| "oqe": 16361, |
| "oqe_gptq": 67158, |
| "oqe_gptq_act_order": 74728 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1668188957085185, |
| "selected_objective": 30.42714910954237, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.42.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.43.mlp.down_proj": { |
| "base_objective": 2.2589961793273687, |
| "choices": { |
| "lloyd": 608718, |
| "lloyd_gptq": 358623, |
| "lloyd_gptq_act_order": 338414, |
| "oqe": 18387, |
| "oqe_gptq": 35987, |
| "oqe_gptq_act_order": 32511 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.11490291967350486, |
| "selected_objective": 1.9994309227913618, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.43.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.43.mlp.gate_proj": { |
| "base_objective": 34.270878829061985, |
| "choices": { |
| "lloyd": 359705, |
| "lloyd_gptq": 434706, |
| "lloyd_gptq_act_order": 439186, |
| "oqe": 17383, |
| "oqe_gptq": 68159, |
| "oqe_gptq_act_order": 73501 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15123568396535725, |
| "selected_objective": 29.087899029254913, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.43.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.43.mlp.up_proj": { |
| "base_objective": 38.18004992604256, |
| "choices": { |
| "lloyd": 359009, |
| "lloyd_gptq": 433557, |
| "lloyd_gptq_act_order": 440863, |
| "oqe": 16988, |
| "oqe_gptq": 68069, |
| "oqe_gptq_act_order": 74154 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16112083711345082, |
| "selected_objective": 32.028448320925236, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.43.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.44.mlp.down_proj": { |
| "base_objective": 2.554251851513982, |
| "choices": { |
| "lloyd": 599746, |
| "lloyd_gptq": 363819, |
| "lloyd_gptq_act_order": 347889, |
| "oqe": 16315, |
| "oqe_gptq": 34034, |
| "oqe_gptq_act_order": 30837 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.11871484431199097, |
| "selected_objective": 2.251024240627885, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.44.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.44.mlp.gate_proj": { |
| "base_objective": 35.07694565504789, |
| "choices": { |
| "lloyd": 352418, |
| "lloyd_gptq": 434780, |
| "lloyd_gptq_act_order": 448141, |
| "oqe": 16843, |
| "oqe_gptq": 66945, |
| "oqe_gptq_act_order": 73513 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1530101949931264, |
| "selected_objective": 29.709815360605717, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.44.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.44.mlp.up_proj": { |
| "base_objective": 39.52928169071674, |
| "choices": { |
| "lloyd": 350467, |
| "lloyd_gptq": 434300, |
| "lloyd_gptq_act_order": 451161, |
| "oqe": 16380, |
| "oqe_gptq": 66325, |
| "oqe_gptq_act_order": 74007 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1622033430566101, |
| "selected_objective": 33.11750005185604, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.44.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.45.mlp.down_proj": { |
| "base_objective": 2.6509535014629364, |
| "choices": { |
| "lloyd": 620095, |
| "lloyd_gptq": 359980, |
| "lloyd_gptq_act_order": 333311, |
| "oqe": 15839, |
| "oqe_gptq": 33593, |
| "oqe_gptq_act_order": 29822 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.11463350302517994, |
| "selected_objective": 2.3470654152333736, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.45.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.45.mlp.gate_proj": { |
| "base_objective": 35.404625222086906, |
| "choices": { |
| "lloyd": 350091, |
| "lloyd_gptq": 434837, |
| "lloyd_gptq_act_order": 450237, |
| "oqe": 16471, |
| "oqe_gptq": 67193, |
| "oqe_gptq_act_order": 73811 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15102366364421527, |
| "selected_objective": 30.057689011096954, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.45.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.45.mlp.up_proj": { |
| "base_objective": 40.0932025462389, |
| "choices": { |
| "lloyd": 348361, |
| "lloyd_gptq": 434745, |
| "lloyd_gptq_act_order": 452788, |
| "oqe": 15890, |
| "oqe_gptq": 66802, |
| "oqe_gptq_act_order": 74054 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1606201623900816, |
| "selected_objective": 33.653425842523575, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.45.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.46.mlp.down_proj": { |
| "base_objective": 3.013966754078865, |
| "choices": { |
| "lloyd": 576657, |
| "lloyd_gptq": 368583, |
| "lloyd_gptq_act_order": 370099, |
| "oqe": 13961, |
| "oqe_gptq": 32033, |
| "oqe_gptq_act_order": 31307 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12459623909132367, |
| "selected_objective": 2.638437831774354, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.46.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.46.mlp.gate_proj": { |
| "base_objective": 35.475910902023315, |
| "choices": { |
| "lloyd": 325269, |
| "lloyd_gptq": 437876, |
| "lloyd_gptq_act_order": 472842, |
| "oqe": 15372, |
| "oqe_gptq": 66013, |
| "oqe_gptq_act_order": 75268 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15988931428645736, |
| "selected_objective": 29.80369183421135, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.46.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.46.mlp.up_proj": { |
| "base_objective": 41.10296951234341, |
| "choices": { |
| "lloyd": 324332, |
| "lloyd_gptq": 438860, |
| "lloyd_gptq_act_order": 473730, |
| "oqe": 15181, |
| "oqe_gptq": 64832, |
| "oqe_gptq_act_order": 75705 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16973105130776323, |
| "selected_objective": 34.12651928514242, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.46.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.47.mlp.down_proj": { |
| "base_objective": 3.729852069169283, |
| "choices": { |
| "lloyd": 566597, |
| "lloyd_gptq": 369281, |
| "lloyd_gptq_act_order": 380141, |
| "oqe": 14847, |
| "oqe_gptq": 31225, |
| "oqe_gptq_act_order": 30549 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12691591126310597, |
| "selected_objective": 3.256474494934082, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.47.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.47.mlp.gate_proj": { |
| "base_objective": 39.14089739322662, |
| "choices": { |
| "lloyd": 351160, |
| "lloyd_gptq": 433539, |
| "lloyd_gptq_act_order": 455719, |
| "oqe": 15709, |
| "oqe_gptq": 64315, |
| "oqe_gptq_act_order": 72198 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15696031024709334, |
| "selected_objective": 32.997329995036125, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.47.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.47.mlp.up_proj": { |
| "base_objective": 45.30843698978424, |
| "choices": { |
| "lloyd": 349551, |
| "lloyd_gptq": 432553, |
| "lloyd_gptq_act_order": 459338, |
| "oqe": 15756, |
| "oqe_gptq": 63093, |
| "oqe_gptq_act_order": 72349 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16592031495217818, |
| "selected_objective": 37.79084685444832, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.47.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.48.mlp.down_proj": { |
| "base_objective": 4.594788264483213, |
| "choices": { |
| "lloyd": 499244, |
| "lloyd_gptq": 384241, |
| "lloyd_gptq_act_order": 430977, |
| "oqe": 12776, |
| "oqe_gptq": 31417, |
| "oqe_gptq_act_order": 33985 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13075363639768334, |
| "selected_objective": 3.994002990424633, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.48.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.48.mlp.gate_proj": { |
| "base_objective": 43.235496282577515, |
| "choices": { |
| "lloyd": 311651, |
| "lloyd_gptq": 436274, |
| "lloyd_gptq_act_order": 488833, |
| "oqe": 14730, |
| "oqe_gptq": 64395, |
| "oqe_gptq_act_order": 76757 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16084609021454754, |
| "selected_objective": 36.28123574703932, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.48.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.48.mlp.up_proj": { |
| "base_objective": 50.1230186522007, |
| "choices": { |
| "lloyd": 310559, |
| "lloyd_gptq": 437277, |
| "lloyd_gptq_act_order": 489236, |
| "oqe": 14480, |
| "oqe_gptq": 63979, |
| "oqe_gptq_act_order": 77109 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1690722007865174, |
| "selected_objective": 41.64860957860947, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.48.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.49.mlp.down_proj": { |
| "base_objective": 5.9500672817230225, |
| "choices": { |
| "lloyd": 449780, |
| "lloyd_gptq": 390904, |
| "lloyd_gptq_act_order": 474625, |
| "oqe": 12428, |
| "oqe_gptq": 29619, |
| "oqe_gptq_act_order": 35284 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.14279070652182846, |
| "selected_objective": 5.100452970713377, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.49.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.49.mlp.gate_proj": { |
| "base_objective": 46.954423651099205, |
| "choices": { |
| "lloyd": 296579, |
| "lloyd_gptq": 434572, |
| "lloyd_gptq_act_order": 505986, |
| "oqe": 14253, |
| "oqe_gptq": 62964, |
| "oqe_gptq_act_order": 78286 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1631219428341909, |
| "selected_objective": 39.29512684047222, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.49.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.49.mlp.up_proj": { |
| "base_objective": 54.474848076701164, |
| "choices": { |
| "lloyd": 294612, |
| "lloyd_gptq": 434088, |
| "lloyd_gptq_act_order": 509082, |
| "oqe": 13473, |
| "oqe_gptq": 62516, |
| "oqe_gptq_act_order": 78869 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17016972652659768, |
| "selected_objective": 45.20487807691097, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.49.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.5.mlp.gate_proj": { |
| "base_objective": 2.1441610823385417, |
| "choices": { |
| "lloyd": 263886, |
| "lloyd_gptq": 437747, |
| "lloyd_gptq_act_order": 506225, |
| "oqe": 15063, |
| "oqe_gptq": 76538, |
| "oqe_gptq_act_order": 93181 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13555316589408695, |
| "selected_objective": 1.8535132594406605, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.5.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.5.mlp.up_proj": { |
| "base_objective": 2.023600204847753, |
| "choices": { |
| "lloyd": 264713, |
| "lloyd_gptq": 438329, |
| "lloyd_gptq_act_order": 505379, |
| "oqe": 14731, |
| "oqe_gptq": 76791, |
| "oqe_gptq_act_order": 92697 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13610618931004848, |
| "selected_objective": 1.7481756922788918, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.5.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.50.mlp.down_proj": { |
| "base_objective": 10.223594278097153, |
| "choices": { |
| "lloyd": 317289, |
| "lloyd_gptq": 388393, |
| "lloyd_gptq_act_order": 603393, |
| "oqe": 9753, |
| "oqe_gptq": 28713, |
| "oqe_gptq_act_order": 45099 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1585315350715355, |
| "selected_objective": 8.602832183241844, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.50.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.50.mlp.gate_proj": { |
| "base_objective": 57.470287680625916, |
| "choices": { |
| "lloyd": 239137, |
| "lloyd_gptq": 430999, |
| "lloyd_gptq_act_order": 562156, |
| "oqe": 12142, |
| "oqe_gptq": 61519, |
| "oqe_gptq_act_order": 86687 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18065236329677814, |
| "selected_objective": 47.08814439177513, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.50.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.50.mlp.up_proj": { |
| "base_objective": 64.5475409924984, |
| "choices": { |
| "lloyd": 237359, |
| "lloyd_gptq": 432574, |
| "lloyd_gptq_act_order": 562636, |
| "oqe": 11941, |
| "oqe_gptq": 61431, |
| "oqe_gptq_act_order": 86699 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1876215248974935, |
| "selected_objective": 52.43703292310238, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.50.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.51.mlp.down_proj": { |
| "base_objective": 12.75507465004921, |
| "choices": { |
| "lloyd": 295831, |
| "lloyd_gptq": 389741, |
| "lloyd_gptq_act_order": 620322, |
| "oqe": 11311, |
| "oqe_gptq": 28980, |
| "oqe_gptq_act_order": 46455 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17503146486482712, |
| "selected_objective": 10.522535249590874, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.51.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.51.mlp.gate_proj": { |
| "base_objective": 65.7016863077879, |
| "choices": { |
| "lloyd": 272854, |
| "lloyd_gptq": 427445, |
| "lloyd_gptq_act_order": 532168, |
| "oqe": 13988, |
| "oqe_gptq": 62829, |
| "oqe_gptq_act_order": 83356 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16352966563187077, |
| "selected_objective": 54.95751151442528, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.51.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.51.mlp.up_proj": { |
| "base_objective": 75.4831803292036, |
| "choices": { |
| "lloyd": 272337, |
| "lloyd_gptq": 426689, |
| "lloyd_gptq_act_order": 534346, |
| "oqe": 13450, |
| "oqe_gptq": 62100, |
| "oqe_gptq_act_order": 83718 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16866211567383027, |
| "selected_objective": 62.752027437090874, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.51.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.52.mlp.down_proj": { |
| "base_objective": 12.249535948038101, |
| "choices": { |
| "lloyd": 296094, |
| "lloyd_gptq": 390663, |
| "lloyd_gptq_act_order": 611897, |
| "oqe": 13257, |
| "oqe_gptq": 31474, |
| "oqe_gptq_act_order": 49255 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18418409430380356, |
| "selected_objective": 9.99336626380682, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.52.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.52.mlp.gate_proj": { |
| "base_objective": 81.82565066218376, |
| "choices": { |
| "lloyd": 233871, |
| "lloyd_gptq": 436389, |
| "lloyd_gptq_act_order": 544083, |
| "oqe": 13895, |
| "oqe_gptq": 70307, |
| "oqe_gptq_act_order": 94095 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16614068939122184, |
| "selected_objective": 68.23108065128326, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.52.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.52.mlp.up_proj": { |
| "base_objective": 82.10960099101067, |
| "choices": { |
| "lloyd": 232337, |
| "lloyd_gptq": 435801, |
| "lloyd_gptq_act_order": 546689, |
| "oqe": 13188, |
| "oqe_gptq": 70241, |
| "oqe_gptq_act_order": 94384 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17089702671092633, |
| "selected_objective": 68.07731431722641, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.52.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.53.mlp.down_proj": { |
| "base_objective": 13.190371006727219, |
| "choices": { |
| "lloyd": 311080, |
| "lloyd_gptq": 386596, |
| "lloyd_gptq_act_order": 594578, |
| "oqe": 15057, |
| "oqe_gptq": 33899, |
| "oqe_gptq_act_order": 51430 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17457245438608904, |
| "selected_objective": 10.88769556581974, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.53.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.53.mlp.gate_proj": { |
| "base_objective": 90.54390305280685, |
| "choices": { |
| "lloyd": 230528, |
| "lloyd_gptq": 438978, |
| "lloyd_gptq_act_order": 539797, |
| "oqe": 13978, |
| "oqe_gptq": 73263, |
| "oqe_gptq_act_order": 96096 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16376960001725893, |
| "selected_objective": 75.7155642658472, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.53.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.53.mlp.up_proj": { |
| "base_objective": 85.9647676050663, |
| "choices": { |
| "lloyd": 230535, |
| "lloyd_gptq": 438609, |
| "lloyd_gptq_act_order": 541100, |
| "oqe": 13733, |
| "oqe_gptq": 72121, |
| "oqe_gptq_act_order": 96542 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16762560917819913, |
| "selected_objective": 71.55487106740475, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.53.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.54.mlp.down_proj": { |
| "base_objective": 17.27578128874302, |
| "choices": { |
| "lloyd": 224105, |
| "lloyd_gptq": 373708, |
| "lloyd_gptq_act_order": 689029, |
| "oqe": 12611, |
| "oqe_gptq": 32690, |
| "oqe_gptq_act_order": 60497 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.22303261226225302, |
| "selected_objective": 13.422718659043312, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.54.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.54.mlp.gate_proj": { |
| "base_objective": 110.97225946187973, |
| "choices": { |
| "lloyd": 233592, |
| "lloyd_gptq": 439469, |
| "lloyd_gptq_act_order": 537362, |
| "oqe": 14141, |
| "oqe_gptq": 72906, |
| "oqe_gptq_act_order": 95170 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1660973143389468, |
| "selected_objective": 92.54006519913673, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.54.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.54.mlp.up_proj": { |
| "base_objective": 99.54399907588959, |
| "choices": { |
| "lloyd": 232836, |
| "lloyd_gptq": 438453, |
| "lloyd_gptq_act_order": 539576, |
| "oqe": 13994, |
| "oqe_gptq": 72396, |
| "oqe_gptq_act_order": 95385 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17096574172712475, |
| "selected_objective": 82.5253854393959, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.54.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.55.mlp.down_proj": { |
| "base_objective": 13.30400662124157, |
| "choices": { |
| "lloyd": 297238, |
| "lloyd_gptq": 402468, |
| "lloyd_gptq_act_order": 584452, |
| "oqe": 17950, |
| "oqe_gptq": 36992, |
| "oqe_gptq_act_order": 53540 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1977003733949796, |
| "selected_objective": 10.67379954457283, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.55.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.55.mlp.gate_proj": { |
| "base_objective": 104.49821153283119, |
| "choices": { |
| "lloyd": 264801, |
| "lloyd_gptq": 430970, |
| "lloyd_gptq_act_order": 519713, |
| "oqe": 15598, |
| "oqe_gptq": 70565, |
| "oqe_gptq_act_order": 90993 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15734810053927517, |
| "selected_objective": 88.05561643838882, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.55.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.55.mlp.up_proj": { |
| "base_objective": 97.40239009261131, |
| "choices": { |
| "lloyd": 262637, |
| "lloyd_gptq": 429701, |
| "lloyd_gptq_act_order": 524171, |
| "oqe": 14933, |
| "oqe_gptq": 69731, |
| "oqe_gptq_act_order": 91467 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.161711467129805, |
| "selected_objective": 81.65130668878555, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.55.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.56.mlp.down_proj": { |
| "base_objective": 13.06900504231453, |
| "choices": { |
| "lloyd": 315221, |
| "lloyd_gptq": 410370, |
| "lloyd_gptq_act_order": 559025, |
| "oqe": 19366, |
| "oqe_gptq": 37887, |
| "oqe_gptq_act_order": 50771 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.21046844776851759, |
| "selected_objective": 10.31839183717966, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.56.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.56.mlp.gate_proj": { |
| "base_objective": 114.62913510203362, |
| "choices": { |
| "lloyd": 250525, |
| "lloyd_gptq": 432258, |
| "lloyd_gptq_act_order": 523172, |
| "oqe": 16039, |
| "oqe_gptq": 74539, |
| "oqe_gptq_act_order": 96107 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16291670272617587, |
| "selected_objective": 95.95413437485695, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.56.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.56.mlp.up_proj": { |
| "base_objective": 103.53454461693764, |
| "choices": { |
| "lloyd": 247857, |
| "lloyd_gptq": 433624, |
| "lloyd_gptq_act_order": 527384, |
| "oqe": 15232, |
| "oqe_gptq": 73303, |
| "oqe_gptq_act_order": 95240 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16691963433034662, |
| "selected_objective": 86.25259628891945, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.56.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.57.mlp.down_proj": { |
| "base_objective": 12.922234252095222, |
| "choices": { |
| "lloyd": 359042, |
| "lloyd_gptq": 399131, |
| "lloyd_gptq_act_order": 525660, |
| "oqe": 21654, |
| "oqe_gptq": 38275, |
| "oqe_gptq_act_order": 48878 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.19755656613547923, |
| "selected_objective": 10.369362026453018, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.57.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.57.mlp.gate_proj": { |
| "base_objective": 130.72459211945534, |
| "choices": { |
| "lloyd": 259871, |
| "lloyd_gptq": 434427, |
| "lloyd_gptq_act_order": 510134, |
| "oqe": 16502, |
| "oqe_gptq": 76302, |
| "oqe_gptq_act_order": 95404 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1607391126863139, |
| "selected_objective": 109.71203717589378, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.57.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.57.mlp.up_proj": { |
| "base_objective": 116.06509140133858, |
| "choices": { |
| "lloyd": 257589, |
| "lloyd_gptq": 435525, |
| "lloyd_gptq_act_order": 514014, |
| "oqe": 16188, |
| "oqe_gptq": 74704, |
| "oqe_gptq_act_order": 94620 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16613053337032768, |
| "selected_objective": 96.78313586115837, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.57.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.58.mlp.down_proj": { |
| "base_objective": 16.639235630631447, |
| "choices": { |
| "lloyd": 332336, |
| "lloyd_gptq": 397895, |
| "lloyd_gptq_act_order": 561106, |
| "oqe": 18472, |
| "oqe_gptq": 34597, |
| "oqe_gptq_act_order": 48234 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.25495150250281917, |
| "selected_objective": 12.397037506103516, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.58.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.58.mlp.gate_proj": { |
| "base_objective": 148.90509143471718, |
| "choices": { |
| "lloyd": 270745, |
| "lloyd_gptq": 434846, |
| "lloyd_gptq_act_order": 501431, |
| "oqe": 16706, |
| "oqe_gptq": 75908, |
| "oqe_gptq_act_order": 93004 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.15912056025513666, |
| "selected_objective": 125.21122986078262, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.58.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.58.mlp.up_proj": { |
| "base_objective": 132.17830076813698, |
| "choices": { |
| "lloyd": 269267, |
| "lloyd_gptq": 434154, |
| "lloyd_gptq_act_order": 505864, |
| "oqe": 16456, |
| "oqe_gptq": 74582, |
| "oqe_gptq_act_order": 92317 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16475080135102127, |
| "selected_objective": 110.4018197953701, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.58.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.59.mlp.down_proj": { |
| "base_objective": 19.23992644250393, |
| "choices": { |
| "lloyd": 239113, |
| "lloyd_gptq": 375510, |
| "lloyd_gptq_act_order": 680424, |
| "oqe": 13178, |
| "oqe_gptq": 30812, |
| "oqe_gptq_act_order": 53603 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.28811135421963197, |
| "selected_objective": 13.696685180068016, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.59.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.59.mlp.gate_proj": { |
| "base_objective": 146.43485543131828, |
| "choices": { |
| "lloyd": 265697, |
| "lloyd_gptq": 429108, |
| "lloyd_gptq_act_order": 507828, |
| "oqe": 17227, |
| "oqe_gptq": 76966, |
| "oqe_gptq_act_order": 95814 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.16536815061125154, |
| "selected_objective": 122.21919420361519, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.59.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.59.mlp.up_proj": { |
| "base_objective": 133.075846016407, |
| "choices": { |
| "lloyd": 264274, |
| "lloyd_gptq": 429400, |
| "lloyd_gptq_act_order": 512085, |
| "oqe": 16498, |
| "oqe_gptq": 75108, |
| "oqe_gptq_act_order": 95275 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1702033110309198, |
| "selected_objective": 110.4258964061737, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.59.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.6.mlp.gate_proj": { |
| "base_objective": 2.906887259334326, |
| "choices": { |
| "lloyd": 256035, |
| "lloyd_gptq": 439384, |
| "lloyd_gptq_act_order": 530046, |
| "oqe": 13459, |
| "oqe_gptq": 67261, |
| "oqe_gptq_act_order": 86455 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18500618208797193, |
| "selected_objective": 2.369095145724714, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.6.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.6.mlp.up_proj": { |
| "base_objective": 2.7272594599053264, |
| "choices": { |
| "lloyd": 255057, |
| "lloyd_gptq": 438697, |
| "lloyd_gptq_act_order": 531337, |
| "oqe": 13448, |
| "oqe_gptq": 67808, |
| "oqe_gptq_act_order": 86293 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.18881946802378624, |
| "selected_objective": 2.212299779523164, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.6.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.60.mlp.down_proj": { |
| "base_objective": 24.469972133636475, |
| "choices": { |
| "lloyd": 166712, |
| "lloyd_gptq": 350421, |
| "lloyd_gptq_act_order": 783120, |
| "oqe": 9164, |
| "oqe_gptq": 25430, |
| "oqe_gptq_act_order": 57793 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.27967102987453474, |
| "selected_objective": 17.626429826021194, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.60.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.60.mlp.gate_proj": { |
| "base_objective": 147.64978513121605, |
| "choices": { |
| "lloyd": 246506, |
| "lloyd_gptq": 431954, |
| "lloyd_gptq_act_order": 520406, |
| "oqe": 16495, |
| "oqe_gptq": 77801, |
| "oqe_gptq_act_order": 99478 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17067267910809303, |
| "selected_objective": 122.45000073313713, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.60.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.60.mlp.up_proj": { |
| "base_objective": 137.2575252354145, |
| "choices": { |
| "lloyd": 244818, |
| "lloyd_gptq": 431720, |
| "lloyd_gptq_act_order": 525558, |
| "oqe": 15899, |
| "oqe_gptq": 76350, |
| "oqe_gptq_act_order": 98295 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17680443983367222, |
| "selected_objective": 112.9897853732109, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.60.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.61.mlp.down_proj": { |
| "base_objective": 32.82849767804146, |
| "choices": { |
| "lloyd": 110420, |
| "lloyd_gptq": 308288, |
| "lloyd_gptq_act_order": 882133, |
| "oqe": 6361, |
| "oqe_gptq": 21999, |
| "oqe_gptq_act_order": 63439 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.30758900630999597, |
| "selected_objective": 22.730812698602676, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.61.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.61.mlp.gate_proj": { |
| "base_objective": 159.1120822429657, |
| "choices": { |
| "lloyd": 222414, |
| "lloyd_gptq": 434198, |
| "lloyd_gptq_act_order": 532981, |
| "oqe": 15853, |
| "oqe_gptq": 81204, |
| "oqe_gptq_act_order": 105990 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.17774123062621727, |
| "selected_objective": 130.8313049376011, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.61.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.61.mlp.up_proj": { |
| "base_objective": 149.12146452069283, |
| "choices": { |
| "lloyd": 220397, |
| "lloyd_gptq": 435596, |
| "lloyd_gptq_act_order": 536006, |
| "oqe": 15482, |
| "oqe_gptq": 80324, |
| "oqe_gptq_act_order": 104835 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1837944158495215, |
| "selected_objective": 121.71377205848694, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.61.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.62.mlp.down_proj": { |
| "base_objective": 58.632885813713074, |
| "choices": { |
| "lloyd": 54941, |
| "lloyd_gptq": 238647, |
| "lloyd_gptq_act_order": 1018377, |
| "oqe": 2464, |
| "oqe_gptq": 14243, |
| "oqe_gptq_act_order": 63968 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.3329566945027226, |
| "selected_objective": 39.11067396402359, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.62.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.62.mlp.gate_proj": { |
| "base_objective": 154.0579994916916, |
| "choices": { |
| "lloyd": 177883, |
| "lloyd_gptq": 438488, |
| "lloyd_gptq_act_order": 550975, |
| "oqe": 15570, |
| "oqe_gptq": 90017, |
| "oqe_gptq_act_order": 119707 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1976323794856063, |
| "selected_objective": 123.61115047335625, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.62.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.62.mlp.up_proj": { |
| "base_objective": 144.79678189754486, |
| "choices": { |
| "lloyd": 176845, |
| "lloyd_gptq": 437551, |
| "lloyd_gptq_act_order": 553403, |
| "oqe": 15173, |
| "oqe_gptq": 89805, |
| "oqe_gptq_act_order": 119863 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.2021813530260831, |
| "selected_objective": 115.52157261967659, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.62.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.63.mlp.down_proj": { |
| "base_objective": 159.80059432983398, |
| "choices": { |
| "lloyd": 23731, |
| "lloyd_gptq": 174042, |
| "lloyd_gptq_act_order": 1118097, |
| "oqe": 898, |
| "oqe_gptq": 9093, |
| "oqe_gptq_act_order": 66779 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.4477737421231748, |
| "selected_objective": 88.24608421325684, |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "source": "model.language_model.layers.63.mlp.down_proj.weight" |
| }, |
| "language_model.model.layers.63.mlp.gate_proj": { |
| "base_objective": 140.77778726816177, |
| "choices": { |
| "lloyd": 133946, |
| "lloyd_gptq": 430857, |
| "lloyd_gptq_act_order": 573975, |
| "oqe": 14483, |
| "oqe_gptq": 98906, |
| "oqe_gptq_act_order": 140473 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.23578961917395636, |
| "selected_objective": 107.58384642004967, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.63.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.63.mlp.up_proj": { |
| "base_objective": 128.1735941171646, |
| "choices": { |
| "lloyd": 134144, |
| "lloyd_gptq": 430943, |
| "lloyd_gptq_act_order": 573594, |
| "oqe": 14363, |
| "oqe_gptq": 99611, |
| "oqe_gptq_act_order": 139985 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.24127234967226785, |
| "selected_objective": 97.24884989857674, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.63.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.7.mlp.gate_proj": { |
| "base_objective": 3.0673272851854563, |
| "choices": { |
| "lloyd": 299325, |
| "lloyd_gptq": 442846, |
| "lloyd_gptq_act_order": 468905, |
| "oqe": 16808, |
| "oqe_gptq": 78290, |
| "oqe_gptq_act_order": 86466 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13105607110960527, |
| "selected_objective": 2.6653354223817587, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.7.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.7.mlp.up_proj": { |
| "base_objective": 2.8666873248293996, |
| "choices": { |
| "lloyd": 298944, |
| "lloyd_gptq": 442867, |
| "lloyd_gptq_act_order": 469899, |
| "oqe": 16465, |
| "oqe_gptq": 78097, |
| "oqe_gptq_act_order": 86368 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1324936193376237, |
| "selected_objective": 2.4868695456534624, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.7.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.8.mlp.gate_proj": { |
| "base_objective": 3.360856929793954, |
| "choices": { |
| "lloyd": 317456, |
| "lloyd_gptq": 443812, |
| "lloyd_gptq_act_order": 452917, |
| "oqe": 17035, |
| "oqe_gptq": 78158, |
| "oqe_gptq_act_order": 83262 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12601554693221106, |
| "selected_objective": 2.937336705625057, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.8.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.8.mlp.up_proj": { |
| "base_objective": 3.165446643717587, |
| "choices": { |
| "lloyd": 317727, |
| "lloyd_gptq": 442813, |
| "lloyd_gptq_act_order": 453128, |
| "oqe": 17086, |
| "oqe_gptq": 78208, |
| "oqe_gptq_act_order": 83678 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.12707590354523968, |
| "selected_objective": 2.7631946513429284, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.8.mlp.up_proj.weight" |
| }, |
| "language_model.model.layers.9.mlp.gate_proj": { |
| "base_objective": 3.7657600846141577, |
| "choices": { |
| "lloyd": 292068, |
| "lloyd_gptq": 445854, |
| "lloyd_gptq_act_order": 471057, |
| "oqe": 16671, |
| "oqe_gptq": 79878, |
| "oqe_gptq_act_order": 87112 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.13021976624875264, |
| "selected_objective": 3.275383686646819, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.9.mlp.gate_proj.weight" |
| }, |
| "language_model.model.layers.9.mlp.up_proj": { |
| "base_objective": 3.5701426742598414, |
| "choices": { |
| "lloyd": 293503, |
| "lloyd_gptq": 445506, |
| "lloyd_gptq_act_order": 469945, |
| "oqe": 16582, |
| "oqe_gptq": 79500, |
| "oqe_gptq_act_order": 87604 |
| }, |
| "objective_split": "held_out_selection", |
| "relative_reduction": 0.1311897692386199, |
| "selected_objective": 3.1017764806747437, |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "source": "model.language_model.layers.9.mlp.up_proj.weight" |
| } |
| }, |
| "objective_split": "held_out_selection", |
| "peak_memory_bytes": 12226340768, |
| "relative_reduction": 0.18088393617309562, |
| "row_chunk": 64, |
| "selected_objective": 4635.816319292295, |
| "student": "/Users/litwein/.omlx/models/Qwen3.8-27B-oQ4e-XL-mtp-backbone", |
| "version": 3 |
| } |
