| { |
| "method": "ria", |
| "sparsity_target": 0.7, |
| "sparsity_actual": 0.6997543415043399, |
| "alpha": 0.5, |
| "scoring": "(|W[r,c]|/||W[r,:]||_1 + |W[r,c]|/||W[:,c]||_1) * act_rms[c]^alpha", |
| "correction": "none", |
| "n_calib_batches": 64, |
| "ppl_before": null, |
| "ppl_after": null, |
| "layer_info": { |
| "model.layers.0.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.0.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.0.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.0.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.0.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.1.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.1.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.1.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.1.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.2.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.2.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.2.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.2.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.3.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.3.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.3.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.3.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.4.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.4.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.4.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.4.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.5.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.5.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.5.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.5.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.6.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.6.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.6.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.6.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.7.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.7.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.7.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.7.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.8.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.8.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.8.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.8.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.9.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.9.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.9.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.9.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.10.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.10.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.10.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.10.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.11.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.11.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.11.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.11.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.12.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.12.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.12.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.12.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.13.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.13.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.13.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.13.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.14.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.14.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.14.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.14.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.15.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.15.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.15.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.15.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.16.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.16.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.16.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.16.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.17.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.17.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.17.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.17.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.18.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.18.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.18.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.18.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.19.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.19.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.19.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.19.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.20.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.20.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.20.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.20.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.21.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.21.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.21.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.21.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.22.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.22.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.22.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.22.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "model.layers.23.attn.i_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.23.attn.f_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.23.attn.g_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.23.attn.o_proj": { |
| "out_features": 2048, |
| "in_features": 2048, |
| "n_pruned": 2934784 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "out_features": 5632, |
| "in_features": 2048, |
| "n_pruned": 8070656 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "out_features": 2048, |
| "in_features": 5632, |
| "n_pruned": 8073216 |
| }, |
| "lm_head": { |
| "out_features": 32000, |
| "in_features": 2048, |
| "n_pruned": 45856000 |
| } |
| } |
| } |
