| { |
| "action_generator_config": { |
| "alpha": 32, |
| "compute_precision": "bfloat16", |
| "d_model": 1024, |
| "dim_accumulation": 4, |
| "factor_init_std": 0.01, |
| "head_dim": 128, |
| "input_dim": 2048, |
| "num_base_model_layers": 18, |
| "num_pg_layers": 6, |
| "output_dim": 1024, |
| "prefix": "expert.layers.", |
| "projections": [ |
| { |
| "factor_geometry": "q", |
| "head_dim": 256, |
| "in_features": 1024, |
| "num_heads": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "factor_geometry": "kv", |
| "head_dim": 256, |
| "in_features": 1024, |
| "num_heads": 1, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "factor_geometry": "kv", |
| "head_dim": 256, |
| "in_features": 1024, |
| "num_heads": 1, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "factor_geometry": "o", |
| "head_dim": 256, |
| "in_features": 2048, |
| "num_heads": 8, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 1024, |
| "num_heads": 1, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 1024, |
| "num_heads": 1, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 4096, |
| "num_heads": 1, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| } |
| ], |
| "rank": 32, |
| "state_dim": 32, |
| "token_dim": 1024 |
| }, |
| "action_target_manifest": [ |
| { |
| "in_features": 1024, |
| "layer_index": 0, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 0, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 0, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 0, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 0, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 0, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 1, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 1, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 1, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 1, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 1, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 1, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 2, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 2, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 2, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 2, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 2, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 2, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 3, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 3, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 3, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 3, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 3, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 3, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 4, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 4, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 4, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 4, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 4, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 4, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 5, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 5, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 5, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 5, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 5, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 5, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 6, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 6, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 6, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 6, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 6, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 6, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 7, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 7, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 7, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 7, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 7, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 7, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 8, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 8, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 8, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 8, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 8, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 9, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 9, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 9, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 9, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 9, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 9, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 10, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 10, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 10, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 10, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 10, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 10, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 11, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 11, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 11, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 11, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 11, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 11, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 12, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 12, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 12, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 12, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 12, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 12, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 13, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 13, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 13, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 13, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 13, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 13, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 14, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 14, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 14, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 14, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 14, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 14, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 15, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 15, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 15, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 15, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 15, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 15, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 16, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 16, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 16, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 16, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 16, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 16, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 17, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 17, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 17, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 1024, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 17, |
| "out_features": 4096, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 1024, |
| "layer_index": 17, |
| "out_features": 4096, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 4096, |
| "layer_index": 17, |
| "out_features": 1024, |
| "suffix": "mlp.down_proj" |
| } |
| ], |
| "adaptation_modules": [ |
| "vlm_generator", |
| "action_generator", |
| "base_model.paligemma_with_expert.paligemma.model.vision_tower", |
| "base_model.paligemma_with_expert.paligemma.model.multi_modal_projector", |
| "base_model.action_in_proj", |
| "base_model.time_mlp_in", |
| "base_model.time_mlp_out", |
| "base_model.action_out_proj" |
| ], |
| "base_checkpoint": "model.safetensors", |
| "base_sha256": "d33d59cd93e8cb6615835f8d85d4840e8fa810e66bb283544905e8d006feb54a", |
| "format_version": 1, |
| "global_step": 20000, |
| "method": "morphact_pi", |
| "metrics": { |
| "action_condition_tokens_mean": 834.3125, |
| "action_generator_calls": 4.5, |
| "action_grad_norm_raw": 0.10613375157117844, |
| "data_loader_workers": 8.0, |
| "effective_batch_size": 16.0, |
| "epoch": 2.1338881442508386, |
| "grad_norm_raw": 0.22475452721118927, |
| "interface_grad_norm_raw": 0.023294083774089813, |
| "loss": 0.0031807039631530643, |
| "lr": 2.500000153785387e-06, |
| "lr_schedule_step": 19999.0, |
| "micro_batches_seen": 20000.0, |
| "optimizer_step": 20000.0, |
| "samples_per_optimizer_step": 16.0, |
| "samples_per_second": 2.2519925728198227, |
| "samples_seen": 320000.0, |
| "seconds_per_optimizer_step": 7.104819169081748, |
| "vision_grad_norm_raw": 0.04660778492689133, |
| "vlm_condition_tokens_mean": 833.3125, |
| "vlm_generator_calls": 4.5, |
| "vlm_grad_norm_raw": 0.19114215672016144 |
| }, |
| "micro_batches_seen": 20000, |
| "model_type": "pi05", |
| "norm_stats_asset_id": "Takizz/piperx-sortletter-0911-0917-140ep-raw", |
| "run_config": { |
| "action_morphact_layers": 6, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_eps": 1e-08, |
| "config_name": "pi05_piperx_sort_letter", |
| "decay_lr": 2.5e-06, |
| "explosion_grad_norm": 1000000.0, |
| "gradient_accumulation_steps": 1, |
| "gradient_checkpointing": true, |
| "log_interval": 10, |
| "max_grad_norm": 1.0, |
| "morphact_compute_precision": "bfloat16", |
| "morphact_width": 1024, |
| "num_devices": 2, |
| "num_train_steps": 20000, |
| "num_workers": 8, |
| "peak_lr": 2.5e-05, |
| "per_device_batch_size": 8, |
| "precision": "bfloat16", |
| "save_interval": 2000, |
| "seed": 42, |
| "task_indices": null, |
| "vlm_morphact_layers": 8, |
| "warmup_steps": 1000, |
| "weight_decay": 1e-10 |
| }, |
| "vlm_generator_config": { |
| "alpha": 16, |
| "compute_precision": "bfloat16", |
| "d_model": 1024, |
| "dim_accumulation": 4, |
| "factor_init_std": 0.01, |
| "head_dim": 128, |
| "input_dim": 2048, |
| "num_base_model_layers": 18, |
| "num_pg_layers": 8, |
| "output_dim": 1024, |
| "prefix": "vlm.layers.", |
| "projections": [ |
| { |
| "factor_geometry": "q", |
| "head_dim": 256, |
| "in_features": 2048, |
| "num_heads": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "factor_geometry": "kv", |
| "head_dim": 256, |
| "in_features": 2048, |
| "num_heads": 1, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "factor_geometry": "kv", |
| "head_dim": 256, |
| "in_features": 2048, |
| "num_heads": 1, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "factor_geometry": "o", |
| "head_dim": 256, |
| "in_features": 2048, |
| "num_heads": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 2048, |
| "num_heads": 1, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 2048, |
| "num_heads": 1, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "factor_geometry": "matrix", |
| "head_dim": null, |
| "in_features": 16384, |
| "num_heads": 1, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| } |
| ], |
| "rank": 16, |
| "state_dim": null, |
| "token_dim": 1024 |
| }, |
| "vlm_target_manifest": [ |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 0, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 0, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 1, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 1, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 2, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 2, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 3, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 3, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 4, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 4, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 5, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 5, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 6, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 6, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 7, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 7, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 8, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 8, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 9, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 9, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 10, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 10, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 11, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 11, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 12, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 12, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 13, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 13, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 14, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 14, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 15, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 15, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 16, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 16, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 2048, |
| "suffix": "self_attn.q_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 256, |
| "suffix": "self_attn.k_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 256, |
| "suffix": "self_attn.v_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 2048, |
| "suffix": "self_attn.o_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 16384, |
| "suffix": "mlp.gate_proj" |
| }, |
| { |
| "in_features": 2048, |
| "layer_index": 17, |
| "out_features": 16384, |
| "suffix": "mlp.up_proj" |
| }, |
| { |
| "in_features": 16384, |
| "layer_index": 17, |
| "out_features": 2048, |
| "suffix": "mlp.down_proj" |
| } |
| ], |
| "world_size": 2 |
| } |
|
|
