{
  "action_generator_config": {
    "alpha": 32,
    "compute_precision": "bfloat16",
    "d_model": 1024,
    "dim_accumulation": 4,
    "factor_init_std": 0.01,
    "head_dim": 128,
    "input_dim": 2048,
    "num_base_model_layers": 18,
    "num_pg_layers": 6,
    "output_dim": 1024,
    "prefix": "expert.layers.",
    "projections": [
      {
        "factor_geometry": "q",
        "head_dim": 256,
        "in_features": 1024,
        "num_heads": 8,
        "out_features": 2048,
        "suffix": "self_attn.q_proj"
      },
      {
        "factor_geometry": "kv",
        "head_dim": 256,
        "in_features": 1024,
        "num_heads": 1,
        "out_features": 256,
        "suffix": "self_attn.k_proj"
      },
      {
        "factor_geometry": "kv",
        "head_dim": 256,
        "in_features": 1024,
        "num_heads": 1,
        "out_features": 256,
        "suffix": "self_attn.v_proj"
      },
      {
        "factor_geometry": "o",
        "head_dim": 256,
        "in_features": 2048,
        "num_heads": 8,
        "out_features": 1024,
        "suffix": "self_attn.o_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 1024,
        "num_heads": 1,
        "out_features": 4096,
        "suffix": "mlp.gate_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 1024,
        "num_heads": 1,
        "out_features": 4096,
        "suffix": "mlp.up_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 4096,
        "num_heads": 1,
        "out_features": 1024,
        "suffix": "mlp.down_proj"
      }
    ],
    "rank": 32,
    "state_dim": 32,
    "token_dim": 1024
  },
  "action_target_manifest": [
    {
      "in_features": 1024,
      "layer_index": 0,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 0,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 0,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 0,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 0,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 0,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 1,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 1,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 1,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 1,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 1,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 1,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 2,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 2,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 2,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 2,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 2,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 2,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 3,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 3,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 3,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 3,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 3,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 3,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 4,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 4,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 4,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 4,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 4,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 4,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 5,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 5,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 5,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 5,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 5,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 5,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 6,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 6,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 6,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 6,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 6,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 6,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 7,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 7,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 7,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 7,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 7,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 7,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 8,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 8,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 8,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 8,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 8,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 8,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 9,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 9,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 9,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 9,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 9,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 9,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 10,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 10,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 10,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 10,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 10,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 10,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 11,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 11,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 11,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 11,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 11,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 11,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 12,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 12,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 12,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 12,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 12,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 12,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 13,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 13,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 13,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 13,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 13,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 13,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 14,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 14,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 14,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 14,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 14,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 14,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 15,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 15,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 15,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 15,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 15,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 15,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 16,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 16,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 16,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 16,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 16,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 16,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 17,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 17,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 17,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 1024,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 17,
      "out_features": 4096,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 1024,
      "layer_index": 17,
      "out_features": 4096,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 4096,
      "layer_index": 17,
      "out_features": 1024,
      "suffix": "mlp.down_proj"
    }
  ],
  "adaptation_modules": [
    "vlm_generator",
    "action_generator",
    "base_model.paligemma_with_expert.paligemma.model.vision_tower",
    "base_model.paligemma_with_expert.paligemma.model.multi_modal_projector",
    "base_model.action_in_proj",
    "base_model.time_mlp_in",
    "base_model.time_mlp_out",
    "base_model.action_out_proj"
  ],
  "base_checkpoint": "model.safetensors",
  "base_sha256": "d33d59cd93e8cb6615835f8d85d4840e8fa810e66bb283544905e8d006feb54a",
  "format_version": 1,
  "global_step": 20000,
  "method": "morphact_pi",
  "metrics": {
    "action_condition_tokens_mean": 834.3125,
    "action_generator_calls": 4.5,
    "action_grad_norm_raw": 0.10613375157117844,
    "data_loader_workers": 8.0,
    "effective_batch_size": 16.0,
    "epoch": 2.1338881442508386,
    "grad_norm_raw": 0.22475452721118927,
    "interface_grad_norm_raw": 0.023294083774089813,
    "loss": 0.0031807039631530643,
    "lr": 2.500000153785387e-06,
    "lr_schedule_step": 19999.0,
    "micro_batches_seen": 20000.0,
    "optimizer_step": 20000.0,
    "samples_per_optimizer_step": 16.0,
    "samples_per_second": 2.2519925728198227,
    "samples_seen": 320000.0,
    "seconds_per_optimizer_step": 7.104819169081748,
    "vision_grad_norm_raw": 0.04660778492689133,
    "vlm_condition_tokens_mean": 833.3125,
    "vlm_generator_calls": 4.5,
    "vlm_grad_norm_raw": 0.19114215672016144
  },
  "micro_batches_seen": 20000,
  "model_type": "pi05",
  "norm_stats_asset_id": "Takizz/piperx-sortletter-0911-0917-140ep-raw",
  "run_config": {
    "action_morphact_layers": 6,
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "adam_eps": 1e-08,
    "config_name": "pi05_piperx_sort_letter",
    "decay_lr": 2.5e-06,
    "explosion_grad_norm": 1000000.0,
    "gradient_accumulation_steps": 1,
    "gradient_checkpointing": true,
    "log_interval": 10,
    "max_grad_norm": 1.0,
    "morphact_compute_precision": "bfloat16",
    "morphact_width": 1024,
    "num_devices": 2,
    "num_train_steps": 20000,
    "num_workers": 8,
    "peak_lr": 2.5e-05,
    "per_device_batch_size": 8,
    "precision": "bfloat16",
    "save_interval": 2000,
    "seed": 42,
    "task_indices": null,
    "vlm_morphact_layers": 8,
    "warmup_steps": 1000,
    "weight_decay": 1e-10
  },
  "vlm_generator_config": {
    "alpha": 16,
    "compute_precision": "bfloat16",
    "d_model": 1024,
    "dim_accumulation": 4,
    "factor_init_std": 0.01,
    "head_dim": 128,
    "input_dim": 2048,
    "num_base_model_layers": 18,
    "num_pg_layers": 8,
    "output_dim": 1024,
    "prefix": "vlm.layers.",
    "projections": [
      {
        "factor_geometry": "q",
        "head_dim": 256,
        "in_features": 2048,
        "num_heads": 8,
        "out_features": 2048,
        "suffix": "self_attn.q_proj"
      },
      {
        "factor_geometry": "kv",
        "head_dim": 256,
        "in_features": 2048,
        "num_heads": 1,
        "out_features": 256,
        "suffix": "self_attn.k_proj"
      },
      {
        "factor_geometry": "kv",
        "head_dim": 256,
        "in_features": 2048,
        "num_heads": 1,
        "out_features": 256,
        "suffix": "self_attn.v_proj"
      },
      {
        "factor_geometry": "o",
        "head_dim": 256,
        "in_features": 2048,
        "num_heads": 8,
        "out_features": 2048,
        "suffix": "self_attn.o_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 2048,
        "num_heads": 1,
        "out_features": 16384,
        "suffix": "mlp.gate_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 2048,
        "num_heads": 1,
        "out_features": 16384,
        "suffix": "mlp.up_proj"
      },
      {
        "factor_geometry": "matrix",
        "head_dim": null,
        "in_features": 16384,
        "num_heads": 1,
        "out_features": 2048,
        "suffix": "mlp.down_proj"
      }
    ],
    "rank": 16,
    "state_dim": null,
    "token_dim": 1024
  },
  "vlm_target_manifest": [
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 0,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 0,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 1,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 1,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 2,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 2,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 3,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 3,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 4,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 4,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 5,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 5,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 6,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 6,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 7,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 7,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 8,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 8,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 9,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 9,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 10,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 10,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 11,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 11,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 12,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 12,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 13,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 13,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 14,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 14,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 15,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 15,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 16,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 16,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 2048,
      "suffix": "self_attn.q_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 256,
      "suffix": "self_attn.k_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 256,
      "suffix": "self_attn.v_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 2048,
      "suffix": "self_attn.o_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 16384,
      "suffix": "mlp.gate_proj"
    },
    {
      "in_features": 2048,
      "layer_index": 17,
      "out_features": 16384,
      "suffix": "mlp.up_proj"
    },
    {
      "in_features": 16384,
      "layer_index": 17,
      "out_features": 2048,
      "suffix": "mlp.down_proj"
    }
  ],
  "world_size": 2
}