| { |
| "alpha": 16, |
| "architecture": "canonical", |
| "base_model_name_or_path": "Qwen/Qwen3-8B", |
| "base_model_revision": "b968826d9c46dd6066d109eabc6255188de91218", |
| "d_core": 1024, |
| "d_layer": 32, |
| "d_z": 256, |
| "format_version": 1, |
| "hidden": 512, |
| "layer_path": "model.layers", |
| "library_name": "portallib", |
| "n_layers": 36, |
| "projection_targets": [ |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 0, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 0, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 1, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 1, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 2, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 2, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 3, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 3, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 4, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 4, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 5, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 5, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 6, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 6, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 7, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 7, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 8, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 8, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 9, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 9, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 10, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 10, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 11, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 11, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 12, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 12, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 13, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 13, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 14, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 14, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 15, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 15, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 16, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 16, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 17, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 17, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 18, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 18, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 19, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 19, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 20, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 20, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 21, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 21, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 22, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 22, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 23, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 23, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 24, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 24, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 25, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 25, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 26, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 26, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 27, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 27, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 28, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 28, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 29, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 29, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 30, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 30, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 31, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 31, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 32, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 32, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 33, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 33, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 34, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 34, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "q", |
| "layer_index": 35, |
| "module_name": "q", |
| "module_path": "self_attn.q_proj", |
| "out_features": 4096, |
| "output_group": "q" |
| }, |
| { |
| "in_features": 4096, |
| "input_group": "v", |
| "layer_index": 35, |
| "module_name": "v", |
| "module_path": "self_attn.v_proj", |
| "out_features": 1024, |
| "output_group": "v" |
| } |
| ], |
| "rank": 8, |
| "task_type": "CAUSAL_LM", |
| "tasks": [ |
| "truthfulqa", |
| "rte", |
| "cb", |
| "copa", |
| "wic", |
| "wsc", |
| "boolq", |
| "arc_easy", |
| "arc_challenge", |
| "hellaswag", |
| "openbookqa", |
| "winogrande", |
| "commonsense_qa", |
| "sciq" |
| ] |
| } |
|
|
