diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9045db41da980a673631345adf29c26776f44204 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +--- +language: en +library_name: mlx +pipeline_tag: image-text-to-text +tags: +- mlx +license: other +license_name: kimi-k3 +license_link: https://huggingface.co/moonshotai/Kimi-K3/blob/main/LICENSE +base_model: moonshotai/Kimi-K3 +base_model_relation: quantized +--- + +# kernelpool/Kimi-K3-2bit-UVMAX + +Mixed-precision (UVMAX) quantization of [moonshotai/Kimi-K3](https://huggingface.co/moonshotai/Kimi-K3). + +## What is UVMAX? + +UVMAX is a mixed-precision scheme: bit widths are assigned per tensor class +from measured round-trip quantization error, rather than uniformly. + +| Tensor class | Bits | Parameters | Size | Share | +|---|---|---|---|---| +| Expert FFNs (routed, latent space) | 2 (gs 128) | 2.72 T | 713.2 GiB | 93.8% | +| Shared experts, MoE latent projections, dense MLP | 8 (gs 64) | 17.5 B | 17.4 GiB | 2.3% | +| Attention (KDA + MLA, all projections) | 6 (gs 64) | 36 B | 27.4 GiB | 3.6% | +| Embeddings, `lm_head` | 4 (gs 64) | 2.4 B | 1.2 GiB | 0.2% | +| MoE routers | 8 (gs 64) | 0.6 B | 0.6 GiB | 0.1% | +| Vision tower + projector (unquantized bf16) | — | 0.4 B | 0.8 GiB | 0.1% | +| Norms, AttnRes projections, gate params (unquantized) | — | — | 0.1 GiB | <0.1% | + +## Use with mlx + +This model requires Kimi K3 support from mlx-lm PR #TBD, which has not yet +been merged. Until it is included in an mlx-lm release, install mlx-lm from +the PR branch: + +```bash +pip install git+https://github.com/ml-explore/mlx-lm.git@refs/pull/TBD/head +pip install tiktoken +``` + +```python +from mlx_lm import load, generate + +model, tokenizer = load( + "kernelpool/Kimi-K3-2bit-UVMAX", + tokenizer_config={"trust_remote_code": True}, + trust_remote_code=True, +) + +prompt = "hello" + +messages = [{"role": "user", "content": prompt}] +prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True) + +response = generate(model, tokenizer, prompt=prompt, verbose=True) +``` diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..ae76ae27f7f2228f42a6ea3828ba018e28f16f73 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,9 @@ +{ + "<|end_header_id|>": 163844, + "<|im_assistant|>": 163842, + "<|im_end|>": 163840, + "<|im_middle|>": 163846, + "<|im_system|>": 163845, + "<|im_user|>": 163841, + "<|start_header_id|>": 163843 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c32a2e7c374429d12baa58de4f373c9354500451 --- /dev/null +++ b/config.json @@ -0,0 +1,14460 @@ +{ + "architectures": [ + "KimiK3ForConditionalGeneration" + ], + "auto_map": { + "AutoConfig": "configuration_kimi_k3.KimiK3Config", + "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration", + "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration" + }, + "bos_token_id": 163584, + "dtype": "bfloat16", + "eos_token_id": 163586, + "ignore_index": -100, + "image_placeholder": "<|kimi_image_placeholder|>", + "media_placeholder_token_id": 163605, + "model_type": "kimi_k3", + "pad_token_id": 163839, + "quantization": { + "group_size": 32, + "bits": 4, + "mode": "mxfp4", + "language_model.model.embed_tokens": { + "bits": 4, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.lm_head": { + "bits": 4, + "group_size": 64, + "mode": "affine" + } + }, + "quantization_config": { + "group_size": 32, + "bits": 4, + "mode": "mxfp4", + "language_model.model.embed_tokens": { + "bits": 4, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.0.mlp.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.1.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.2.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.3.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.4.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.5.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.6.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.7.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.8.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.9.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.10.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.11.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.12.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.13.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.14.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.15.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.16.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.17.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.18.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.19.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.20.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.21.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.22.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.23.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.24.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.25.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.26.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.27.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.28.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.29.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.30.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.31.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.32.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.33.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.34.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.35.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.36.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.37.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.38.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.39.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.40.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.41.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.42.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.43.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.44.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.45.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.46.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.47.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.48.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.49.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.50.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.51.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.52.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.53.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.54.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.55.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.56.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.57.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.58.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.59.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.60.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.61.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.62.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.63.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.64.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.65.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.66.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.67.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.68.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.69.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.70.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.71.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.72.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.73.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.74.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.75.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.76.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.77.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.78.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.79.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.80.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.81.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.82.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.83.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.84.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.85.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.86.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.87.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.88.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.89.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.qkv_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.f_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.f_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.90.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.91.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.q_a_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.q_b_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.embed_q": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.unembed_out": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.o_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.self_attn.g_proj": { + "bits": 6, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.gate": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.gate_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.up_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.switch_mlp.down_proj": { + "bits": 2, + "group_size": 128, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.routed_expert_down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.routed_expert_up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.gate_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.up_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.model.layers.92.mlp.shared_experts.down_proj": { + "bits": 8, + "group_size": 64, + "mode": "affine" + }, + "language_model.lm_head": { + "bits": 4, + "group_size": 64, + "mode": "affine" + } + }, + "text_config": { + "_name_or_path": "", + "activation_situ_beta": 4.0, + "activation_situ_linear_beta": 25.0, + "add_cross_attention": false, + "architectures": [ + "KimiLinearForCausalLM" + ], + "attn_res_block_size": 12, + "auto_map": { + "AutoConfig": "configuration_kimi_k3.KimiLinearConfig", + "AutoModel": "modeling_kimi_linear.KimiLinearModel", + "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM" + }, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": 163584, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 163586, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "first_k_dense_replace": 1, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "situ", + "hidden_size": 7168, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 33792, + "is_decoder": false, + "is_encoder_decoder": false, + "kv_lora_rank": 512, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "latent_moe_use_norm": true, + "length_penalty": 1.0, + "linear_attn_config": { + "full_attn_layers": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64, + 68, + 72, + 76, + 80, + 84, + 88, + 92, + 93 + ], + "gate_lower_bound": -5.0, + "head_dim": 128, + "kda_layers": [ + 1, + 2, + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 29, + 30, + 31, + 33, + 34, + 35, + 37, + 38, + 39, + 41, + 42, + 43, + 45, + 46, + 47, + 49, + 50, + 51, + 53, + 54, + 55, + 57, + 58, + 59, + 61, + 62, + 63, + 65, + 66, + 67, + 69, + 70, + 71, + 73, + 74, + 75, + 77, + 78, + 79, + 81, + 82, + 83, + 85, + 86, + 87, + 89, + 90, + 91 + ], + "num_heads": 96, + "short_conv_kernel_size": 4, + "use_full_rank_gate": true + }, + "max_length": 20, + "max_position_embeddings": 1048576, + "min_length": 0, + "mla_use_nope": true, + "mla_use_output_gate": true, + "model_type": "kimi_linear", + "moe_intermediate_size": 3072, + "moe_layer_freq": 1, + "moe_renormalize": true, + "moe_router_activation_func": "sigmoid", + "no_repeat_ngram_size": 0, + "num_attention_heads": 96, + "num_beam_groups": 1, + "num_beams": 1, + "num_expert_group": 1, + "num_experts": 896, + "num_experts_per_token": 16, + "num_hidden_layers": 93, + "num_key_value_heads": 96, + "num_nextn_predict_layers": 0, + "num_return_sequences": 1, + "num_shared_experts": 2, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 163839, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "quantization_config": { + "config_groups": { + "group_0": { + "format": "mxfp4-pack-quantized", + "input_activations": null, + "output_activations": null, + "targets": [ + "Linear" + ], + "weights": { + "actorder": null, + "block_structure": null, + "dynamic": false, + "group_size": 32, + "num_bits": 4, + "observer": "minmax", + "observer_kwargs": {}, + "scale_dtype": "torch.uint8", + "strategy": "group", + "symmetric": true, + "type": "float", + "zp_dtype": null + } + } + }, + "format": "mxfp4-pack-quantized", + "global_compression_ratio": null, + "ignore": [ + "re:.*self_attn.*", + "re:.*shared_experts.*", + "re:.*mlp\\.(gate|up|gate_up|down)_proj.*", + "re:.*lm_head.*", + "re:.*vision_tower.*", + "re:.*mm_projector.*" + ], + "kv_cache_scheme": null, + "quant_method": "compressed-tensors", + "quantization_status": "compressed" + }, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-05, + "routed_expert_hidden_size": 3584, + "routed_scaling_factor": 1.0, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "topk_group": 1, + "topk_method": "noaux_tc", + "torchscript": false, + "transformers_version": "4.56.2", + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_grouped_topk": true, + "v_head_dim": 128, + "vocab_size": 163840 + }, + "tie_word_embeddings": false, + "vision_config": { + "_attn_implementation": "flash_attention_2", + "activation_func": "gelu_pytorch_tanh", + "attn_bias": false, + "init_pos_emb_height": 64, + "init_pos_emb_time": 4, + "init_pos_emb_width": 64, + "linear_bias": false, + "merge_kernel_size": [ + 2, + 2 + ], + "merge_type": "sd2_tpool", + "mlp_type": "mlp2", + "mm_hidden_size": 1024, + "mm_projector_type": "patchmergerv2", + "norm_type": "rmsnorm", + "patch_embed_proj_bias": false, + "patch_size": 14, + "pos_emb_interpolation_mode": "bilinear", + "pos_emb_type": "divided_fixed", + "projector_hidden_act": "gelu", + "projector_ln_eps": 1e-05, + "qkv_hidden_size": 1536, + "text_hidden_size": 7168, + "vt_hidden_size": 1024, + "vt_intermediate_size": 4096, + "vt_num_attention_heads": 12, + "vt_num_hidden_layers": 27 + } +} \ No newline at end of file diff --git a/configuration_kimi_k3.py b/configuration_kimi_k3.py new file mode 100644 index 0000000000000000000000000000000000000000..b38c4f8fcab1ce295aea236772ea1cba57f978c9 --- /dev/null +++ b/configuration_kimi_k3.py @@ -0,0 +1,285 @@ +from typing import Optional + +from transformers.configuration_utils import PretrainedConfig + + +class KimiLinearConfig(PretrainedConfig): + model_type = "kimi_linear" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + model_type="kimi_linear", + vocab_size=163840, + hidden_size=4096, + head_dim=None, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + rope_theta=10000.0, + rope_scaling=None, + tie_word_embeddings=False, + moe_intermediate_size: Optional[int] = None, + moe_renormalize: bool = True, + moe_router_activation_func: str = "sigmoid", + num_experts: Optional[int] = None, + num_experts_per_token: Optional[int] = None, + num_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, + first_k_dense_replace: int = 0, + moe_layer_freq: int = 1, + use_grouped_topk: bool = True, + num_expert_group: int = 1, + topk_group: int = 1, + q_lora_rank: Optional[int] = None, + kv_lora_rank: Optional[int] = None, + qk_nope_head_dim: Optional[int] = None, + qk_rope_head_dim: Optional[int] = None, + v_head_dim: Optional[int] = None, + mla_use_nope: Optional[bool] = False, + mla_use_output_gate: Optional[bool] = False, + num_nextn_predict_layers: int = 0, + linear_attn_config: Optional[dict] = None, + attn_res_block_size: Optional[int] = None, + latent_moe_use_norm: bool = False, + activation_situ_beta: Optional[float] = None, + activation_situ_linear_beta: Optional[float] = None, + max_position_embeddings: int = 4096, + routed_expert_hidden_size: Optional[int] = None, + topk_method: str = "noaux_tc", + **kwargs, + ): + self.model_type = model_type + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.head_dim = ( + head_dim if head_dim is not None else hidden_size // num_attention_heads + ) + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.mla_use_nope = mla_use_nope + self.mla_use_output_gate = mla_use_output_gate + # moe config + self.num_experts = num_experts + self.num_experts_per_token = num_experts_per_token + self.moe_renormalize = moe_renormalize + self.num_shared_experts = num_shared_experts + self.routed_scaling_factor = routed_scaling_factor + self.moe_router_activation_func = moe_router_activation_func + assert self.moe_router_activation_func in ("softmax", "sigmoid") + self.moe_intermediate_size = moe_intermediate_size + self.first_k_dense_replace = first_k_dense_replace + self.moe_layer_freq = moe_layer_freq + self.use_grouped_topk = use_grouped_topk + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.num_nextn_predict_layers = num_nextn_predict_layers + + self.attn_res_block_size = attn_res_block_size + self.latent_moe_use_norm = latent_moe_use_norm + self.activation_situ_beta = activation_situ_beta + self.activation_situ_linear_beta = activation_situ_linear_beta + self.max_position_embeddings = max_position_embeddings + self.routed_expert_hidden_size = routed_expert_hidden_size + self.topk_method = topk_method + + if linear_attn_config is not None: + assert linear_attn_config["kda_layers"] is not None + assert linear_attn_config["full_attn_layers"] is not None + self.linear_attn_config = linear_attn_config + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def is_mla(self): + return ( + self.q_lora_rank is not None + or self.kv_lora_rank is not None + or self.qk_nope_head_dim is not None + or self.qk_rope_head_dim is not None + or self.v_head_dim is not None + or self.mla_use_nope is True + ) + + @property + def is_moe(self): + return self.num_experts is not None + + @property + def is_linear_attn(self) -> bool: + return not ( + self.linear_attn_config is None + or ( + isinstance(self.linear_attn_config, dict) + and self.linear_attn_config["kda_layers"] is not None + and len(self.linear_attn_config["kda_layers"]) == 0 + ) + ) + + def is_kda_layer(self, layer_idx: int): + return ( + self.linear_attn_config is not None + and (layer_idx + 1) in self.linear_attn_config["kda_layers"] + ) + + +class KimiK3VisionConfig(PretrainedConfig): + + def __init__( + self, + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + init_pos_emb_time: int = 4, + pos_emb_type: str = 'divided_fixed', + vt_num_attention_heads: int = 12, + vt_num_hidden_layers: int = 27, + vt_hidden_size: int = 1024, + vt_intermediate_size: int = 4096, + merge_kernel_size: tuple = (2, 2), + merge_type: str = 'sd2_tpool', + _attn_implementation: str = 'flash_attention_2', + # MM Projector parameters + mm_projector_type: str = 'patchmergerv2', + mm_hidden_size: int | None = None, + projector_hidden_act: str = "gelu", + projector_ln_eps: float = 1e-5, + # vision tower parameters + qkv_hidden_size: int = 1536, + norm_type: str = 'rmsnorm', + attn_bias: bool = False, + patch_embed_proj_bias: bool = False, + mlp_type: str = 'mlp2', + linear_bias: bool = False, + activation_func: str = 'gelu_pytorch_tanh', + pos_emb_interpolation_mode: str = 'bilinear', + # Other parameters + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + text_hidden_size=7168, + **kwargs): + + self.patch_size = patch_size + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + self.init_pos_emb_time = init_pos_emb_time + self.pos_emb_type = pos_emb_type + self.vt_num_attention_heads = vt_num_attention_heads + self.vt_num_hidden_layers = vt_num_hidden_layers + self.vt_hidden_size = vt_hidden_size + self.vt_intermediate_size = vt_intermediate_size + self.merge_kernel_size = merge_kernel_size + self.merge_type = merge_type + self._attn_implementation = _attn_implementation + + # MM Projector config + self.mm_projector_type = mm_projector_type + self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else vt_hidden_size + self.projector_hidden_act = projector_hidden_act + self.projector_ln_eps = projector_ln_eps + self.text_hidden_size = text_hidden_size + + # vision tower parameters + self.qkv_hidden_size = qkv_hidden_size + self.norm_type = norm_type + self.attn_bias = attn_bias + self.patch_embed_proj_bias = patch_embed_proj_bias + self.mlp_type = mlp_type + self.linear_bias = linear_bias + self.activation_func = activation_func + self.pos_emb_interpolation_mode = pos_emb_interpolation_mode + + super().__init__(**kwargs) + + +class KimiK3Config(PretrainedConfig): + """Kimi-K3 model configuration. + + Args: + text_config (dict | KimiLinearConfig): Configuration for the text model. + + Vision Tower Parameters (from MoonViT3dConfig): + patch_size (int): Patch size for vision tower. + init_pos_emb_height (int): Initial position embedding height. + init_pos_emb_width (int): Initial position embedding width. + init_pos_emb_time (int): Initial position embedding time dimension. + pos_emb_type (str): Type of position embedding. + vt_num_attention_heads (int): Number of attention heads in vision tower. + vt_num_hidden_layers (int): Number of hidden layers in vision tower. + vt_hidden_size (int): Hidden size of vision tower. + vt_intermediate_size (int): Intermediate size in vision tower FFN. + merge_kernel_size (tuple): Kernel size for patch merging. + merge_type (str): Type of merge operation. + _attn_implementation (str): Attention implementation type. + + MM Projector Parameters (from MultiModalProjectorConfig): + mm_projector_type (str): Type of multimodal projector. + mm_hidden_size (int): Hidden size from vision tower (should match vt_hidden_size). + projector_hidden_act (str): Activation function for projector. + projector_ln_eps (float): Layer norm epsilon for projector. + + Other Parameters: + ignore_index (int): The ignore index for the loss function. + media_placeholder_token_id (int): The token ID to use for media placeholders. + pad_token_id (int): The token ID to use for padding. + """ + + model_type = "kimi_k3" + + def __init__( + self, + text_config: dict | KimiLinearConfig = None, + vision_config: dict | KimiK3VisionConfig = None, + # Other parameters + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + **kwargs, + ): + if isinstance(text_config, dict): + text_config = KimiLinearConfig(**text_config) + if isinstance(vision_config, dict): + vision_config = KimiK3VisionConfig(**vision_config) + self.text_config = text_config + self.vision_config = vision_config + # Other config + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + if getattr(self.text_config, "quantization_config", None) is not None: + self.quantization_config = self.text_config.quantization_config + + super().__init__(pad_token_id=pad_token_id, **kwargs) diff --git a/encoding_k3.py b/encoding_k3.py new file mode 100644 index 0000000000000000000000000000000000000000..44a56b7371fab32513da0eaed6b9a8a7c56499a2 --- /dev/null +++ b/encoding_k3.py @@ -0,0 +1,647 @@ +"""Kimi K3 XTML encoding helpers. + +This module keeps chat rendering in Python. +Callers that need token IDs should consume ``EncodeSegment`` objects directly: +structural markers may be encoded as tiktoken special tokens, while user/tool +text and attribute values are encoded as ordinary text. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Iterable, Optional + +OPEN_TOKEN = "<|open|>" +CLOSE_TOKEN = "<|close|>" +SEP_TOKEN = "<|sep|>" +END_OF_MSG_TOKEN = "<|end_of_msg|>" +IMAGE_PLACEHOLDER = "<|kimi_image_placeholder|>" + +_VALID_THINKING_EFFORTS = {"low", "high", "max"} + + +@dataclass(frozen=True) +class EncodeSegment: + text: str + allow_special: bool = False + + +class _ImagePromptState: + def __init__(self, image_prompts: Optional[list[str]] = None): + self.image_prompts = image_prompts + self.index = 0 + + def next_prompt(self) -> str: + if self.image_prompts is None: + return IMAGE_PLACEHOLDER + if self.index >= len(self.image_prompts): + raise ValueError("More image placeholders than image prompts.") + prompt = self.image_prompts[self.index] + self.index += 1 + return prompt + + def assert_consumed(self) -> None: + if self.image_prompts is None: + return + if self.index != len(self.image_prompts): + raise ValueError( + f"image prompt count {len(self.image_prompts)} != " + f"consumed placeholder count {self.index}" + ) + + +def _segment(text: Any, *, allow_special: bool = False) -> list[EncodeSegment]: + text = str(text) + if not text: + return [] + return [EncodeSegment(text, allow_special=allow_special)] + + +def _control(text: str) -> list[EncodeSegment]: + return _segment(text, allow_special=True) + + +def _text(text: Any) -> list[EncodeSegment]: + return _segment(text, allow_special=False) + + +def _append_text( + segments: list[EncodeSegment], + text: Any, + image_state: _ImagePromptState, +) -> None: + text = str(text) + if text == "": + return + if image_state.image_prompts is None or IMAGE_PLACEHOLDER not in text: + segments.extend(_text(text)) + return + + parts = text.split(IMAGE_PLACEHOLDER) + for i, part in enumerate(parts): + segments.extend(_text(part)) + if i < len(parts) - 1: + segments.extend(_segment(image_state.next_prompt(), + allow_special=True)) + + +def _escape_attr_value(value: Any) -> str: + return str(value).replace("&", "&").replace('"', """) + + +def _attr(key: str, value: Any) -> list[EncodeSegment]: + return ( + _text(f" {key}") + + _text('="') + + _text(_escape_attr_value(value)) + + _text('"') + ) + + +def _open_tag(tag: str, attrs: Iterable[tuple[str, Any]] = ()) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend(_control(OPEN_TOKEN)) + segments.extend(_text(tag)) + for key, value in attrs: + segments.extend(_attr(key, value)) + segments.extend(_control(SEP_TOKEN)) + return segments + +def _close_tag(tag: str) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend(_control(CLOSE_TOKEN)) + segments.extend(_text(tag)) + segments.extend(_control(SEP_TOKEN)) + return segments + + +def _end_of_msg() -> list[EncodeSegment]: + return _control(END_OF_MSG_TOKEN) + + +def _json_compact(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _is_mapping(value: Any) -> bool: + return isinstance(value, dict) + + +def _xtml_type(value: Any) -> str: + if isinstance(value, bool): + return "boolean" + if value is None: + return "null" + if isinstance(value, (int, float)) and not isinstance(value, bool): + return "number" + if isinstance(value, str): + return "string" + if _is_mapping(value): + return "object" + return "array" + + +def _xtml_value(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +def _get_value(obj: Any, key: str, default: Any = None) -> Any: + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + +def extract_response_schema(response_format: Any) -> Any: + if response_format is None: + return None + + json_schema = _get_value(response_format, "json_schema") + if json_schema is None: + return None + + if isinstance(json_schema, dict): + return json_schema.get( + "schema", + json_schema.get("json_schema", json_schema), + ) + + schema = _get_value(json_schema, "schema") + if schema is not None: + return schema + + schema = _get_value(json_schema, "json_schema") + if schema is not None: + return schema + + return json_schema + + +def deep_sort_dict(obj: Any) -> Any: + if isinstance(obj, dict): + return {k: deep_sort_dict(v) for k, v in sorted(obj.items())} + if isinstance(obj, list): + return [deep_sort_dict(item) for item in obj] + return obj + + +def normalize_tool_arguments(arguments: Any) -> tuple[dict[str, Any], Optional[str]]: + if arguments is None: + return {}, None + if isinstance(arguments, dict): + return arguments, None + if isinstance(arguments, str): + if not arguments.strip(): + return {}, None + try: + parsed = json.loads(arguments) + except json.JSONDecodeError: + return {}, arguments + if not isinstance(parsed, dict): + raise ValueError("Kimi K3 tool call arguments must be a JSON object.") + return parsed, None + raise TypeError( + "Kimi K3 tool call arguments must be a dict or a JSON object string." + ) + + +def normalize_message(message: Any) -> Any: + if not isinstance(message, dict): + return message + + normalized = dict(message) + + tools = normalized.get("tools") + if tools is not None: + normalized["tools"] = deep_sort_dict(tools) + + tool_calls = normalized.get("tool_calls") + if not tool_calls: + return normalized + + normalized_calls = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + normalized_calls.append(tool_call) + continue + + tc = dict(tool_call) + function = tc.get("function") + if isinstance(function, dict): + fn = dict(function) + arguments, json_block = normalize_tool_arguments(fn.get("arguments")) + fn["arguments"] = arguments + if json_block is None: + fn.pop("_xtml_json_block", None) + else: + fn["_xtml_json_block"] = json_block + tc["function"] = fn + else: + arguments, json_block = normalize_tool_arguments(tc.get("arguments")) + tc["arguments"] = arguments + if json_block is None: + tc.pop("_xtml_json_block", None) + else: + tc["_xtml_json_block"] = json_block + normalized_calls.append(tc) + + normalized["tool_calls"] = normalized_calls + return normalized + + +def normalize_conversation(conversation: Any) -> Any: + if not isinstance(conversation, list): + return conversation + + def normalize_messages(messages: list[Any]) -> list[Any]: + return [normalize_message(message) for message in messages] + + if conversation and isinstance(conversation[0], list): + return [normalize_messages(messages) for messages in conversation] + return normalize_messages(conversation) + + +def _tool_call_id_index(tool_calls: Any) -> dict: + """Map assistant ``tool_calls[].id`` to ``(1-based position, function name)``. + + The position mirrors the chat template's enumeration over ``tool_calls`` + (every entry advances the position, even an id-less one). Duplicate ids keep + their first occurrence. + """ + index: dict = {} + if not isinstance(tool_calls, list): + return index + for position, tool_call in enumerate(tool_calls, start=1): + if not isinstance(tool_call, dict): + continue + call_id = tool_call.get("id") + if call_id is None: + continue + key = str(call_id) + if key in index: + continue + function = tool_call.get("function") + name = ( + function.get("name") if isinstance(function, dict) else tool_call.get("name") + ) + index[key] = (position, name) + return index + + +def normalize_xtml_tool_result_messages(messages: list[Any]) -> list[Any]: + """Re-sort K3 XTML tool results into assistant ``tool_calls`` order. + + Serving frameworks generally deliver tool results already in call order. A + direct Transformers caller, however, may pass OpenAI-style tool messages in any + order, so each run of consecutive tool messages is matched against the most + recent preceding assistant ``tool_calls`` by opaque ``tool_call_id`` == + ``tool_calls[].id`` (K3 drops the ``func:index`` format requirement) and + sorted by the matched 1-based position. The matched call is authoritative, + so each matched message's ``tool`` is set to that call's function name -- + this keeps an explicit (and possibly stale) ``tool``/``name`` from drifting + out of sync with the reordered position. ``index`` is still derived from the + rendered position by the chat template. A run that cannot be fully matched is + left untouched. Re-running is idempotent. + + This function is side-effect free: matched tool messages are shallow-copied + before their ``tool``/``name`` is rewritten, and every other message is + appended to the output as-is. The input list and its message objects are + never mutated. + """ + if not isinstance(messages, list): + return messages + + output: list[Any] = [] + current_index: dict = {} + i = 0 + n = len(messages) + + while i < n: + message = messages[i] + + if isinstance(message, dict) and message.get("role") == "assistant": + tool_calls = message.get("tool_calls") + current_index = _tool_call_id_index(tool_calls) if tool_calls else {} + output.append(message) + i += 1 + continue + + if not isinstance(message, dict) or message.get("role") != "tool": + output.append(message) + i += 1 + continue + + run: list[tuple] = [] # (position, original_offset, message, name) + unresolved = False + offset = 0 + while ( + i < n and isinstance(messages[i], dict) and messages[i].get("role") == "tool" + ): + tool_message = messages[i] + call_id = tool_message.get("tool_call_id", tool_message.get("id")) + matched = current_index.get(str(call_id)) if call_id is not None else None + if matched is None: + unresolved = True + run.append((None, offset, tool_message, None)) + else: + position, name = matched + run.append((position, offset, tool_message, name)) + offset += 1 + i += 1 + + if unresolved: + output.extend(item[2] for item in run) + else: + run.sort(key=lambda item: (item[0], item[1])) + for _, _, tool_message, name in run: + if name is None: + output.append(tool_message) + continue + # The id-matched call is authoritative: align tool (and any + # explicit name) so the rendered XTML tool attribute cannot + # disagree with the reordered position. Copy first so the + # caller's message object is never mutated. + resolved = dict(tool_message) + resolved["tool"] = name + if "name" in resolved: + resolved["name"] = name + output.append(resolved) + + return output + + +def is_batched_conversation(conversation: Any) -> bool: + return ( + isinstance(conversation, list) + and bool(conversation) + and isinstance(conversation[0], list) + ) + + +def _render_content_segments( + content: Any, + image_state: _ImagePromptState, +) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + if isinstance(content, str): + _append_text(segments, content, image_state) + elif content is not None: + for part in content: + if part["type"] in ["image", "image_url"]: + segments.extend( + _segment(image_state.next_prompt(), allow_special=True)) + else: + _append_text(segments, part["text"], image_state) + return segments + + +def _internal_system_message(message_type: str, body: str) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + segments.extend(_open_tag("message", [("role", "system"), ("type", message_type)])) + segments.extend(_text(body.strip())) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + return segments + + +def _render_assistant_segments( + message: dict[str, Any], + image_state: _ImagePromptState, + thinking: bool = True, +) -> list[EncodeSegment]: + segments: list[EncodeSegment] = [] + # The channel is structural: in thinking mode every assistant + # message carries the open/close tags even when there is no reasoning + # content to fill in. In non-thinking mode the channel is dropped + # entirely. + if thinking: + reasoning_content = message.get("reasoning_content") or message.get( + "reasoning" + ) + segments.extend(_open_tag("think")) + if reasoning_content is not None and str(reasoning_content).strip(): + _append_text(segments, reasoning_content, image_state) + segments.extend(_close_tag("think")) + + segments.extend(_open_tag("response")) + segments.extend(_render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("response")) + + tool_calls = message.get("tool_calls") + if tool_calls: + segments.extend(_open_tag("tools")) + for index, tool_call in enumerate(tool_calls, start=1): + fn = tool_call.get("function", tool_call) + segments.extend( + _open_tag("call", [("tool", fn["name"]), ("index", index)]) + ) + args = fn.get("arguments", {}) + json_block = fn.get("_xtml_json_block") + if json_block is not None: + segments.extend(_open_tag("json", [("type", "object")])) + _append_text(segments, json_block, image_state) + segments.extend(_close_tag("json")) + elif _is_mapping(args): + for key, value in args.items(): + segments.extend( + _open_tag( + "argument", + [("key", key), ("type", _xtml_type(value))], + ) + ) + _append_text(segments, _xtml_value(value), image_state) + segments.extend(_close_tag("argument")) + segments.extend(_close_tag("call")) + segments.extend(_close_tag("tools")) + + return segments + + +def _render_tool_declare(tools: Any, *, dynamic: bool = False) -> list[EncodeSegment]: + if dynamic: + body = ( + "## New Tools Available\n" + "The system dynamically extends the toolset via lazy-loading.\n" + "You have access to all existing and extended tools.\n" + "Here are the specs for the extended tools.\n\n" + "```json\n" + f"{_json_compact(tools)}\n" + "```" + ) + else: + body = ( + "# Tools\n" + "Here are the available tools, described in JSONSchema.\n\n" + "```json\n" + f"{_json_compact(tools)}\n" + "```" + ) + segments: list[EncodeSegment] = [] + segments.extend(_open_tag("message", [("role", "system"), ("type", "tool-declare")])) + segments.extend(_text(body)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + return segments + + +def build_chat_segments( + messages: list[Any], + tools: Optional[list[dict]] = None, + *, + add_generation_prompt: bool = True, + thinking: bool = True, + image_prompts: Optional[list[str]] = None, + **kwargs: Any, +) -> list[EncodeSegment]: + # Re-sort tool results by tool_call_id at the lowest layer so every caller + # (processor or direct tokenizer) gets correctly ordered XTML. The helper is + # side-effect free, so the caller's message objects are left untouched. + messages = normalize_xtml_tool_result_messages(messages) + messages = normalize_conversation(messages) + tools = deep_sort_dict(tools) + + kwargs = dict(kwargs) + response_format = kwargs.get("response_format") + if "response_schema" not in kwargs: + response_schema = extract_response_schema(response_format) + if response_schema is not None: + kwargs["response_schema"] = response_schema + if kwargs.get("response_schema") is not None: + kwargs["response_schema"] = deep_sort_dict(kwargs["response_schema"]) + + image_state = _ImagePromptState(image_prompts) + segments: list[EncodeSegment] = [] + + tool_calls = None + tool_index = 0 + + if tools: + segments.extend(_render_tool_declare(tools)) + + thinking_effort = kwargs.get("thinking_effort") + if thinking and thinking_effort is not None: + assert thinking_effort in _VALID_THINKING_EFFORTS, ( + f"Unsupported thinking_effort={thinking_effort!r}; " + f"supported values are {sorted(_VALID_THINKING_EFFORTS)}." + ) + if thinking and thinking_effort in _VALID_THINKING_EFFORTS: + segments.extend( + _internal_system_message( + "thinking-effort", + "`thinking_effort` guides on how much to think in your " + "thinking channel (not including the response channel), " + "supported values include `low`, `medium`, `high`, and `max`.\n" + f"Now the system is invoked with `thinking_effort={thinking_effort}`.", + ) + ) + + for message_index, message in enumerate(messages): + if not isinstance(message, dict): + continue + + role = message["role"] + if role == "user": + attrs = [("role", "user")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend(_render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "system" and message.get("tools"): + segments.extend(_render_tool_declare(message["tools"], dynamic=True)) + elif role == "system": + attrs = [("role", "system")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend(_render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "tool": + tool_index += 1 + tool_name = message.get("tool", message.get("name")) + if ( + tool_name is None + and tool_calls is not None + and tool_index <= len(tool_calls) + ): + tc = tool_calls[tool_index - 1] + fn = tc.get("function", tc) + tool_name = fn["name"] + if tool_name is None: + raise ValueError( + "Kimi K3 tool messages need a resolvable tool name: " + "carry `tool`/`name`, or match a preceding assistant " + "tool_call by order." + ) + segments.extend( + _open_tag( + "message", + [("role", "tool"), ("tool", tool_name), ("index", tool_index)], + ) + ) + segments.extend(_render_content_segments(message.get("content"), image_state)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + elif role == "assistant": + tool_calls = message.get("tool_calls") + tool_index = 0 + attrs = [("role", "assistant")] + if message.get("name"): + attrs.append(("name", message["name"])) + segments.extend(_open_tag("message", attrs)) + segments.extend(_render_assistant_segments(message, image_state, thinking)) + segments.extend(_close_tag("message")) + segments.extend(_end_of_msg()) + + tool_choice = kwargs.get("tool_choice") + if tool_choice == "required": + segments.extend( + _internal_system_message( + "tool-choice", + "The system is invoked with `tool_choice=required`.\n" + "You MUST call tools in the next message.", + ) + ) + elif tool_choice == "none": + segments.extend( + _internal_system_message( + "tool-choice", + "The system is invoked with `tool_choice=none`.\n" + "You MUST NOT call any tools in the next message.", + ) + ) + + rf = kwargs.get("response_format") + rf_type = _get_value(rf, "type", rf) if isinstance(rf, dict) else rf + if rf_type == "json_object": + segments.extend( + _internal_system_message( + "response-format", + "The system is invoked with `response_format=json_object`.\n" + "Your response must be raw JSON data without markdown code " + "blocks (```json) or any additional formatting.", + ) + ) + elif rf_type == "json_schema": + schema = _json_compact(kwargs.get("response_schema")) + segments.extend( + _internal_system_message( + "response-format", + "The system is invoked with `response_format=json_schema`.\n" + "Your response must be raw JSON data without markdown code " + "blocks (```json) or any additional formatting.\n" + "The JSON data must match the following schema:\n" + f"```json\n{schema}\n```", + ) + ) + + if add_generation_prompt: + segments.extend(_open_tag("message", [("role", "assistant")])) + segments.extend(_open_tag("think" if thinking else "response")) + + image_state.assert_consumed() + return segments diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3908a4a5477122aa4b09386acab760fb7887c0fc --- /dev/null +++ b/generation_config.json @@ -0,0 +1,4 @@ +{ + "max_length": 1048576, + "eos_token_id": 163586 +} \ No newline at end of file diff --git a/kimi_k3_processor.py b/kimi_k3_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..033a5773885c144509788e89a445217b76d6c046 --- /dev/null +++ b/kimi_k3_processor.py @@ -0,0 +1,187 @@ +"""Kimi-K3 processor: wraps vision processor + tokenizer into a single interface. + +Chat rendering (including XTML tool-result ordering) is handled by the +tokenizer's Python encoder; this processor adds multimodal media preprocessing. +""" + +from transformers.feature_extraction_utils import BatchFeature +from transformers.processing_utils import ProcessorMixin +from transformers.utils import logging + +from .media_utils import ensure_media_type + +logger = logging.get_logger(__name__) + +# ── KimiK3Processor ─────────────────────────────────────────────────── + + +class KimiK3Processor(ProcessorMixin): + r""" + Constructs a KimiK3 processor which wraps a KimiK3 image processor + and a tokenizer into a single processor. + + [`KimiK3Processor`] offers all the functionalities of + [`KimiK3VisionProcessor`] and [`TikTokenTokenizer`]. + + Args: + image_processor ([`KimiK3VisionProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`TikTokenTokenizer`], *optional*): + The tokenizer is a required input. + chat_template (`str`, *optional*): Kept for ProcessorMixin + compatibility. Kimi K3 chat encoding is implemented in Python by + the tokenizer. + """ + + attributes = ["image_processor", "tokenizer"] + valid_kwargs = ["chat_template"] + image_processor_class = "AutoImageProcessor" + tokenizer_class = "AutoTokenizer" + + def __init__( + self, + image_processor=None, + tokenizer=None, + chat_template=None, + **kwargs, + ): + super().__init__(image_processor, + tokenizer, + chat_template=chat_template) + self.media_processor = image_processor + self.image_placeholder = "<|kimi_image_placeholder|>" + + # ── Media preprocessing ──────────────────────────────────────────── + + def update_raw_text(self, text: str, image_prompts: list[str]) -> str: + # Replace image placeholders + image_count = text.count(self.image_placeholder) + if image_count > 0: + assert image_count == len(image_prompts), ( + f"image placeholder count {image_count} != " + f"image_prompts count {len(image_prompts)}") + text_parts = text.split(self.image_placeholder) + assert len(text_parts) == len(image_prompts) + 1 + text = "".join([ + text_parts[i] + image_prompts[i] + for i in range(len(image_prompts)) + ]) + text += text_parts[-1] + + return text + + def preprocess_medias(self, + medias: list[dict]) -> tuple[list[dict], list[str]]: + """Process media items and generate corresponding prompts. + + Returns: + A tuple of (updated_medias, image_prompts). + """ + updated_medias = [] + image_prompts = [] + for media in medias: + if media['type'] == 'image': + updated_medias.append(media) + img = ensure_media_type( + media, + transparent_bg_config=self.media_processor. + _transparent_bg_config, + transparent_bg_fill_stage=self.media_processor. + _transparent_bg_fill_stage, + )['image'] + w, h = img.size + image_prompts.append( + self.media_processor.make_image_prompt(w, h)) + else: + raise ValueError(f"unsupported media type: {media['type']}") + return updated_medias, image_prompts + + # ── Main entry points ────────────────────────────────────────────── + + def __call__(self, + messages: list[dict] = None, + medias: list[dict] = None, + text: str = None, + return_tensors: str = "pt", + **kwargs) -> BatchFeature: + """ + Process multimodal inputs for Kimi-K3 model. + + Args: + messages: List of message dicts with 'role' and 'content' fields. + If provided, medias and text will be extracted automatically. + medias: Pre-extracted list of media dicts. + text: Pre-formatted text string. + return_tensors: Format of returned tensors. Default: 'pt'. + **kwargs: Additional arguments passed to apply_chat_template. + + Returns: + BatchFeature with fields: input_ids, attention_mask, + pixel_values, grid_thws. + """ + if messages is None and (medias is None or text is None): + raise ValueError( + "Provide either 'messages' or both 'medias' and 'text'") + + if medias is not None and text is not None: + updated_medias, image_prompts = (self.preprocess_medias(medias)) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + text = self.update_raw_text(text, image_prompts) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + if medias is None: + medias = self._extract_medias_from_messages(messages) + updated_medias, image_prompts = (self.preprocess_medias(medias)) + preprocessed = self.media_processor.preprocess( + updated_medias, return_tensors=return_tensors) + + if text is None: + text_inputs = self.tokenizer.apply_chat_template( + messages, + tokenize=True, + return_tensors=return_tensors, + return_dict=True, + image_prompts=image_prompts, + **kwargs) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + text = self.update_raw_text(text, image_prompts) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + return BatchFeature(data={**text_inputs, **preprocessed.data}) + + @staticmethod + def _extract_medias_from_messages(messages: list[dict]) -> list[dict]: + """Extract media items from messages in a single pass.""" + medias = [] + for msg in messages: + if msg['role'] != 'user' or not msg.get('content'): + continue + + for content_part in msg['content']: + if not isinstance(content_part, dict): + continue + + content_type = content_part.get('type') + if content_type in ['image_url', 'image']: + image_data = content_part.get(content_type) + assert image_data is not None, f"image data is missing for content part: {content_part}" + medias.append({ + 'type': 'image', + 'image': image_data, + }) + return medias + + def apply_chat_template(self, messages, **kwargs): + return self.tokenizer.apply_chat_template(messages, **kwargs) + + def batch_decode(self, *args, **kwargs): + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + return ['input_ids', 'attention_mask', 'pixel_values', 'grid_thws'] diff --git a/kimi_k3_vision_processing.py b/kimi_k3_vision_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..58154424ff951289570c8fb5578e7372bfbfaa8b --- /dev/null +++ b/kimi_k3_vision_processing.py @@ -0,0 +1,179 @@ +"""Image processor class for Kimi-K3. +""" + +import json +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +from PIL import Image +from transformers.image_processing_utils import (BaseImageProcessor, + BatchFeature) +from transformers.utils import TensorType + +from .media_utils import (MediaInput, TransparentBgConfig, _to_tensor, + ensure_media_type, image_to_np, navit_patchify, + navit_resize_image, normalize) + + +class KimiK3VisionProcessor(BaseImageProcessor): + model_type = "kimi_k3" + + def __init__( + self, + media_proc_cfg: dict, + **kwargs, + ): + super().__init__(**kwargs) + self.media_proc_cfg = media_proc_cfg + + @property + def _transparent_bg_config(self) -> Optional[TransparentBgConfig]: + cfg = self.media_proc_cfg.get("transparent_bg_config") + if cfg is None: + return None + if isinstance(cfg, TransparentBgConfig): + return cfg + return TransparentBgConfig(**cfg) + + @property + def _transparent_bg_fill_stage(self) -> str: + return self.media_proc_cfg.get("transparent_bg_fill_stage", + "before_resize") + + def media_tokens_calculator(self, media: MediaInput): + media = ensure_media_type( + media, + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + ret = self.get_resize_config(media) + return ret['num_tokens'] + + @classmethod + def make_image_prompt(cls, width: int, height: int) -> str: + """Build the K3 image placeholder with resolution info.""" + return (f"<|media_begin|>image {width}x{height}" + f"<|media_content|><|media_pad|><|media_end|>") + + def get_resize_config(self, media_input: MediaInput) -> dict: + if media_input['type'] == 'image': + w, h = media_input['image'].size + ret = navit_resize_image( + w, h, self.media_proc_cfg['patch_size'], + self.media_proc_cfg['merge_kernel_size'], + self.media_proc_cfg['in_patch_limit'], + self.media_proc_cfg['patch_limit_on_one_side'], + self.media_proc_cfg['fixed_output_tokens']) + return ret + else: + raise ValueError("Unsupported type: {}".format( + media_input['type'])) + + def resize_image(self, image: Image.Image, new_width: int, new_height: int, + pad_width: int, pad_height: int) -> np.ndarray: + image_np = image_to_np( + image, + (new_width, new_height), + "resize", + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + image_np = np.pad( + image_np, + ((0, pad_height), (0, pad_width), (0, 0)), + mode="constant", + constant_values=0, + ) + return image_np + + def preprocess( + self, + medias: list[MediaInput], + return_tensors: Optional[Union[str, TensorType]] = None, + ) -> BatchFeature: + """ + Preprocess a atom vision input (images) into model-ready tensors. + + Args: + medias: List of MediaInput. + return_tensors: Desired output format ('pt', 'np', 'tf', or None). + + Returns: + BatchFeature containing 'pixel_values' and 'grid_thws' tensors. + """ + if not isinstance(medias, list): + medias = [medias] + if medias: + pixel_values = [] + for item in medias: + item = ensure_media_type( + item, + transparent_bg_config=self._transparent_bg_config, + transparent_bg_fill_stage=self._transparent_bg_fill_stage, + ) + resize_config = self.get_resize_config(item) + new_width, new_height, pad_width, pad_height = resize_config[ + 'new_width'], resize_config['new_height'], resize_config[ + 'pad_width'], resize_config['pad_height'] + if item['type'] == 'image': + image = item['image'] + image_np = self.resize_image(image, new_width, new_height, + pad_width, pad_height) + pixel_values.append(np.expand_dims(image_np, axis=0)) + else: + raise ValueError("Unsupported type: {}".format( + item['type'])) + normalized_pixel_values = [] + image_std_inv = 1.0 / np.array(self.media_proc_cfg['image_std']) + image_mean = np.array(self.media_proc_cfg['image_mean']) + for pixels in pixel_values: + pixels = normalize(pixels, image_mean, image_std_inv) + pixels_and_thw = navit_patchify( + pixels, + self.media_proc_cfg['patch_size'], + ) + normalized_pixel_values.append(pixels_and_thw) + + pixel_values = torch.cat([ + _to_tensor(pixel_value['pixel_values']) + for pixel_value in normalized_pixel_values + ]) + grid_thws = torch.cat([ + _to_tensor(pixel_value['grid_thw'], + dtype=torch.int64).unsqueeze(0) + for pixel_value in normalized_pixel_values + ]) + + data = { + 'pixel_values': pixel_values, + 'grid_thws': grid_thws, + } + + else: + data = {} + + return BatchFeature(data=data, tensor_type=return_tensors) + + def __repr__(self): + return f"KimiK3VisionProcessor(media_proc_cfg={self.media_proc_cfg})" + + def to_dict(self) -> Dict[str, Any]: + output = super().to_dict() + output["media_proc_cfg"] = self.media_proc_cfg + if "media_processor" in output: + del output["media_processor"] + return output + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any], **kwargs): + config = config_dict.copy() + media_proc_cfg = config.pop("media_proc_cfg", {}) + return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs) + + def to_json_string(self): + dictionary = self.to_dict() + for key, value in dictionary.items(): + if hasattr(value, 'tolist'): + dictionary[key] = value.tolist() + return json.dumps(dictionary, indent=2, sort_keys=True) + "\n" diff --git a/media_utils.py b/media_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b2ae127e254e364e6b568b1093a7e12ac576e0 --- /dev/null +++ b/media_utils.py @@ -0,0 +1,376 @@ +import base64 +import functools +import io +import math +from dataclasses import dataclass +from typing import Literal, TypedDict + +import numpy as np +from PIL import Image + + +class ImageInput(TypedDict): + type: Literal['image'] + image: Image.Image + + +MediaInput = ImageInput + + +@dataclass +class TransparentBgConfig: + """The config of the transparent background.""" + + pattern: Literal["white", "black", "gray", "chessboard"] = "black" + """The pattern of the transparent background.""" + + chessboard_square_size: int = 16 + """The size of the squares in the chessboard background.""" + + chessboard_square_on_top_left: bool = True + """Whether to start the chessboard with a white square on the top left.""" + + chessboard_white_value: int = 255 + """The value of the white pixels in the background.""" + + chessboard_gray_value: int = 200 + """The value of the gray pixels in the background.""" + + +@functools.lru_cache(maxsize=256) +def _create_chessboard_background( + height: int, + width: int, + square_size: int, + square_on_top_left: bool, + white_value: int, + gray_value: int, +) -> np.ndarray: + """Create a chessboard background.""" + bg = np.ones((height, width, 3), dtype=np.uint8) * white_value + for y in range(0, height, square_size): + for x in range(0, width, square_size): + if (y // square_size + x // square_size) % 2 == ( + 1 if square_on_top_left else 0): + bg[y:y + square_size, x:x + square_size] = gray_value + return bg + + +def fill_transparent_bg_with( + image: Image.Image, + transparent_bg_config: TransparentBgConfig | None = None, +) -> Image.Image: + """Composite a (possibly) transparent image onto a configured background. + + When ``transparent_bg_config`` is ``None``, the image is simply converted + to RGB (preserving the historical behavior). Otherwise the alpha channel + is alpha-composited over a background generated according to the config. + """ + if transparent_bg_config is None: + return image.convert("RGB") + + if image.mode == "RGB": + return image + + has_alpha = "A" in image.getbands() or "transparency" in image.info + if not has_alpha: + return image.convert("RGB") + + img = np.array(image.convert("RGBA")) + height, width = img.shape[:2] + bg_pattern = transparent_bg_config.pattern + if bg_pattern == "white": + bg = np.full((height, width, 3), 255, dtype=np.uint8) + elif bg_pattern == "black": + bg = np.zeros((height, width, 3), dtype=np.uint8) + elif bg_pattern == "gray": + bg = np.full((height, width, 3), 128, dtype=np.uint8) + elif bg_pattern == "chessboard": + bg = _create_chessboard_background( + height, + width, + transparent_bg_config.chessboard_square_size, + transparent_bg_config.chessboard_square_on_top_left, + transparent_bg_config.chessboard_white_value, + transparent_bg_config.chessboard_gray_value, + ) + else: + raise ValueError(f"Invalid background pattern: {bg_pattern}") + + alpha = img[:, :, 3] + img_rgb = img[:, :, :3] + alpha_normalized = alpha.astype(np.float32) / 255.0 + alpha_3d = np.stack([alpha_normalized] * 3, axis=2) + result = alpha_3d * img_rgb + (1 - alpha_3d) * bg + result = result.astype(np.uint8) + return Image.fromarray(result) + + +def navit_resize_image( + width: int, + height: int, + patch_size: int, + merge_kernel_size: int, + in_patch_limit: int, + patch_limit_on_one_side: int, + fixed_output_tokens: int | None, +): + # Apply the patch limits. + s1 = math.sqrt( + in_patch_limit / + (max(1.0, width // patch_size) * max(1.0, height // patch_size))) + s2 = patch_limit_on_one_side * patch_size / width + s3 = patch_limit_on_one_side * patch_size / height + scale = min(1.0, s1, s2, s3) + new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale)) + new_w = min(new_w, patch_limit_on_one_side * patch_size) + new_h = min(new_h, patch_limit_on_one_side * patch_size) + + # Calculate the padding to make the height and width divisible by the merge kernel size and patch size. + factor = merge_kernel_size * patch_size + + pad_height = (factor - new_h % factor) % factor + pad_width = (factor - new_w % factor) % factor + + if fixed_output_tokens is not None: + num_tokens = fixed_output_tokens + else: + # Calculate new dimensions after padding and patching + token_height = (new_h + pad_height) // factor + token_width = (new_w + pad_width) // factor + + assert token_height * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_height {token_height} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + assert token_width * merge_kernel_size <= patch_limit_on_one_side, ( + f"token_width {token_width} * merge_kernel_size {merge_kernel_size} > patch_limit_on_one_side {patch_limit_on_one_side}" + ) + + num_tokens = token_height * token_width + return { + "num_tokens": num_tokens, + "new_width": new_w, + "new_height": new_h, + "pad_width": pad_width, + "pad_height": pad_height, + "sampled_nframes": 1, + } + + +def _to_pil( + data: str | bytes | Image.Image, + transparent_bg_config: TransparentBgConfig | None = None, + to_rgb: bool = True, +) -> Image.Image: + """Load an image and (optionally) composite its transparent background. + + Args: + data: A PIL Image, a base64 ``data:`` URL, a file path, or raw bytes. + transparent_bg_config: The config used to fill the transparent + background. ``None`` keeps the historical behavior of converting + to RGB without compositing. + to_rgb: If ``False`` the image is returned as-is (the + ``transparent_bg_config`` is ignored). The caller is then + expected to call :func:`fill_transparent_bg_with` later — e.g. + after a resize. + """ + if isinstance(data, Image.Image): + image = data + elif isinstance(data, str): + if data.startswith("data:"): + raw_base64 = data.split(",")[1] + image = Image.open(io.BytesIO(base64.b64decode(raw_base64))) + else: + image = Image.open(data) + elif isinstance(data, bytes): + image = Image.open(io.BytesIO(data)) + else: + raise ValueError(f"Unsupported data type: {type(data)}") + + if not to_rgb: + return image + + return fill_transparent_bg_with(image, transparent_bg_config) + + +def ensure_media_type( + media: MediaInput, + transparent_bg_config: TransparentBgConfig | None = None, + transparent_bg_fill_stage: Literal["before_resize", + "after_resize"] = "before_resize", +) -> MediaInput: + if media['type'] == 'image': + media['image'] = _to_pil( + media['image'], + transparent_bg_config=transparent_bg_config, + to_rgb=transparent_bg_fill_stage == "before_resize", + ) + return media + else: + raise ValueError(f"Unsupported media type: {media['type']}") + + +def image_to_np( + image: Image.Image, + resize_to: tuple[int, int] | None = None, + mode: str = "resize", + raise_error_for_ill_resize: bool = True, + transparent_bg_config: TransparentBgConfig | None = None, + transparent_bg_fill_stage: Literal["before_resize", + "after_resize"] = "before_resize", +) -> np.ndarray: + """Convert an image to a numpy array. + + Args: + content: The image to convert. + resize_to: The size to resize the image to. + mode: The mode to resize the image to. + raise_error_for_ill_resize: Whether to raise an error for ill-sized resize. + transparent_bg_config: The config of the transparent background. Only + used when ``transparent_bg_fill_stage == "after_resize"`` (the + caller is responsible for filling before resize otherwise). + transparent_bg_fill_stage: When to composite the transparent + background — before or after the resize step. + + Returns: + A numpy array. + """ + assert isinstance(image, Image.Image), "image must be a PIL Image" + if resize_to is not None: + if mode == "resize": + image = image.resize(resize_to, resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + + elif mode == "rescale_and_pad_to_center": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + padding_left = (resize_to[0] - new_width) // 2 + padding_right = resize_to[0] - new_width - padding_left + padding_top = (resize_to[1] - new_height) // 2 + padding_bottom = resize_to[1] - new_height - padding_top + image = np.asarray(image) + image = np.pad( + image, + ((padding_top, padding_bottom), (padding_left, padding_right), + (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + elif mode == "rescale_and_pad_to_rightbottom": + scale = min(resize_to[0] / image.width, + resize_to[1] / image.height, 1.0) + new_width = round(image.width * scale) + new_height = round(image.height * scale) + if new_width == 0 or new_height == 0: + if raise_error_for_ill_resize: + raise ValueError( + f"Invalid resize to: {resize_to}, from image size: {image.size}" + ) + else: + return np.zeros((resize_to[1], resize_to[0], 3), + dtype=np.uint8) + + image = image.resize((new_width, new_height), + resample=Image.Resampling.BICUBIC) + if transparent_bg_fill_stage == "after_resize": + image = fill_transparent_bg_with(image, transparent_bg_config) + padding_right = resize_to[0] - new_width + padding_bottom = resize_to[1] - new_height + image = np.asarray(image) + image = np.pad( + image, + ((0, padding_bottom), (0, padding_right), (0, 0)), + mode="constant", + constant_values=0, + ) + assert image.shape == (resize_to[1], resize_to[0], 3) + + else: + raise ValueError(f"Invalid mode: {mode}") + + if isinstance(image, Image.Image): + return np.asarray(image) + else: + return image + + +def navit_patchify(pixel_values: np.ndarray, + patch_size: int) -> dict[str, np.ndarray]: + """Reshape the pixel values to a navit shape. + + Args: + pixel_values: np.ndarray, shape (t, h, w, c) + patch_size: int + + Returns: + dict[str, np.ndarray] + - patches: np.ndarray, shape (t * h//patch_size * w//patch_size, c, patch_size, patch_size) + - grid_thw: np.ndarray, (t, h//patch_size, w//patch_size) + """ + T, H, W, C = pixel_values.shape + assert C == 3, "pixel_values must have 3 channels" + + patches = pixel_values.reshape(T, H // patch_size, patch_size, + W // patch_size, patch_size, C) + # (T, H//patch_size, W//patch_size, C, patch_size, patch_size) + patches = patches.transpose(0, 1, 3, 5, 2, 4) + patches = patches.reshape(-1, C, patch_size, patch_size) + grid_thw = np.array([T, H // patch_size, W // patch_size]) + return {"pixel_values": patches, "grid_thw": grid_thw} + + +def normalize(x: np.ndarray, + mean, + std_inv, + pixels_dtype: np.dtype = np.float32) -> np.ndarray: + """Normalize the image. + + Args: + x: The image to normalize. The shape is (..., 3). The dtype is uint8. The range is [0, 255]. + mean: The mean of the image. + std_inv: The inverse of the std of the image. + pixels_dtype: The dtype of the image. + Returns: + The normalized image. The shape is (..., 3). The dtype is determined by the pixels_dtype. + """ + x = (x / 255.0).astype(pixels_dtype) + x -= mean + x *= std_inv + return x + + +def _to_tensor(data, **kwargs): + import torch + + if isinstance(data, np.ndarray): + return torch.from_numpy(data).to(**kwargs) + elif isinstance(data, torch.Tensor): + return data.to(**kwargs) + elif isinstance(data, list): + return [_to_tensor(item, **kwargs) for item in data] + elif isinstance(data, tuple): + return tuple(_to_tensor(item, **kwargs) for item in data) + elif isinstance(data, dict): + return {k: _to_tensor(v, **kwargs) for k, v in data.items()} + elif data is None: + return None + else: + raise ValueError(f"Unsupported data type: {type(data)}") diff --git a/model-00001-of-00185.safetensors b/model-00001-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c2ed4c0fcfd574b48449f7ff3da6cbf1bcc8784 --- /dev/null +++ b/model-00001-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a60ccb44d2c70085717a0a0100fe2a950460ae95810a53e5a4324f057f9bd9 +size 4936231310 diff --git a/model-00002-of-00185.safetensors b/model-00002-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..28d84cc6b4d5193c671589e3a2d724b9ce308107 --- /dev/null +++ b/model-00002-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d80ecd05bd201eae321c4586090340d42ade4b9442668b02e063afb8e95d95 +size 5240783442 diff --git a/model-00003-of-00185.safetensors b/model-00003-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9dd9bc055c1f8994a8647578993973a4644a5bb3 --- /dev/null +++ b/model-00003-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:726818a99125faa1d487a620f2c305a73510d05b6520a1ac73412a34706a00f5 +size 3645759823 diff --git a/model-00004-of-00185.safetensors b/model-00004-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4ebe2dea16eb9520ac0030f8dc7fdc8c653a6e26 --- /dev/null +++ b/model-00004-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93fb29164cc8cbbc280a737a9f48e0f96f6ff01a476d9c952e59e088a9ee19e7 +size 5240783440 diff --git a/model-00005-of-00185.safetensors b/model-00005-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b4c7a7e8ea082c4e4ae2e17eb956906dda3de1fd --- /dev/null +++ b/model-00005-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfcbf3605e7342e25b905352424e15b10edf3f84712da07c59f48305d9670c09 +size 3473372860 diff --git a/model-00006-of-00185.safetensors b/model-00006-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b64d4347fd7c647f74501e3aeef470fd21ef8fa --- /dev/null +++ b/model-00006-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ee31b98a9f4cfe1613546d5765a1035a1df5153e6fb1ab193bb62b63cc28215 +size 5240783442 diff --git a/model-00007-of-00185.safetensors b/model-00007-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8948202d3b778e9e9d7b475ee5422bef2aa826b3 --- /dev/null +++ b/model-00007-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddba61cee4c9d21983a860c8307e1c0b7a1e4fb3ec99c20397b64ca7052a3f9 +size 3645759841 diff --git a/model-00008-of-00185.safetensors b/model-00008-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..49de37d53f0bca659b25264a58cc3f391d0294d8 --- /dev/null +++ b/model-00008-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c711e0a5e1f564ec80a3f4c04223a5854ac6748098bb1242fb3c1612b143cf +size 5240783442 diff --git a/model-00009-of-00185.safetensors b/model-00009-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c1acede2176fd25b22bdd605f354a682c445215b --- /dev/null +++ b/model-00009-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8a5cce236a8e74d99e1db3a7001183ab75d1fcc25254432b29294e55e6426c +size 3645759791 diff --git a/model-00010-of-00185.safetensors b/model-00010-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aea90a6005498856328a44a10e122170ca92c2e7 --- /dev/null +++ b/model-00010-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbab4be97bff48cac50ef176f410fb00e67dc6a6856bb2b7c791a12d270aaf09 +size 5240783442 diff --git a/model-00011-of-00185.safetensors b/model-00011-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..83ca66cfc633dce25ff5bedb7161e664d25f42d1 --- /dev/null +++ b/model-00011-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3531c5d8094cd2f4f47a08fd6fb9a7d5b0120b0cbb7d39dbd5261875c3e757 +size 3645759843 diff --git a/model-00012-of-00185.safetensors b/model-00012-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cae8cb01e7b9683dae28edce9f5a23bbc5713176 --- /dev/null +++ b/model-00012-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f0a9182d8ece493a78d0f24f2ba76c92bd63c5773e4a64f3c489894aee30db +size 5240783442 diff --git a/model-00013-of-00185.safetensors b/model-00013-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f827d61f30e96307b9604dda54dc5a5da668e8e --- /dev/null +++ b/model-00013-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd1fce85d0092395b468b1ea9317814b8a16ea484e1052e58178e121017d4d5 +size 3473372798 diff --git a/model-00014-of-00185.safetensors b/model-00014-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5aefcd69aedee330e460aaa61ccf685e77631c59 --- /dev/null +++ b/model-00014-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cfbeba263044db8170129b3f81b489e23541f6fcc89e59e4f7cec5dd82b1b7 +size 5240783442 diff --git a/model-00015-of-00185.safetensors b/model-00015-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fa9c40f74a9caca7c1f44130ae911d1f0020f9c --- /dev/null +++ b/model-00015-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddc5381e8674b0da7d51109b48e7407a9ebc2b6192421a0739cd579ac43ea865 +size 3645759831 diff --git a/model-00016-of-00185.safetensors b/model-00016-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..19d33818cb40667d748a4229204402775f894513 --- /dev/null +++ b/model-00016-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f912afd3d01adf32a7e7732a6b89ca22b63eccd9584b93115bae5bb180ff0109 +size 5240783442 diff --git a/model-00017-of-00185.safetensors b/model-00017-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb60a42ed96a68de80dfdbb3f1ec4b4e11f802c2 --- /dev/null +++ b/model-00017-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68fe5808be766e7b87bfb11ac8da91d1941b76e5996aaed7b6df2a17e5f65c01 +size 3645759841 diff --git a/model-00018-of-00185.safetensors b/model-00018-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5cbe533f775e313679954aa23fe4225f85d13488 --- /dev/null +++ b/model-00018-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b4a892622abfed3c92bab16461de495cfc5d51cbbf395a3cd35980b4b204245 +size 5240783442 diff --git a/model-00019-of-00185.safetensors b/model-00019-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b28f486557f08c7e466281865952c5d63304bf3f --- /dev/null +++ b/model-00019-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf62390fbc5fae3a9073ca8d50b35380721374f912da372c8ce0fde92caab870 +size 3645759871 diff --git a/model-00020-of-00185.safetensors b/model-00020-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9f16558898fd113cd4a8c3042ce46f248b94780 --- /dev/null +++ b/model-00020-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:555bd270de301a64e523758785b192c16a65d8d26e4d294592b755e2f18c36f9 +size 5240783444 diff --git a/model-00021-of-00185.safetensors b/model-00021-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a2a0e7c6b7508a00f941edf457f5d6a33a9477d6 --- /dev/null +++ b/model-00021-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a599037e73adcadaac5cff3719c8be06264b7c6aecf41a0c532a2e3b834c634 +size 3473372912 diff --git a/model-00022-of-00185.safetensors b/model-00022-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cb21138105085766436c835fc8ca101c0b5898f9 --- /dev/null +++ b/model-00022-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c245aec45b5047995b29c3bb8eb62e80a6654340cd5b52edb031999b5c4b4f +size 5240783446 diff --git a/model-00023-of-00185.safetensors b/model-00023-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e65cd669ca3a79fad11459ac4036302904b09afa --- /dev/null +++ b/model-00023-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf701db169d3522a2b40d5af86ed9c1c6208c8607697d7b0032467e66467852e +size 3645759894 diff --git a/model-00024-of-00185.safetensors b/model-00024-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..79aaaa517f86459348532d04f8278c7153ba83cd --- /dev/null +++ b/model-00024-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d67ce61bc274ffa289a3f07136bfb751a49747ed2c6cf47dbd624c407c5398 +size 5240783446 diff --git a/model-00025-of-00185.safetensors b/model-00025-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..94174a3f687da5d9c5bb0a7804c081c4bf4e62b5 --- /dev/null +++ b/model-00025-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ef21f857bba47bddef11522183a58ab8309f236ed711dd7d71b58eb749626b +size 3645759820 diff --git a/model-00026-of-00185.safetensors b/model-00026-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9deb9df335f66b7a9a93aff880ed2587d33d45d8 --- /dev/null +++ b/model-00026-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a1b0f53d75af4a74e24c5181cc92822a49a9166c1333d49d2725393d81d7510 +size 5240783446 diff --git a/model-00027-of-00185.safetensors b/model-00027-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c78bd580597b81961cac6abf5acd05fff8c8d2c7 --- /dev/null +++ b/model-00027-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7da5b618e2f5efc06033eedfef58943e1bf5b4271698b590f2c3129631321d7 +size 3645759816 diff --git a/model-00028-of-00185.safetensors b/model-00028-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..93845bba0a749ed58d62dcba170ac97fa2ac5021 --- /dev/null +++ b/model-00028-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4466b8b7b5beabfd21af28a387e0cf6b194cecd845dcdab9e323cfe8ae15802 +size 5240783442 diff --git a/model-00029-of-00185.safetensors b/model-00029-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0a1412fdb7fe34fd394d7b6026f89c879acd6fa0 --- /dev/null +++ b/model-00029-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76335537d6c7bc6d5e3c8ced7e7bdbee6a66bca0a241e3816f521bdfb402f85e +size 3473372910 diff --git a/model-00030-of-00185.safetensors b/model-00030-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7ed56fcede080317a20debde7ff0331b803ff647 --- /dev/null +++ b/model-00030-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:221364c03a3f92a55ad44180269993b6e423fd99403ce1d35d812c56d89203c7 +size 5240783446 diff --git a/model-00031-of-00185.safetensors b/model-00031-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97a41c599fdad97b1a76638e6ab8caaa416d3825 --- /dev/null +++ b/model-00031-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91eeff8ee44d5e6389b7b9fe730b1f880ae53e17b9d2d087d295460f44be07d5 +size 3645759808 diff --git a/model-00032-of-00185.safetensors b/model-00032-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d449647d9846dc066dab4d3abe9315f84667fe0e --- /dev/null +++ b/model-00032-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bddbb57aedc66efc1d63b79bd2d0c203f8086ee6215862ce63e5b5046d6e30f +size 5240783446 diff --git a/model-00033-of-00185.safetensors b/model-00033-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5296a89ba6d65c534d8ebae79bd04ea7cffde9d1 --- /dev/null +++ b/model-00033-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911bc1259e5672b7dc4d565f1ef5c9b9229019fa0044334113850727396a43cb +size 3645759898 diff --git a/model-00034-of-00185.safetensors b/model-00034-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c0dd0777912f00aa68cc79de9caae87113dcf846 --- /dev/null +++ b/model-00034-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e59bd8c1ffecdf3effba49609389b6cce2246b54464ee3f1ae481a046be16c6c +size 5240783446 diff --git a/model-00035-of-00185.safetensors b/model-00035-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2c7e1ebbf0776776aa08c78aa76f883a92b4a2ef --- /dev/null +++ b/model-00035-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b86f3b55bff7f005c2911e92bebe70a7e4c0cac3a076758f45f0131529bf5a +size 3645759894 diff --git a/model-00036-of-00185.safetensors b/model-00036-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ccfb24f0bcdaa097b03db94d9aa385b56405c34f --- /dev/null +++ b/model-00036-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e58c766c07e49851dab73a896fe530dc8dabac60d73b2ff637c7c7495cb85f9 +size 5240783446 diff --git a/model-00037-of-00185.safetensors b/model-00037-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef3f3ba9deeeb4828ae9e8b315ef90fd1e7b1b11 --- /dev/null +++ b/model-00037-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b79e949543ad129f22db48c59e68edd1b4260142051bac165f196d499e1a47 +size 3473372912 diff --git a/model-00038-of-00185.safetensors b/model-00038-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dcb85a18ea1cf3c5068f7223494359f732ef6372 --- /dev/null +++ b/model-00038-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef490d649a1e97523688ba123b3caa589c883a9df2614f325fde96cd25cdd360 +size 5240783446 diff --git a/model-00039-of-00185.safetensors b/model-00039-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea71ba9f26f513733b23884c857d267e7b1ae213 --- /dev/null +++ b/model-00039-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d40e9e62be243c7bddf5c37b5d4b379a490a5b88b1cc850dff7d554bf46394f +size 3645759814 diff --git a/model-00040-of-00185.safetensors b/model-00040-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac6ca0e4f864e49c02e3ad2ed8c0f4ab1099ed2b --- /dev/null +++ b/model-00040-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1daa5d84bc3d5b568509ac86c477c8f02b45bc7d7e4e305b0b034ca3288bacf9 +size 5240783446 diff --git a/model-00041-of-00185.safetensors b/model-00041-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b9214ef21ef356ab481ec82985c4da63fae113d5 --- /dev/null +++ b/model-00041-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aeac53f439fb92aee6a380def5513bee59653549b42a83baf4ee19fc7a1ffdd +size 3645759896 diff --git a/model-00042-of-00185.safetensors b/model-00042-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0e8c952f5f1f4d2ced5aeac69e195e25e1fdd0e --- /dev/null +++ b/model-00042-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3100af9d8e51bfb7bb268525309990d63b4c49df6640461d96f7b9321e32f78 +size 5240783446 diff --git a/model-00043-of-00185.safetensors b/model-00043-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08c812354bfb88b92fecdc8dccc1195061c39b21 --- /dev/null +++ b/model-00043-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95b08dfb2a0cca6a50a3ae71b09543d7219e2714a2be89e9dc00f6fe89293f32 +size 3645759896 diff --git a/model-00044-of-00185.safetensors b/model-00044-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9bbaeceab937dca8e6e025a6285e36269d6461a5 --- /dev/null +++ b/model-00044-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b487a16ff20a3c6c3bd3059de132ecf6ea909d3f590a44e1dfb3f5487bd4fc +size 5240783444 diff --git a/model-00045-of-00185.safetensors b/model-00045-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cbd9c76d0e33d3aa5f130faea590c9367b374b8b --- /dev/null +++ b/model-00045-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:543248e2b6077f13cfa30f34db8e8a7fce5f7af1c4618093374ad82934b548aa +size 3473372912 diff --git a/model-00046-of-00185.safetensors b/model-00046-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6eb01b75696a5ad3f9596ace790b45f92180a1b6 --- /dev/null +++ b/model-00046-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba6781df0d93a8fe457c1a29c03321e6eb5f39cf5e5f35a1f2302ce8dfb2637a +size 5240783446 diff --git a/model-00047-of-00185.safetensors b/model-00047-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b6413babdd371d73e879b3fb138d9f4b35e0ce8 --- /dev/null +++ b/model-00047-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28dc7c70be9ff93610b130ef6dc69edab77f771757694d578c7758859004136e +size 3645759784 diff --git a/model-00048-of-00185.safetensors b/model-00048-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f560e7e4c1444036ca469587d19a9fa219047225 --- /dev/null +++ b/model-00048-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0a6548035eb81b4885bf489e0e6d4836facd496fd24c183b024a95bc946cb0 +size 5240783446 diff --git a/model-00049-of-00185.safetensors b/model-00049-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7999fa4e2d7d13b777da21ad7edcc101b79c7fc7 --- /dev/null +++ b/model-00049-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0351b475bbbf7a7397892aedd7f8e6df6db3dba80d21c8631c37a52f6d028f2f +size 3645759884 diff --git a/model-00050-of-00185.safetensors b/model-00050-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..13441ccca45d1217bce46816193ad6358f50e308 --- /dev/null +++ b/model-00050-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8135c37692e82428bdd5a5e234216dc430c328ea3e75c51ddd9d57fc1beaf658 +size 5240783446 diff --git a/model-00051-of-00185.safetensors b/model-00051-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..55dffd2e6c5513aeab04bbde2394c9deeba32aaa --- /dev/null +++ b/model-00051-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:210b1e144eed91ef0b6189f511fac33b69b713ec8ab99f2a1ecae8b087ffd96f +size 3645759894 diff --git a/model-00052-of-00185.safetensors b/model-00052-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..17be28dcbc02d9ace5d77b59f352619525360e93 --- /dev/null +++ b/model-00052-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c98ac3817c67fd3470ba831e039a7c8d4c7b0126f67ce19a514088485be9b511 +size 5240783446 diff --git a/model-00053-of-00185.safetensors b/model-00053-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2d89e0d4f0c1f6d15805e5d9db2cc77f4a13dc5f --- /dev/null +++ b/model-00053-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae9f6136b320796378b4cc4efb4ac8e48564c410fc28c9f6e4b7bc273bda356 +size 3473372834 diff --git a/model-00054-of-00185.safetensors b/model-00054-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f5865cc8c5d871dd2036ed4632bdd70d410e5d00 --- /dev/null +++ b/model-00054-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f30afcc5d9df80d024b2a26a76c10873753c2aae0b99c7d3c36310d87a6598 +size 5240783446 diff --git a/model-00055-of-00185.safetensors b/model-00055-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b271f665925d77681e067d96e3b766a90c668b1c --- /dev/null +++ b/model-00055-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b50e4b6286c7935922136f7657565201b7a35033eff89e0b0c568eae49b53a1 +size 3645759824 diff --git a/model-00056-of-00185.safetensors b/model-00056-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..016ab01605792cb1bc2a76e0472f4d7ef1af3470 --- /dev/null +++ b/model-00056-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fec7bb231d707b0aea1148b8ae01f0c863a15bb55c7e3d68c12f357952a486af +size 5240783446 diff --git a/model-00057-of-00185.safetensors b/model-00057-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..746743bc78788c67e9ae84ac3bebcaed75ae752c --- /dev/null +++ b/model-00057-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1604a18e5ec43370734c697d45ac62d0e7c1df1f268344e22747b3b11438c12 +size 3645759858 diff --git a/model-00058-of-00185.safetensors b/model-00058-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..caaaf23fc2de450d86eb901c041d83dd94ce3c78 --- /dev/null +++ b/model-00058-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4938186383d891c0d7ef60e6698ed9d10d25957056e4551260e3716e5a98b73e +size 5240783446 diff --git a/model-00059-of-00185.safetensors b/model-00059-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8fb87433060d9024fb36e4ec699587d81622c661 --- /dev/null +++ b/model-00059-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1c42256a794bb4800d3e595ce1e6fb88a264528f749ef56e143a765908b0a8a +size 3645759860 diff --git a/model-00060-of-00185.safetensors b/model-00060-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fed81312c451bb4c2aa5927e18354b6e6887b1e3 --- /dev/null +++ b/model-00060-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dda6e3af2e0a073544807ea481e4d7bf2efb6aebca4b5b75e41a04a07f41dc0 +size 5240783444 diff --git a/model-00061-of-00185.safetensors b/model-00061-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..070ab0c4473b31dd431ac8425bea77a84819d764 --- /dev/null +++ b/model-00061-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5c80472454ce07064b1f921cb1d464dfdbd9064d0d016a1264f8ebbbf98808 +size 3473372882 diff --git a/model-00062-of-00185.safetensors b/model-00062-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1192f77ac853a637246b3956c47bf852556cb39a --- /dev/null +++ b/model-00062-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ff643cf794ed867c724052e191ac457995bc13ec049f9b75f5ee44a4bf25992 +size 5240783446 diff --git a/model-00063-of-00185.safetensors b/model-00063-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..576535e87382ad35d8a712c53aa541e7a082231f --- /dev/null +++ b/model-00063-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1c15adabba7b8319e86a182307261ce9c92bee46415bcdef256920c55a4e666 +size 3645759896 diff --git a/model-00064-of-00185.safetensors b/model-00064-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac64035cb3b7f20db81a400b25da2bf4bc5782a4 --- /dev/null +++ b/model-00064-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1f4bd870902f1aad5a2c3f3b8fdecd284725d0be9ae135c3d806c55e0bb7e02 +size 5240783446 diff --git a/model-00065-of-00185.safetensors b/model-00065-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f91c6ba9bd2e313bd720ddf5275519eea6c17b23 --- /dev/null +++ b/model-00065-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d08d53acefa63662fc31db65b593f5a681f9c49a19005d216f1c6cd66294a854 +size 3645759896 diff --git a/model-00066-of-00185.safetensors b/model-00066-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0a06c856a3b4c0d1b91147cc9442f769d73fe570 --- /dev/null +++ b/model-00066-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae42431a717f2b3a0eee8cf48e697d92fbb79cb03127fb0acaf1bd9c272974a8 +size 5240783446 diff --git a/model-00067-of-00185.safetensors b/model-00067-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..42e051ccc3fc7a6fea6167403c898e10588751ef --- /dev/null +++ b/model-00067-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43a3da0e8a92cd76c08f8f23d915f3950be97334107f64eadd31d9f23ff55a01 +size 3645759896 diff --git a/model-00068-of-00185.safetensors b/model-00068-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2fd68bd54a565a5e5191114b3a626a40dcf6a8c3 --- /dev/null +++ b/model-00068-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d39a501bd99f6f6183a4e34e076a2d2cdb8338442965cb2b7d73d6447c7b09 +size 5240783442 diff --git a/model-00069-of-00185.safetensors b/model-00069-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3526ab6cbd29217f70c074f4255633fcb68eaf3f --- /dev/null +++ b/model-00069-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b33cfabb083ea18f3f832ceffe44821b863e995e0996ffb5a145b03aa58d2a1f +size 3473372844 diff --git a/model-00070-of-00185.safetensors b/model-00070-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18392c51141a88381004d5e1f961f0eb3881a9c1 --- /dev/null +++ b/model-00070-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e782f8ee909dd6e0a0ff5fada24e5936bd226f630d6db4301c26f0f63eefa482 +size 5240783446 diff --git a/model-00071-of-00185.safetensors b/model-00071-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..df7e1e0b9dd26d78764e350288cea594c14d31bf --- /dev/null +++ b/model-00071-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c1e55e4e54af49548ac1ef4543e8d6b2936990c246a4403130acdb4cb8bdf6 +size 3645759806 diff --git a/model-00072-of-00185.safetensors b/model-00072-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..49a1bd42fa763cd33a7180261fd41acc3f93b757 --- /dev/null +++ b/model-00072-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e03c6e5dd85325c85377562f36cf12aaa633deab56d3130cc54b8652c7a65c5 +size 5240783446 diff --git a/model-00073-of-00185.safetensors b/model-00073-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dd1d125c67c03ef411a7aef6aa5a956efca1351b --- /dev/null +++ b/model-00073-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e130a9ce56f274656f5bd87bc2f0746182cbd3137dd87197a341f3636385fbec +size 3645759808 diff --git a/model-00074-of-00185.safetensors b/model-00074-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1ac2ac235a18e7cdaf6cb6d70bd421a55a83ec01 --- /dev/null +++ b/model-00074-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfeb37e9da53315f359aa956549bba9a48ed383481c6767c52aa4a29b341ebd +size 5240783446 diff --git a/model-00075-of-00185.safetensors b/model-00075-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad2b3c032fc08678649bb0cce41bedae5301706d --- /dev/null +++ b/model-00075-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b065a7a274b2011003145ee95f832d183cb65b06c339ec4bab2ab73207591c0 +size 3645759894 diff --git a/model-00076-of-00185.safetensors b/model-00076-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..acaab52f76612896429982a4d6fafb8c75b2ce55 --- /dev/null +++ b/model-00076-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98459bf070358dcd7ae7005f53a216df9f203ab235b53d7612e57e68c7a02276 +size 5240783446 diff --git a/model-00077-of-00185.safetensors b/model-00077-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74dd3bc1936daeab6ccbd86286126633771f2cf3 --- /dev/null +++ b/model-00077-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b97f07cba9ff5456dab2f479e6b9920c304ac5d328a1a006fdd766e1c683ac8 +size 3473372912 diff --git a/model-00078-of-00185.safetensors b/model-00078-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7609539720c181b566ce7d5ab893ca67ca0436dd --- /dev/null +++ b/model-00078-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c493209757b954e7ce7604fb3ae19a0c2297969cddee9a3fd5c6c73c30bb761 +size 5240783446 diff --git a/model-00079-of-00185.safetensors b/model-00079-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..01a9ae7730d1a0411c0ff0a0cb05342d2e56cbea --- /dev/null +++ b/model-00079-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b46a753dbdbdea8869fecc0486bcae56b98f438cd3f640a88a933e5b4f8873d7 +size 3645759898 diff --git a/model-00080-of-00185.safetensors b/model-00080-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c120ded4badfb4e3ef4bacca72d4de1f5ab8ff60 --- /dev/null +++ b/model-00080-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90f4db332d90ad525367fbcfcdcc52c264f90ac15fcae733beb1607b5226ec0d +size 5240783442 diff --git a/model-00081-of-00185.safetensors b/model-00081-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dfe37b887f383b8462149c4b3c72011c0229493b --- /dev/null +++ b/model-00081-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5445cb3685bb469f1904a14c80a7b3c77dad5d4e501e6d94f40fd2a75b093d58 +size 3645759894 diff --git a/model-00082-of-00185.safetensors b/model-00082-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fcbe8476d1ed6ab0973d23395466ba91e114e619 --- /dev/null +++ b/model-00082-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f8a75906cdac2d2f8d37ea202452bb28a553effd5527e644ff0fbc80cc22fb +size 5240783442 diff --git a/model-00083-of-00185.safetensors b/model-00083-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dfcff012eed1524903ceb92ef2d1b92156ba2ad7 --- /dev/null +++ b/model-00083-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:351821a8778aa026c02404de8ab3e2c0bf903750ace6d6b0202f52ac30fe06b7 +size 3645759814 diff --git a/model-00084-of-00185.safetensors b/model-00084-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bf79f08fbeccca99e1f6a5cce01a4411e2620c87 --- /dev/null +++ b/model-00084-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04b768d772681423d166d97ad0fb88706be77ffd8c6f297ff6221d8d84a81d94 +size 5240783442 diff --git a/model-00085-of-00185.safetensors b/model-00085-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70d0c55df70834372bc85cca98fbb4d773260bf7 --- /dev/null +++ b/model-00085-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ccaed8a9ef0dfd0b0e10f972af3783060605e62111e4be6f690d7ce08905788 +size 3473372912 diff --git a/model-00086-of-00185.safetensors b/model-00086-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..21809923724ccf18a28c08b380c459b509e93f3d --- /dev/null +++ b/model-00086-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3fb4565eab5f447397e59bbe4b77ea2630b3d6f604ba560ade9ac8fc898c50 +size 5240783446 diff --git a/model-00087-of-00185.safetensors b/model-00087-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9214e53104c8e1fe0da662e948f60e727a900068 --- /dev/null +++ b/model-00087-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c27454772eebb611db7bda05fc0eac98b0c8b0ecd76df6bf1753f5e69599525 +size 3645759892 diff --git a/model-00088-of-00185.safetensors b/model-00088-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..036c674330e4fd5cfab70ed4b1a746a5f6335c45 --- /dev/null +++ b/model-00088-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d16594ba3e5625f58e42714adc54e69ef18c8dd61e38f6a4a045dacaa0ee9b +size 5240783444 diff --git a/model-00089-of-00185.safetensors b/model-00089-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b3ff9b72a8232d41929fa113ae62b71f682fcf80 --- /dev/null +++ b/model-00089-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef147a64cfaf491d77a5f2203715bd5d9432e6085b398384232896ba7548fff0 +size 3645759894 diff --git a/model-00090-of-00185.safetensors b/model-00090-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc45e2ed9eefc5887dd8a7fde0f6537e09cece79 --- /dev/null +++ b/model-00090-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d64f23891bf40cabcf1f3d0331a05f4ffafeb9da6ff18afdbc05f03ce39f65a7 +size 5240783446 diff --git a/model-00091-of-00185.safetensors b/model-00091-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..43f2943133f6187c5305ca7d7589068bec0462f1 --- /dev/null +++ b/model-00091-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51cf53a83f15f966c88248b5de81aa8be8687f7e975cbea226b811cca63317ff +size 3645759894 diff --git a/model-00092-of-00185.safetensors b/model-00092-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7c7748e19c7df1dd92679fbb86dc163fc82945c4 --- /dev/null +++ b/model-00092-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f858f9ae97782e9d7fcfdd2974ede3d524177e5fe93410b1125ddad477b7c1b +size 5240783442 diff --git a/model-00093-of-00185.safetensors b/model-00093-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a97c5e22fc20ffb62aa902325d0b43124abbc452 --- /dev/null +++ b/model-00093-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1415b04f149339217a472c3f8586229a6bd522bf6d14f3bbfb7d434c628773e6 +size 3473372912 diff --git a/model-00094-of-00185.safetensors b/model-00094-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a698b47e881454223bfedde0a1070969270e32b --- /dev/null +++ b/model-00094-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41ba51d612f1285369628acec242a32bee7c41d1ce5c807074b30bf409eb9f03 +size 5240783444 diff --git a/model-00095-of-00185.safetensors b/model-00095-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9dbb8635884d5a2e879a1469c73a4129f2072609 --- /dev/null +++ b/model-00095-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f63b43b167ee2ebeae4b19b227d3a90052d6340fef35e7629a093941d4e7b29 +size 3645759896 diff --git a/model-00096-of-00185.safetensors b/model-00096-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad439068c4261b5caa916ef800dd0d01fa398c2b --- /dev/null +++ b/model-00096-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1948f0386265f5d8f19153934bb50920956c5fcb0841806214c9aafb1649d7a1 +size 5240783446 diff --git a/model-00097-of-00185.safetensors b/model-00097-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f08bf23c22861896f47aa7fabda0364ac1c90ba7 --- /dev/null +++ b/model-00097-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9854213d3befb1615fcc4975ebabb842c3e6846e791de4ac7d0be021d5d63d42 +size 3645759894 diff --git a/model-00098-of-00185.safetensors b/model-00098-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d1ccb8bcf8e35e3f21389e442108ba6c4f9e667 --- /dev/null +++ b/model-00098-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470eb54d7dc7085654ad3a87f4a9b60cdc54312ef99d390fc86a244d942f282a +size 5240783446 diff --git a/model-00099-of-00185.safetensors b/model-00099-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6f8ea4ac8793b10dcad151810a5cf9da4f8f67b2 --- /dev/null +++ b/model-00099-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2265d8095d39d897f1cc210279f81c0e0e2ca2b008cb1e85c9d321dd71965d9f +size 3645759896 diff --git a/model-00100-of-00185.safetensors b/model-00100-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a4b53f8b61ed8dd5e66272f903d642be65f52107 --- /dev/null +++ b/model-00100-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d8b8fadc6971185eab103c04ae2cfcc0fcd3097b81cdb1a68d22a742245de05 +size 5240783444 diff --git a/model-00101-of-00185.safetensors b/model-00101-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..23e28974401f26ada00789ed71564c13d6bbee2d --- /dev/null +++ b/model-00101-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9a87920e45c4aa1b43bf6fcb3bea61b27479733cde2198774fb6ca867c8e024 +size 3473372890 diff --git a/model-00102-of-00185.safetensors b/model-00102-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..01ab0808d4f615cd1eaf65d71cec74baf69b0b18 --- /dev/null +++ b/model-00102-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f901f53e9e8a5f61cd87ed001a21db49d9ada1f4daff087a8f8fe0e05bda3862 +size 5240783446 diff --git a/model-00103-of-00185.safetensors b/model-00103-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9a6877315dae8ba526425076b030c3b3149f1287 --- /dev/null +++ b/model-00103-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19db6f26363471d61268ed33d5b807894698c795b691059e7f5c336153b25b7a +size 3645759822 diff --git a/model-00104-of-00185.safetensors b/model-00104-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a252a36ed2d53e854bca5098b243107ce03ed6c9 --- /dev/null +++ b/model-00104-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df155b64a0e35a222f9241f26405665b9c02bfad21b18c73d8b61916ceddff9a +size 5240783446 diff --git a/model-00105-of-00185.safetensors b/model-00105-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee7f1246caebaa6103b5e6dfd4354c73a9d99797 --- /dev/null +++ b/model-00105-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f5b7371dc0f00bd297d43b7adf31edbd85c55eb6e75cde2917708342f8cf816 +size 3645759816 diff --git a/model-00106-of-00185.safetensors b/model-00106-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..189de12b218eb8e6eb1f9bfebd355891de3234cb --- /dev/null +++ b/model-00106-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791c1bfed8209403ef4c2cf37f21682225a5e52182c723f1d1a264f7a87f1167 +size 5240783446 diff --git a/model-00107-of-00185.safetensors b/model-00107-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d2c8e2afb4a4cea7f90d18bbc9ef33ba39550a3 --- /dev/null +++ b/model-00107-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5baf608130c47a3c9d0acd7191cf7bf9df433382d6efc82162abaa997ffb7cb8 +size 3645759894 diff --git a/model-00108-of-00185.safetensors b/model-00108-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f60a91ce72aa5ccaf36cd2f8b62262f9bcfee3d --- /dev/null +++ b/model-00108-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72b93afc8d40830acc42abf319e53e0fd2cc67041175f1660e70e200854a6fa6 +size 5240783446 diff --git a/model-00109-of-00185.safetensors b/model-00109-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8dc11dcbbbe94dd58dba263e7eea1183a84e0df4 --- /dev/null +++ b/model-00109-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:754da9230e6427d6ac743e91663414568a8ddc7c16cb8222df29cb82fbcc21f2 +size 3473372820 diff --git a/model-00110-of-00185.safetensors b/model-00110-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d042c8fba017e625d9cc1f4b372708ec1f6fc1dc --- /dev/null +++ b/model-00110-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d900f5777094d1aeb6d1a7135b48797f465b8518f47e8e518bbe3cedd7eca06 +size 5240783446 diff --git a/model-00111-of-00185.safetensors b/model-00111-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6818c147dd7d52137d76a90575c657ad6c69fc70 --- /dev/null +++ b/model-00111-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f0b13a46d4765eda8551a998beaf1da2f55783bc40784b3636552567e183a30 +size 3645759896 diff --git a/model-00112-of-00185.safetensors b/model-00112-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8cc3bcd0eacd97fec33859f15b6c3c3180c1eaa --- /dev/null +++ b/model-00112-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2b6c289e4f222e64e8359f9a305a459c0041a56a4aebaae3a35634e0b5d3d24 +size 5240783444 diff --git a/model-00113-of-00185.safetensors b/model-00113-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e26c09c91faa9c72cffb1616b8fa13da30861048 --- /dev/null +++ b/model-00113-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:698a75f24de0afd31d53803fcc2287f148dca627711989ac2c5eb1ae3eb2d3fa +size 3645759794 diff --git a/model-00114-of-00185.safetensors b/model-00114-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7fe6162f82c6abeaa956b3b3a497889d9911e116 --- /dev/null +++ b/model-00114-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db9ba0b046f16bfeb8122375c090f6961a709f674868d1e76a6620a9f443e64 +size 5240783446 diff --git a/model-00115-of-00185.safetensors b/model-00115-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2d60fd625a2a425d14728c8327605d1e9d8cd6dc --- /dev/null +++ b/model-00115-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4394294e7a0cdf855d06ed2ff90a8406271608912c137d4cd4d705763082be7 +size 3645759874 diff --git a/model-00116-of-00185.safetensors b/model-00116-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..52e772d8e29ea00cce36cc36d316c525db17007a --- /dev/null +++ b/model-00116-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a443bba7596af9122defa509c83ea93b1e5cbd766f390ac1312d46a7b76e9738 +size 5240783442 diff --git a/model-00117-of-00185.safetensors b/model-00117-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2c645deffb9095d591eaf5780d7c674ca3d7c79 --- /dev/null +++ b/model-00117-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46c2bacbf7c3adff740f41e284975a385262ad6a27c8c20a74055eb9e6d1b5c4 +size 3473372858 diff --git a/model-00118-of-00185.safetensors b/model-00118-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cdff4a9f4588a27429d1b98a244669fc396df741 --- /dev/null +++ b/model-00118-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70ae55cc4100ee18ee1d1a11f05d699b701e80eb1e28dff66ea07edea5cc0438 +size 5240783446 diff --git a/model-00119-of-00185.safetensors b/model-00119-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..53a35e2943b6125d7d6cf5b4dec45eb7899afb4c --- /dev/null +++ b/model-00119-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:043ac414cf166ede285efc1a98f0b5287df525f5c635fc3b5faec80318cea66c +size 3645759836 diff --git a/model-00120-of-00185.safetensors b/model-00120-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d58343e846f796e307cbb6953f6133bb38a731fb --- /dev/null +++ b/model-00120-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d58529f8ae2edd9b39489002202ad1855183824fcddfcf292126c3f9bc7bd06f +size 5240783446 diff --git a/model-00121-of-00185.safetensors b/model-00121-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2e7abd2a37c5e0e8165c63add15654242e14f7e --- /dev/null +++ b/model-00121-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2206b3ace9579ee7d5270ef1bf754b9a0205744ad68bce0506888779ccf0a841 +size 3645759898 diff --git a/model-00122-of-00185.safetensors b/model-00122-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12015fb40eb54aaca076a1ac250012e97cda1e84 --- /dev/null +++ b/model-00122-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e984b64d677c24152fb90bd5bbbbf0e95dfd95d5fd78b1abd33a860ffd3886f7 +size 5240783444 diff --git a/model-00123-of-00185.safetensors b/model-00123-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e06f2341fe8e181d7522a58ca512dc75c25f7bf2 --- /dev/null +++ b/model-00123-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dc19a160c31c72a50d992cf887e63c5061c6500ab2dcadda991c3000a8d69e7 +size 3645759834 diff --git a/model-00124-of-00185.safetensors b/model-00124-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ce5801826c66c48af8e27f6919b81f5a14965b8 --- /dev/null +++ b/model-00124-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca1d18fed4a2b805fbb43db6b2a2506a6fab2ff311dacf7f31fc29040dff06a +size 5240783446 diff --git a/model-00125-of-00185.safetensors b/model-00125-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cefc1d68b4269ad7e361231c5aa9b50d24c10604 --- /dev/null +++ b/model-00125-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90efa12bf33cfad909b159be96d526ab07ad6e5289c3c8d5ccb4bbbaf37a23bd +size 3473372834 diff --git a/model-00126-of-00185.safetensors b/model-00126-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c0afcd61520f622e9dc0910f831d95fc33ef2886 --- /dev/null +++ b/model-00126-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ac3fda05737462716e8279e06e39259b0fe36c5f1058ea65fd651667657e0de +size 5240783446 diff --git a/model-00127-of-00185.safetensors b/model-00127-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bed40291852bd8248baf9bf8638980bf2300a41d --- /dev/null +++ b/model-00127-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41d4683cdab6ce7dfd0af93c5497eab74ded7205a210544c7f1123c4426f8bca +size 3645759896 diff --git a/model-00128-of-00185.safetensors b/model-00128-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..925ca1f0556b77de2e757842f12ec508ba4d0064 --- /dev/null +++ b/model-00128-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a66de134369ef8cfec6861d52b4167631aee538b0b66a7dc06032453945c83c3 +size 5240783446 diff --git a/model-00129-of-00185.safetensors b/model-00129-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c41038ff2fb9277c05cca7a2091ff96132371e5 --- /dev/null +++ b/model-00129-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83381033c99825fba688198a22ad4eccb8487d4f6de6882af0cf2ce0ca451f01 +size 3645759896 diff --git a/model-00130-of-00185.safetensors b/model-00130-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e43e9812724803569605c550c43350f048eeef6 --- /dev/null +++ b/model-00130-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:395717e331c47ff570ec601808d3056ed803be43fa24bcea2d4f9479261b7296 +size 5240783442 diff --git a/model-00131-of-00185.safetensors b/model-00131-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d12c9df8750b6a12d7f6904489e35f6262a49aa9 --- /dev/null +++ b/model-00131-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e82b147cd74930106a6144fac2935ff7f8ec56aa8a5c580c7245e7797f5d934 +size 3645759896 diff --git a/model-00132-of-00185.safetensors b/model-00132-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..feb2860813f1896b1893f56a9cab74c31e7f2ec0 --- /dev/null +++ b/model-00132-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa34b0ea3ecf9dc5bc0292c8767b218c7a393180cd2232d949350a716a73d9d2 +size 5240783444 diff --git a/model-00133-of-00185.safetensors b/model-00133-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9834b436e25537ed72e63cc1f44efcf37b10727b --- /dev/null +++ b/model-00133-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d67d0f24be17ca7495e5eef8cc0f77e5b6c61e843b4b02fc3a9da0554e5003b8 +size 3473372914 diff --git a/model-00134-of-00185.safetensors b/model-00134-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..71f59eaf55a5b6fcd38ba1282dc4b06ae1dd4c45 --- /dev/null +++ b/model-00134-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00e7d6c10a8281d9dd03d022d7d9df82271f2205f18cb312f4d6f00c8afab5a7 +size 5240783444 diff --git a/model-00135-of-00185.safetensors b/model-00135-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..877b8d552c8bca28b5280dbdf51edb57aca939af --- /dev/null +++ b/model-00135-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03913e3838f1b88f61b9f941d05e16f74e903f3a4218404c8b8b57ac8bbe739e +size 3645759814 diff --git a/model-00136-of-00185.safetensors b/model-00136-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d8bf6dcca03c6b41fbc6a26c1de4cf66a0e1d1fb --- /dev/null +++ b/model-00136-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab6a35d44b9707cd8ba65a727ab49af3679668c482f425b508257f7f631ae0f +size 5240783446 diff --git a/model-00137-of-00185.safetensors b/model-00137-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..afb84bdff8adc62a031f2e547fd18bc18a71ec3c --- /dev/null +++ b/model-00137-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c7b3357abf288f15067834dbc682c98fdf50eea440ebd06d94e634e32f2482d +size 3645759898 diff --git a/model-00138-of-00185.safetensors b/model-00138-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fffb225da2e7e6883d3f52cf0e66ff3d55dc473f --- /dev/null +++ b/model-00138-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c31a41dbdcd01aa92e1af712919ac3958054854373d1ea6e43469c06c10531 +size 5240783446 diff --git a/model-00139-of-00185.safetensors b/model-00139-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2227c18c9ae4d02f2026af2186fbe5b8d3c4a50c --- /dev/null +++ b/model-00139-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a7b6f2a396b5baab6af143ea38983032569e14f35f751d1c10c7486eab74630 +size 3645759896 diff --git a/model-00140-of-00185.safetensors b/model-00140-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d47214802608a19ff26d48465ad1574e2a85e5a --- /dev/null +++ b/model-00140-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3faf6c0f1b2b219d4d21eb756c69468684f8a89977fdb2c4a0a3acfe8c4ba7d8 +size 5240783442 diff --git a/model-00141-of-00185.safetensors b/model-00141-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f274a2cb3d51179113da2c5b1e3249ec3684d27 --- /dev/null +++ b/model-00141-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1471264fa927393f638927850ce316a48e00813d31fa688fa9a2e8b37f06cfb0 +size 3473372910 diff --git a/model-00142-of-00185.safetensors b/model-00142-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..46099eae6bccb3b526a153286eeeb456fe45adab --- /dev/null +++ b/model-00142-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f868d900e161cc494ea40ede44c7d14b4aeb0f6fcacd71f10b87e3783183d5 +size 5240783446 diff --git a/model-00143-of-00185.safetensors b/model-00143-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..297876974ec922b2e145c2a95d52a3da7bc02569 --- /dev/null +++ b/model-00143-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182265b34e4a3181e33dfa03aae41cb2b8a15210d78d7886d39aff355c28ca02 +size 3645759894 diff --git a/model-00144-of-00185.safetensors b/model-00144-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..845df30fb68315c833639992b2b18ccc415faf4b --- /dev/null +++ b/model-00144-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec739b2aa075d4772799649ef6481bcc7191bac119bff81334fc5ad37448a28c +size 5240783446 diff --git a/model-00145-of-00185.safetensors b/model-00145-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f48f63d612a6516891b105d82898e03157718559 --- /dev/null +++ b/model-00145-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05428c83015c0a8c8cf9ce4a71d5c244e23911068743a163f34cfa64510301c8 +size 3645759898 diff --git a/model-00146-of-00185.safetensors b/model-00146-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..22ac3a2c9983e66b8913c9c75f91f034a75d3416 --- /dev/null +++ b/model-00146-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052b908cfa686e5c88c40364414dc06aaead208befd1e83e1c1dec42760f9d60 +size 5240783444 diff --git a/model-00147-of-00185.safetensors b/model-00147-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e65c3c2d229d8db9d0f6fdebe4f9f1e5d9ce34a1 --- /dev/null +++ b/model-00147-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20bbb45e93e71ea871c76dd349b03d7f72b8416b4154cb83347ea8766ce5ee21 +size 3645759840 diff --git a/model-00148-of-00185.safetensors b/model-00148-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a71bdf850ed2a4d659ba6259bc861f31cf20c14d --- /dev/null +++ b/model-00148-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2382c6b4f1e3060ee844a84fa7093edbc7ebe5e7b6b3961eff6989380fa08f3f +size 5240783444 diff --git a/model-00149-of-00185.safetensors b/model-00149-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..561fa1ae18e5f348ae96edad763f88bdde87ea1a --- /dev/null +++ b/model-00149-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b129f1ba4ca9f8a6adbb7827549777b3236cf55f2c4ea1da5c3a553e6b262465 +size 3473372910 diff --git a/model-00150-of-00185.safetensors b/model-00150-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05467aab3a49cf7a582dc0747c465e9723106efd --- /dev/null +++ b/model-00150-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:655f33d519cd23648b7b199f0b10ace9ef86af65a6d4748dead26db0b0a8f9c5 +size 5240783446 diff --git a/model-00151-of-00185.safetensors b/model-00151-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4352e47198f89913ac471eabc3c9f0bee79c35c8 --- /dev/null +++ b/model-00151-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ff4ee638e191ec338bd13b28e12e19e78db4352d1f5f938598747281de4c8d +size 3645759894 diff --git a/model-00152-of-00185.safetensors b/model-00152-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b58dd3b1185b0c9c0593f3019100945ee7b69879 --- /dev/null +++ b/model-00152-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece250b992236056c2852f4df451eebdecd48bce0d6d59860f1f121829c4c85b +size 5240783446 diff --git a/model-00153-of-00185.safetensors b/model-00153-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..beac84605ad3aaee6f6e252585b5ed1dc9af44fc --- /dev/null +++ b/model-00153-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb3134723bb830ee723e135a1d31ae1205bdb722390715267f5995edab6acf5 +size 3645759870 diff --git a/model-00154-of-00185.safetensors b/model-00154-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b7fb5fca03f522708cf905c2580a2f1545e642e9 --- /dev/null +++ b/model-00154-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37a6053b2e9182412856c60235d4bd11e309b8cd6e3a5216112bfdf1c24c32cb +size 5240783446 diff --git a/model-00155-of-00185.safetensors b/model-00155-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5b3eb068703bc5ed9f04b64f76d100c7612b5af2 --- /dev/null +++ b/model-00155-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b488ce770acb8ec93a1241a2dc4b88719e380627a394e916714d6e45f847733c +size 3645759896 diff --git a/model-00156-of-00185.safetensors b/model-00156-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d727816e42718ebd752394b29fa94f2ac7dd8ac9 --- /dev/null +++ b/model-00156-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86b05a3b721bbb7b67603fe31fccaa76ec66b36447a02de4a036621a19941f7b +size 5240783446 diff --git a/model-00157-of-00185.safetensors b/model-00157-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab7330f5c3af219d75fdd3ec671454bb37a72e73 --- /dev/null +++ b/model-00157-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df330ba4731c152b6ad10f4b82a1d86cb09a61012af60641d41c28fd73100b33 +size 3473372886 diff --git a/model-00158-of-00185.safetensors b/model-00158-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12eb1044d56411c1765259f20469cc2e0d2bd274 --- /dev/null +++ b/model-00158-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79fee74a821d0751880c3a992e702edd9b05d995875b51dc3cd831d5cc2b540 +size 5240783446 diff --git a/model-00159-of-00185.safetensors b/model-00159-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f5f7c8fbace2d00801772cafb170a8fc59177cda --- /dev/null +++ b/model-00159-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b53b64328ce622b32fa74b0f9e0584cbd66e92d1b810976a878e1d0d72e62ad +size 3645759894 diff --git a/model-00160-of-00185.safetensors b/model-00160-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e88000a4ac97ba779c4fb02fd977acf5b69500a0 --- /dev/null +++ b/model-00160-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b33f74eadc0cf5fb685fbf8a008d12d94357dddfa5c07ac2f88f01a70b6cd35 +size 5240783446 diff --git a/model-00161-of-00185.safetensors b/model-00161-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2d41f6aa6baeed5216a1deea77a04ee21a507e16 --- /dev/null +++ b/model-00161-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5623dd277b77192ebd1b269d36cf11aa8b090828f6053d2e060cbc94b2ce6c8 +size 3645759864 diff --git a/model-00162-of-00185.safetensors b/model-00162-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff8e19f87c04e792dcbd90d47613d05582699304 --- /dev/null +++ b/model-00162-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d29b3f8dfc6ec05e97fe7a827f89003dab72e7d46d407b9e8180e4c73ce3f0b6 +size 5240783446 diff --git a/model-00163-of-00185.safetensors b/model-00163-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..61901c430eab8a4a5a08f827338388e25384d82b --- /dev/null +++ b/model-00163-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727e55ab39956e185fb47dbe518f0c6e9439d8c74643c735d0d2a27f316cc067 +size 3645759828 diff --git a/model-00164-of-00185.safetensors b/model-00164-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5242fc8d98cc186919853987def896c1a927eafa --- /dev/null +++ b/model-00164-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b471d4e221c226701c3e5d5410de99effaa16d27df0708723e53bc131060307 +size 5240783446 diff --git a/model-00165-of-00185.safetensors b/model-00165-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea189fae18fb804fbf705b6ee98a3368b6057fbe --- /dev/null +++ b/model-00165-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8714bc6bf34c9320cf14233e6b5e9e70fd98a8ec2ec946b73682603efc54faa +size 3473372912 diff --git a/model-00166-of-00185.safetensors b/model-00166-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f683a720d796bb41ebee04562d457f4d6acc216e --- /dev/null +++ b/model-00166-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c37f04c6b3816451c348dce8ffac13bef016018e87d0321dbc9954d667782131 +size 5240783444 diff --git a/model-00167-of-00185.safetensors b/model-00167-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..42859d045614a55797c130baafe36ee018ddbb84 --- /dev/null +++ b/model-00167-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4616f70a13a49f87fbb2a83b062e850910e0693e02ba53df7a5d5887504b88d +size 3645759894 diff --git a/model-00168-of-00185.safetensors b/model-00168-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..036a3e42989fa1628f2a73be77b9aebe5902d020 --- /dev/null +++ b/model-00168-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:453b434f4551b590a42045893ecf8574f67ca3744b3cc90c5fdcd21bc50eeaa8 +size 5240783446 diff --git a/model-00169-of-00185.safetensors b/model-00169-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b17c97cd15682b267a77841657036869ce4c793c --- /dev/null +++ b/model-00169-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574cb09ea4f846386e66b565a238cac71859b5cb7945455b60677ccc3106af97 +size 3645759816 diff --git a/model-00170-of-00185.safetensors b/model-00170-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..115a88e0d6c6d737c6fec5a3179f0a7a228ebca1 --- /dev/null +++ b/model-00170-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4009c57e2c1549c9f34cd70b692867c1848b8d7ca31f6c0cd59d44fc67433e02 +size 5240783446 diff --git a/model-00171-of-00185.safetensors b/model-00171-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..15ef2c335778439de3c0a0c661578e0fee627ba7 --- /dev/null +++ b/model-00171-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89952ff0b2dc3f3fcb26bbf80e3aef4a78a84c6d1ea1ac22ae50e94e27fb4a0 +size 3645759896 diff --git a/model-00172-of-00185.safetensors b/model-00172-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5a5ba71fca908de70ed57385b68b9363f807bb0c --- /dev/null +++ b/model-00172-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da492d628235f90ba20c71fdde5b2e54cdbf255b81de9f601273f94bddeb210d +size 5240783446 diff --git a/model-00173-of-00185.safetensors b/model-00173-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9d25dd89ece4d824ad5196090c164ff61aea8dd2 --- /dev/null +++ b/model-00173-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:920bf375ee471430a420ee5c998f17b928a193bd0a8cdb177953ab7aa8970e2c +size 3473372910 diff --git a/model-00174-of-00185.safetensors b/model-00174-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1ad8dada1d7633ece26951d10b0a7c92c8b6129 --- /dev/null +++ b/model-00174-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ab0aab0d1d350dd01d26f9b3ac3069f311caaa71a220bba75595bff71d3e69 +size 5240783442 diff --git a/model-00175-of-00185.safetensors b/model-00175-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b7c9ffe8800523d2e4327375bccb92d7380ffb0e --- /dev/null +++ b/model-00175-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfa5f1df7756b81ae37d72801c160b7d11459f09bb790e94174089fb430b5483 +size 3645759896 diff --git a/model-00176-of-00185.safetensors b/model-00176-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..743a772d5acd4e87c74dfaf7faac488d6fb39d12 --- /dev/null +++ b/model-00176-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84671ed3b2c749e22c7b34fa8bb19e49a3e1d70569e6762d4ea76882f443b92 +size 5240783446 diff --git a/model-00177-of-00185.safetensors b/model-00177-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3aef8693de53f475d661a0552a3d534c0e55facd --- /dev/null +++ b/model-00177-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fc6be25643812373486514f0da8f9859e99694765990c3d05e3378bbb27d1e5 +size 3645759894 diff --git a/model-00178-of-00185.safetensors b/model-00178-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..61f278ed0fb91da692815a326b88d5156cd0c087 --- /dev/null +++ b/model-00178-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a76c695ad7ad5d5a1018b585b2d27aa076db04382034c594865ed6c9dd8276bf +size 5240783444 diff --git a/model-00179-of-00185.safetensors b/model-00179-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8978b3a78ada9f9ec7c8446d836ed86481837e3f --- /dev/null +++ b/model-00179-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce9d5030ec1dbae48011ad8ffeb443df84d731e9d9d7a0016e287ecfc90e7da9 +size 3645759894 diff --git a/model-00180-of-00185.safetensors b/model-00180-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..23b5fadde1acc199ac359a36b261b671222f544e --- /dev/null +++ b/model-00180-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b81ecca9b5405faec9395de83e0cfd398cfc30e8ae1734454a838f72b9d77d7 +size 5240783446 diff --git a/model-00181-of-00185.safetensors b/model-00181-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f6284edc644f7fae169f6abe5a8a66e62c8b071 --- /dev/null +++ b/model-00181-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71563c0aa18ac4a724e6f2c0f5c28550810a978279c4b658592feb42a007b07 +size 3473372838 diff --git a/model-00182-of-00185.safetensors b/model-00182-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cecc35270b448875d44b45e0662b0e79a1562282 --- /dev/null +++ b/model-00182-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a783c9d1f07491c15e5bbc76db97f2f30a19d52441fbc05924726ad65699816 +size 5240783444 diff --git a/model-00183-of-00185.safetensors b/model-00183-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..24ff82125dcdbf56037bb29a97471e3df7a8f334 --- /dev/null +++ b/model-00183-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ecb6d26fa24eeeca24dabc124f35fd424a56841af2cb7a3ee095d94bb19334 +size 3473372912 diff --git a/model-00184-of-00185.safetensors b/model-00184-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9ecf7a36105d4b09e9dbb592a74729c850e01cc --- /dev/null +++ b/model-00184-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2ba636a72139ec440788ff601ac6cc03249e122c151d19d68ef31c0e652764 +size 5240783446 diff --git a/model-00185-of-00185.safetensors b/model-00185-of-00185.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..164182badee0cc94edb0e0a333ea8a9a868fb55e --- /dev/null +++ b/model-00185-of-00185.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b89387b3cace2409246007e0a97d019da09cbd10d85d4dec99d1f8365ad6c2c +size 1163997726 diff --git a/model-passthrough-00001.safetensors b/model-passthrough-00001.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..548e07fb676e0c9baa736a52d447e931d774c08b --- /dev/null +++ b/model-passthrough-00001.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49f9f84276c7c0aa3d2feeb9dc31bb69b6e6ee59da374c4cf9ac0b9c96a94df +size 894737694 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..d1f74071cf73c5d48edc7f8fc12ea8410c52d716 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,5490 @@ +{ + "metadata": { + "total_size": 816773159296, + "total_parameters": 2779483539072 + }, + "weight_map": { + "language_model.lm_head.biases": "model-00185-of-00185.safetensors", + "language_model.lm_head.scales": "model-00185-of-00185.safetensors", + "language_model.lm_head.weight": "model-00185-of-00185.safetensors", + "language_model.model.embed_tokens.biases": "model-00001-of-00185.safetensors", + "language_model.model.embed_tokens.scales": "model-00001-of-00185.safetensors", + "language_model.model.embed_tokens.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.down_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.down_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.gate_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.gate_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.up_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.up_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp_res_norm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.mlp_res_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attention_res_norm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attention_res_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.A_log": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.b_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.b_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.b_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.dt_bias": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_a_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_a_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_a_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_b_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_b_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.f_b_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.g_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.g_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.g_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.o_norm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.o_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.o_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.qkv_conv.conv.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.qkv_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.qkv_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.input_layernorm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.e_score_correction_bias": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.gate.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.gate.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.gate.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_down_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_down_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_down_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_norm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_up_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_up_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.routed_expert_up_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.down_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.down_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.down_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.gate_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.gate_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.up_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.up_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.shared_experts.up_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.down_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.down_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight": "model-00002-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.up_proj.biases": "model-00002-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.up_proj.scales": "model-00002-of-00185.safetensors", + "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight": "model-00002-of-00185.safetensors", + "language_model.model.layers.1.mlp_res_norm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.mlp_res_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.post_attention_layernorm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.self_attention_res_norm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.self_attention_res_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.1.self_attn.A_log": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.b_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.b_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.b_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.dt_bias": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_a_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_a_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_a_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_b_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_b_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.f_b_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.g_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.g_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.g_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.o_norm.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.o_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.o_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.qkv_conv.conv.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.qkv_proj.biases": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.qkv_proj.scales": "model-00001-of-00185.safetensors", + "language_model.model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00185.safetensors", + "language_model.model.layers.10.input_layernorm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.e_score_correction_bias": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.gate.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.gate.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.gate.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_down_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_down_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_down_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_norm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_up_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_up_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.routed_expert_up_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.down_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.down_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.down_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.gate_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.gate_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.up_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.up_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.shared_experts.up_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.down_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.down_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight": "model-00020-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.gate_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.gate_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.up_proj.biases": "model-00020-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.up_proj.scales": "model-00020-of-00185.safetensors", + "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight": "model-00020-of-00185.safetensors", + "language_model.model.layers.10.mlp_res_norm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.mlp_res_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.post_attention_layernorm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.self_attention_res_norm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.self_attention_res_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.10.self_attn.A_log": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.b_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.b_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.b_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.dt_bias": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_a_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_a_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_a_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_b_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_b_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.f_b_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.g_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.g_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.g_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.o_norm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.o_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.o_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.o_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.qkv_conv.conv.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.qkv_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.qkv_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.10.self_attn.qkv_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.11.input_layernorm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.e_score_correction_bias": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.gate.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.gate.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.gate.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_down_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_down_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_down_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_norm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_up_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_up_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.routed_expert_up_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.down_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.down_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.down_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.gate_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.gate_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.up_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.up_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.shared_experts.up_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.down_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.down_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight": "model-00022-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.gate_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.gate_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.up_proj.biases": "model-00022-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.up_proj.scales": "model-00022-of-00185.safetensors", + "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight": "model-00022-of-00185.safetensors", + "language_model.model.layers.11.mlp_res_norm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.mlp_res_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.post_attention_layernorm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.self_attention_res_norm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.self_attention_res_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.11.self_attn.embed_q.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.embed_q.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.embed_q.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.g_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.g_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.g_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.kv_a_layernorm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.o_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.o_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.o_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_a_layernorm.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_a_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_a_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_a_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_b_proj.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_b_proj.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.q_b_proj.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.unembed_out.biases": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.unembed_out.scales": "model-00021-of-00185.safetensors", + "language_model.model.layers.11.self_attn.unembed_out.weight": "model-00021-of-00185.safetensors", + "language_model.model.layers.12.input_layernorm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.e_score_correction_bias": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.gate.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.gate.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.gate.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_down_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_down_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_down_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_norm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_up_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_up_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.routed_expert_up_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.down_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.down_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.down_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.gate_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.gate_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.up_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.up_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.shared_experts.up_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.down_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.down_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight": "model-00024-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.gate_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.gate_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.up_proj.biases": "model-00024-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.up_proj.scales": "model-00024-of-00185.safetensors", + "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight": "model-00024-of-00185.safetensors", + "language_model.model.layers.12.mlp_res_norm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.mlp_res_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.post_attention_layernorm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.self_attention_res_norm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.self_attention_res_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.12.self_attn.A_log": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.b_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.b_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.b_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.dt_bias": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_a_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_a_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_a_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_b_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_b_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.f_b_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.g_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.g_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.g_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.o_norm.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.o_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.o_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.o_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.qkv_conv.conv.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.qkv_proj.biases": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.qkv_proj.scales": "model-00023-of-00185.safetensors", + "language_model.model.layers.12.self_attn.qkv_proj.weight": "model-00023-of-00185.safetensors", + "language_model.model.layers.13.input_layernorm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.e_score_correction_bias": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.gate.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.gate.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.gate.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_down_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_down_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_down_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_norm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_up_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_up_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.routed_expert_up_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.down_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.down_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.down_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.gate_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.gate_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.up_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.up_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.shared_experts.up_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.down_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.down_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight": "model-00026-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.gate_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.gate_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.up_proj.biases": "model-00026-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.up_proj.scales": "model-00026-of-00185.safetensors", + "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight": "model-00026-of-00185.safetensors", + "language_model.model.layers.13.mlp_res_norm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.mlp_res_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.post_attention_layernorm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.self_attention_res_norm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.self_attention_res_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.13.self_attn.A_log": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.b_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.b_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.b_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.dt_bias": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_a_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_a_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_a_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_b_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_b_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.f_b_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.g_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.g_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.g_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.o_norm.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.o_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.o_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.o_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.qkv_conv.conv.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.qkv_proj.biases": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.qkv_proj.scales": "model-00025-of-00185.safetensors", + "language_model.model.layers.13.self_attn.qkv_proj.weight": "model-00025-of-00185.safetensors", + "language_model.model.layers.14.input_layernorm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.e_score_correction_bias": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.gate.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.gate.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.gate.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_down_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_down_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_down_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_norm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_up_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_up_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.routed_expert_up_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.down_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.down_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.down_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.gate_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.gate_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.up_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.up_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.shared_experts.up_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.down_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.down_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight": "model-00028-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.gate_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.gate_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.up_proj.biases": "model-00028-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.up_proj.scales": "model-00028-of-00185.safetensors", + "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight": "model-00028-of-00185.safetensors", + "language_model.model.layers.14.mlp_res_norm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.mlp_res_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.post_attention_layernorm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.self_attention_res_norm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.self_attention_res_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.14.self_attn.A_log": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.b_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.b_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.b_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.dt_bias": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_a_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_a_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_a_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_b_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_b_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.f_b_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.g_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.g_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.g_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.o_norm.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.o_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.o_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.o_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.qkv_conv.conv.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.qkv_proj.biases": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.qkv_proj.scales": "model-00027-of-00185.safetensors", + "language_model.model.layers.14.self_attn.qkv_proj.weight": "model-00027-of-00185.safetensors", + "language_model.model.layers.15.input_layernorm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.e_score_correction_bias": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.gate.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.gate.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.gate.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_down_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_down_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_down_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_norm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_up_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_up_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.routed_expert_up_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.down_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.down_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.down_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.gate_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.gate_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.up_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.up_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.shared_experts.up_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.down_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.down_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight": "model-00030-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.gate_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.gate_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.up_proj.biases": "model-00030-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.up_proj.scales": "model-00030-of-00185.safetensors", + "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight": "model-00030-of-00185.safetensors", + "language_model.model.layers.15.mlp_res_norm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.mlp_res_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.post_attention_layernorm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.self_attention_res_norm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.self_attention_res_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.15.self_attn.embed_q.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.embed_q.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.embed_q.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.g_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.g_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.g_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.kv_a_layernorm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.o_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.o_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.o_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_a_layernorm.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_a_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_a_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_a_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_b_proj.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_b_proj.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.q_b_proj.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.unembed_out.biases": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.unembed_out.scales": "model-00029-of-00185.safetensors", + "language_model.model.layers.15.self_attn.unembed_out.weight": "model-00029-of-00185.safetensors", + "language_model.model.layers.16.input_layernorm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.e_score_correction_bias": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.gate.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.gate.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.gate.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_down_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_down_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_down_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_norm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_up_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_up_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.routed_expert_up_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.down_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.down_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.down_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.gate_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.gate_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.up_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.up_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.shared_experts.up_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.down_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.down_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight": "model-00032-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.gate_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.gate_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.up_proj.biases": "model-00032-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.up_proj.scales": "model-00032-of-00185.safetensors", + "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight": "model-00032-of-00185.safetensors", + "language_model.model.layers.16.mlp_res_norm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.mlp_res_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.post_attention_layernorm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.self_attention_res_norm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.self_attention_res_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.16.self_attn.A_log": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.b_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.b_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.b_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.dt_bias": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_a_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_a_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_a_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_b_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_b_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.f_b_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.g_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.g_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.g_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.o_norm.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.o_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.o_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.o_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.qkv_conv.conv.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.qkv_proj.biases": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.qkv_proj.scales": "model-00031-of-00185.safetensors", + "language_model.model.layers.16.self_attn.qkv_proj.weight": "model-00031-of-00185.safetensors", + "language_model.model.layers.17.input_layernorm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.e_score_correction_bias": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.gate.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.gate.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.gate.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_down_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_down_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_down_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_norm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_up_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_up_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.routed_expert_up_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.down_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.down_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.down_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.gate_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.gate_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.up_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.up_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.shared_experts.up_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.down_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.down_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight": "model-00034-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.gate_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.gate_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.up_proj.biases": "model-00034-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.up_proj.scales": "model-00034-of-00185.safetensors", + "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight": "model-00034-of-00185.safetensors", + "language_model.model.layers.17.mlp_res_norm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.mlp_res_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.post_attention_layernorm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.self_attention_res_norm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.self_attention_res_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.17.self_attn.A_log": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.b_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.b_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.b_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.dt_bias": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_a_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_a_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_a_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_b_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_b_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.f_b_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.g_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.g_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.g_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.o_norm.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.o_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.o_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.o_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.qkv_conv.conv.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.qkv_proj.biases": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.qkv_proj.scales": "model-00033-of-00185.safetensors", + "language_model.model.layers.17.self_attn.qkv_proj.weight": "model-00033-of-00185.safetensors", + "language_model.model.layers.18.input_layernorm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.e_score_correction_bias": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.gate.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.gate.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.gate.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_down_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_down_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_down_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_norm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_up_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_up_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.routed_expert_up_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.down_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.down_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.down_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.gate_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.gate_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.up_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.up_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.shared_experts.up_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.down_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.down_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight": "model-00036-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.gate_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.gate_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.up_proj.biases": "model-00036-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.up_proj.scales": "model-00036-of-00185.safetensors", + "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight": "model-00036-of-00185.safetensors", + "language_model.model.layers.18.mlp_res_norm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.mlp_res_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.post_attention_layernorm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.self_attention_res_norm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.self_attention_res_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.18.self_attn.A_log": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.b_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.b_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.b_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.dt_bias": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_a_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_a_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_a_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_b_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_b_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.f_b_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.g_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.g_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.g_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.o_norm.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.o_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.o_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.o_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.qkv_conv.conv.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.qkv_proj.biases": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.qkv_proj.scales": "model-00035-of-00185.safetensors", + "language_model.model.layers.18.self_attn.qkv_proj.weight": "model-00035-of-00185.safetensors", + "language_model.model.layers.19.input_layernorm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.e_score_correction_bias": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.gate.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.gate.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.gate.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_down_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_down_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_down_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_norm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_up_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_up_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.routed_expert_up_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.down_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.down_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.down_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.gate_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.gate_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.up_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.up_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.shared_experts.up_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.down_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.down_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight": "model-00038-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.gate_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.gate_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.up_proj.biases": "model-00038-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.up_proj.scales": "model-00038-of-00185.safetensors", + "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight": "model-00038-of-00185.safetensors", + "language_model.model.layers.19.mlp_res_norm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.mlp_res_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.post_attention_layernorm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.self_attention_res_norm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.self_attention_res_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.19.self_attn.embed_q.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.embed_q.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.embed_q.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.g_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.g_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.g_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.kv_a_layernorm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.o_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.o_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.o_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_a_layernorm.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_a_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_a_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_a_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_b_proj.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_b_proj.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.q_b_proj.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.unembed_out.biases": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.unembed_out.scales": "model-00037-of-00185.safetensors", + "language_model.model.layers.19.self_attn.unembed_out.weight": "model-00037-of-00185.safetensors", + "language_model.model.layers.2.input_layernorm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.e_score_correction_bias": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.gate.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.gate.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.gate.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_down_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_down_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_down_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_norm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_up_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_up_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.routed_expert_up_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.down_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.down_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.down_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.gate_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.gate_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.gate_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.up_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.up_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.shared_experts.up_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.down_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.down_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight": "model-00004-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.up_proj.biases": "model-00004-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.up_proj.scales": "model-00004-of-00185.safetensors", + "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight": "model-00004-of-00185.safetensors", + "language_model.model.layers.2.mlp_res_norm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.mlp_res_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.post_attention_layernorm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.self_attention_res_norm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.self_attention_res_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.2.self_attn.A_log": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.b_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.b_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.b_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.dt_bias": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_a_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_a_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_a_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_b_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_b_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.f_b_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.g_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.g_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.g_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.o_norm.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.o_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.o_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.o_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.qkv_conv.conv.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.qkv_proj.biases": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.qkv_proj.scales": "model-00003-of-00185.safetensors", + "language_model.model.layers.2.self_attn.qkv_proj.weight": "model-00003-of-00185.safetensors", + "language_model.model.layers.20.input_layernorm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.e_score_correction_bias": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.gate.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.gate.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.gate.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_down_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_down_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_down_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_norm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_up_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_up_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.routed_expert_up_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.down_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.down_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.down_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.gate_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.gate_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.up_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.up_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.shared_experts.up_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.down_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.down_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight": "model-00040-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.gate_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.gate_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.up_proj.biases": "model-00040-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.up_proj.scales": "model-00040-of-00185.safetensors", + "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight": "model-00040-of-00185.safetensors", + "language_model.model.layers.20.mlp_res_norm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.mlp_res_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.post_attention_layernorm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.self_attention_res_norm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.self_attention_res_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.20.self_attn.A_log": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.b_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.b_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.b_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.dt_bias": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_a_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_a_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_a_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_b_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_b_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.f_b_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.g_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.g_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.g_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.o_norm.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.o_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.o_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.o_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.qkv_conv.conv.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.qkv_proj.biases": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.qkv_proj.scales": "model-00039-of-00185.safetensors", + "language_model.model.layers.20.self_attn.qkv_proj.weight": "model-00039-of-00185.safetensors", + "language_model.model.layers.21.input_layernorm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.e_score_correction_bias": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.gate.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.gate.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.gate.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_down_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_down_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_down_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_norm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_up_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_up_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.routed_expert_up_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.down_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.down_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.down_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.gate_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.gate_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.up_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.up_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.shared_experts.up_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.down_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.down_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight": "model-00042-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.gate_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.gate_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.up_proj.biases": "model-00042-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.up_proj.scales": "model-00042-of-00185.safetensors", + "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight": "model-00042-of-00185.safetensors", + "language_model.model.layers.21.mlp_res_norm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.mlp_res_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.post_attention_layernorm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.self_attention_res_norm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.self_attention_res_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.21.self_attn.A_log": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.b_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.b_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.b_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.dt_bias": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_a_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_a_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_a_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_b_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_b_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.f_b_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.g_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.g_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.g_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.o_norm.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.o_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.o_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.o_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.qkv_conv.conv.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.qkv_proj.biases": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.qkv_proj.scales": "model-00041-of-00185.safetensors", + "language_model.model.layers.21.self_attn.qkv_proj.weight": "model-00041-of-00185.safetensors", + "language_model.model.layers.22.input_layernorm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.e_score_correction_bias": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.gate.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.gate.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.gate.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_down_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_down_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_down_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_norm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_up_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_up_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.routed_expert_up_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.down_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.down_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.down_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.gate_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.gate_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.up_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.up_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.shared_experts.up_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.down_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.down_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight": "model-00044-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.gate_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.gate_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.up_proj.biases": "model-00044-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.up_proj.scales": "model-00044-of-00185.safetensors", + "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight": "model-00044-of-00185.safetensors", + "language_model.model.layers.22.mlp_res_norm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.mlp_res_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.post_attention_layernorm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.self_attention_res_norm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.self_attention_res_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.22.self_attn.A_log": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.b_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.b_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.b_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.dt_bias": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_a_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_a_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_a_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_b_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_b_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.f_b_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.g_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.g_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.g_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.o_norm.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.o_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.o_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.o_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.qkv_conv.conv.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.qkv_proj.biases": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.qkv_proj.scales": "model-00043-of-00185.safetensors", + "language_model.model.layers.22.self_attn.qkv_proj.weight": "model-00043-of-00185.safetensors", + "language_model.model.layers.23.input_layernorm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.e_score_correction_bias": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.gate.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.gate.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.gate.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_down_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_down_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_down_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_norm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_up_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_up_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.routed_expert_up_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.down_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.down_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.down_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.gate_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.gate_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.up_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.up_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.shared_experts.up_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.down_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.down_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight": "model-00046-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.gate_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.gate_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.up_proj.biases": "model-00046-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.up_proj.scales": "model-00046-of-00185.safetensors", + "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight": "model-00046-of-00185.safetensors", + "language_model.model.layers.23.mlp_res_norm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.mlp_res_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.post_attention_layernorm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.self_attention_res_norm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.self_attention_res_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.23.self_attn.embed_q.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.embed_q.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.embed_q.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.g_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.g_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.g_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.kv_a_layernorm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.o_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.o_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.o_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_a_layernorm.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_a_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_a_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_a_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_b_proj.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_b_proj.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.q_b_proj.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.unembed_out.biases": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.unembed_out.scales": "model-00045-of-00185.safetensors", + "language_model.model.layers.23.self_attn.unembed_out.weight": "model-00045-of-00185.safetensors", + "language_model.model.layers.24.input_layernorm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.e_score_correction_bias": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.gate.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.gate.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.gate.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_down_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_down_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_down_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_norm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_up_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_up_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.routed_expert_up_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.down_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.down_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.down_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.gate_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.gate_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.gate_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.up_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.up_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.shared_experts.up_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.down_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.down_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight": "model-00048-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.gate_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.gate_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.up_proj.biases": "model-00048-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.up_proj.scales": "model-00048-of-00185.safetensors", + "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight": "model-00048-of-00185.safetensors", + "language_model.model.layers.24.mlp_res_norm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.mlp_res_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.post_attention_layernorm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.self_attention_res_norm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.self_attention_res_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.24.self_attn.A_log": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.b_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.b_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.b_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.dt_bias": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_a_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_a_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_a_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_b_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_b_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.f_b_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.g_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.g_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.g_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.o_norm.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.o_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.o_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.o_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.qkv_conv.conv.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.qkv_proj.biases": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.qkv_proj.scales": "model-00047-of-00185.safetensors", + "language_model.model.layers.24.self_attn.qkv_proj.weight": "model-00047-of-00185.safetensors", + "language_model.model.layers.25.input_layernorm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.e_score_correction_bias": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.gate.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.gate.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.gate.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_down_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_down_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_down_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_norm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_up_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_up_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.routed_expert_up_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.down_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.down_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.down_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.gate_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.gate_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.gate_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.up_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.up_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.shared_experts.up_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.down_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.down_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight": "model-00050-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.gate_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.gate_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.up_proj.biases": "model-00050-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.up_proj.scales": "model-00050-of-00185.safetensors", + "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight": "model-00050-of-00185.safetensors", + "language_model.model.layers.25.mlp_res_norm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.mlp_res_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.post_attention_layernorm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.self_attention_res_norm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.self_attention_res_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.25.self_attn.A_log": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.b_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.b_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.b_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.dt_bias": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_a_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_a_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_a_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_b_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_b_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.f_b_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.g_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.g_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.g_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.o_norm.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.o_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.o_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.o_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.qkv_conv.conv.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.qkv_proj.biases": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.qkv_proj.scales": "model-00049-of-00185.safetensors", + "language_model.model.layers.25.self_attn.qkv_proj.weight": "model-00049-of-00185.safetensors", + "language_model.model.layers.26.input_layernorm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.e_score_correction_bias": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.gate.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.gate.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.gate.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_down_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_down_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_down_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_norm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_up_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_up_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.routed_expert_up_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.down_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.down_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.down_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.gate_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.gate_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.gate_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.up_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.up_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.shared_experts.up_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.down_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.down_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight": "model-00052-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.gate_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.gate_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.up_proj.biases": "model-00052-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.up_proj.scales": "model-00052-of-00185.safetensors", + "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight": "model-00052-of-00185.safetensors", + "language_model.model.layers.26.mlp_res_norm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.mlp_res_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.post_attention_layernorm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.self_attention_res_norm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.self_attention_res_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.26.self_attn.A_log": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.b_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.b_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.b_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.dt_bias": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_a_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_a_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_a_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_b_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_b_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.f_b_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.g_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.g_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.g_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.o_norm.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.o_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.o_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.o_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.qkv_conv.conv.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.qkv_proj.biases": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.qkv_proj.scales": "model-00051-of-00185.safetensors", + "language_model.model.layers.26.self_attn.qkv_proj.weight": "model-00051-of-00185.safetensors", + "language_model.model.layers.27.input_layernorm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.e_score_correction_bias": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.gate.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.gate.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.gate.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_down_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_down_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_down_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_norm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_up_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_up_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.routed_expert_up_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.down_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.down_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.down_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.gate_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.gate_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.gate_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.up_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.up_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.shared_experts.up_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.down_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.down_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight": "model-00054-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.gate_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.gate_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.up_proj.biases": "model-00054-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.up_proj.scales": "model-00054-of-00185.safetensors", + "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight": "model-00054-of-00185.safetensors", + "language_model.model.layers.27.mlp_res_norm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.mlp_res_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.post_attention_layernorm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.self_attention_res_norm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.self_attention_res_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.27.self_attn.embed_q.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.embed_q.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.embed_q.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.g_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.g_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.g_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.kv_a_layernorm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.o_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.o_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.o_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_a_layernorm.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_a_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_a_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_a_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_b_proj.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_b_proj.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.q_b_proj.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.unembed_out.biases": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.unembed_out.scales": "model-00053-of-00185.safetensors", + "language_model.model.layers.27.self_attn.unembed_out.weight": "model-00053-of-00185.safetensors", + "language_model.model.layers.28.input_layernorm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.e_score_correction_bias": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.gate.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.gate.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.gate.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_down_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_down_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_down_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_norm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_up_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_up_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.routed_expert_up_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.down_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.down_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.down_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.gate_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.gate_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.gate_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.up_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.up_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.shared_experts.up_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.down_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.down_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight": "model-00056-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.gate_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.gate_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.up_proj.biases": "model-00056-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.up_proj.scales": "model-00056-of-00185.safetensors", + "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight": "model-00056-of-00185.safetensors", + "language_model.model.layers.28.mlp_res_norm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.mlp_res_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.post_attention_layernorm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.self_attention_res_norm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.self_attention_res_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.28.self_attn.A_log": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.b_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.b_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.b_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.dt_bias": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_a_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_a_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_a_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_b_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_b_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.f_b_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.g_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.g_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.g_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.o_norm.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.o_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.o_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.o_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.qkv_conv.conv.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.qkv_proj.biases": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.qkv_proj.scales": "model-00055-of-00185.safetensors", + "language_model.model.layers.28.self_attn.qkv_proj.weight": "model-00055-of-00185.safetensors", + "language_model.model.layers.29.input_layernorm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.e_score_correction_bias": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.gate.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.gate.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.gate.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_down_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_down_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_down_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_norm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_up_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_up_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.routed_expert_up_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.down_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.down_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.down_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.gate_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.gate_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.gate_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.up_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.up_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.shared_experts.up_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.down_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.down_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight": "model-00058-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.gate_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.gate_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.up_proj.biases": "model-00058-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.up_proj.scales": "model-00058-of-00185.safetensors", + "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight": "model-00058-of-00185.safetensors", + "language_model.model.layers.29.mlp_res_norm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.mlp_res_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.post_attention_layernorm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.self_attention_res_norm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.self_attention_res_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.29.self_attn.A_log": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.b_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.b_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.b_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.dt_bias": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_a_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_a_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_a_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_b_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_b_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.f_b_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.g_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.g_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.g_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.o_norm.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.o_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.o_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.o_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.qkv_conv.conv.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.qkv_proj.biases": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.qkv_proj.scales": "model-00057-of-00185.safetensors", + "language_model.model.layers.29.self_attn.qkv_proj.weight": "model-00057-of-00185.safetensors", + "language_model.model.layers.3.input_layernorm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.e_score_correction_bias": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.gate.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.gate.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.gate.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_down_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_down_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_down_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_norm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_up_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_up_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.routed_expert_up_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.down_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.down_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.down_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.gate_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.gate_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.up_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.up_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.shared_experts.up_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.down_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.down_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight": "model-00006-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.gate_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.gate_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.up_proj.biases": "model-00006-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.up_proj.scales": "model-00006-of-00185.safetensors", + "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight": "model-00006-of-00185.safetensors", + "language_model.model.layers.3.mlp_res_norm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.mlp_res_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.post_attention_layernorm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.self_attention_res_norm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.self_attention_res_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.3.self_attn.embed_q.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.embed_q.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.embed_q.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.g_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.g_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.g_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.kv_a_layernorm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.o_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.o_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.o_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_a_layernorm.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_a_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_a_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_a_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_b_proj.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_b_proj.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.q_b_proj.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.unembed_out.biases": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.unembed_out.scales": "model-00005-of-00185.safetensors", + "language_model.model.layers.3.self_attn.unembed_out.weight": "model-00005-of-00185.safetensors", + "language_model.model.layers.30.input_layernorm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.e_score_correction_bias": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.gate.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.gate.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.gate.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_down_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_down_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_down_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_norm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_up_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_up_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.routed_expert_up_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.down_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.down_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.down_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.gate_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.gate_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.gate_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.up_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.up_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.shared_experts.up_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.down_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.down_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight": "model-00060-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.gate_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.gate_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.up_proj.biases": "model-00060-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.up_proj.scales": "model-00060-of-00185.safetensors", + "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight": "model-00060-of-00185.safetensors", + "language_model.model.layers.30.mlp_res_norm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.mlp_res_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.post_attention_layernorm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.self_attention_res_norm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.self_attention_res_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.30.self_attn.A_log": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.b_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.b_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.b_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.dt_bias": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_a_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_a_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_a_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_b_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_b_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.f_b_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.g_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.g_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.g_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.o_norm.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.o_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.o_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.o_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.qkv_conv.conv.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.qkv_proj.biases": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.qkv_proj.scales": "model-00059-of-00185.safetensors", + "language_model.model.layers.30.self_attn.qkv_proj.weight": "model-00059-of-00185.safetensors", + "language_model.model.layers.31.input_layernorm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.e_score_correction_bias": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.gate.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.gate.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.gate.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_down_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_down_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_down_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_norm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_up_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_up_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.routed_expert_up_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.down_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.down_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.down_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.gate_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.gate_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.gate_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.up_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.up_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.shared_experts.up_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.down_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.down_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight": "model-00062-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.gate_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.gate_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.up_proj.biases": "model-00062-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.up_proj.scales": "model-00062-of-00185.safetensors", + "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight": "model-00062-of-00185.safetensors", + "language_model.model.layers.31.mlp_res_norm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.mlp_res_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.post_attention_layernorm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.self_attention_res_norm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.self_attention_res_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.31.self_attn.embed_q.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.embed_q.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.embed_q.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.g_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.g_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.g_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.kv_a_layernorm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.o_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.o_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.o_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_a_layernorm.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_a_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_a_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_a_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_b_proj.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_b_proj.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.q_b_proj.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.unembed_out.biases": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.unembed_out.scales": "model-00061-of-00185.safetensors", + "language_model.model.layers.31.self_attn.unembed_out.weight": "model-00061-of-00185.safetensors", + "language_model.model.layers.32.input_layernorm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.e_score_correction_bias": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.gate.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.gate.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.gate.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_down_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_down_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_down_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_norm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_up_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_up_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.routed_expert_up_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.down_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.down_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.down_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.gate_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.gate_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.gate_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.up_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.up_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.shared_experts.up_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.down_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.down_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight": "model-00064-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.gate_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.gate_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.up_proj.biases": "model-00064-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.up_proj.scales": "model-00064-of-00185.safetensors", + "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight": "model-00064-of-00185.safetensors", + "language_model.model.layers.32.mlp_res_norm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.mlp_res_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.post_attention_layernorm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.self_attention_res_norm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.self_attention_res_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.32.self_attn.A_log": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.b_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.b_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.b_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.dt_bias": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_a_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_a_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_a_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_b_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_b_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.f_b_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.g_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.g_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.g_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.o_norm.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.o_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.o_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.o_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.qkv_conv.conv.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.qkv_proj.biases": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.qkv_proj.scales": "model-00063-of-00185.safetensors", + "language_model.model.layers.32.self_attn.qkv_proj.weight": "model-00063-of-00185.safetensors", + "language_model.model.layers.33.input_layernorm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.e_score_correction_bias": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.gate.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.gate.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.gate.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_down_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_down_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_down_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_norm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_up_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_up_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.routed_expert_up_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.down_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.down_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.down_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.gate_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.gate_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.gate_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.up_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.up_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.shared_experts.up_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.down_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.down_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight": "model-00066-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.gate_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.gate_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.up_proj.biases": "model-00066-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.up_proj.scales": "model-00066-of-00185.safetensors", + "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight": "model-00066-of-00185.safetensors", + "language_model.model.layers.33.mlp_res_norm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.mlp_res_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.post_attention_layernorm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.self_attention_res_norm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.self_attention_res_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.33.self_attn.A_log": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.b_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.b_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.b_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.dt_bias": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_a_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_a_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_a_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_b_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_b_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.f_b_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.g_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.g_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.g_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.o_norm.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.o_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.o_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.o_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.qkv_conv.conv.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.qkv_proj.biases": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.qkv_proj.scales": "model-00065-of-00185.safetensors", + "language_model.model.layers.33.self_attn.qkv_proj.weight": "model-00065-of-00185.safetensors", + "language_model.model.layers.34.input_layernorm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.e_score_correction_bias": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.gate.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.gate.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.gate.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_down_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_down_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_down_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_norm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_up_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_up_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.routed_expert_up_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.down_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.down_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.down_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.gate_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.gate_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.gate_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.up_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.up_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.shared_experts.up_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.down_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.down_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight": "model-00068-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.gate_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.gate_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.up_proj.biases": "model-00068-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.up_proj.scales": "model-00068-of-00185.safetensors", + "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight": "model-00068-of-00185.safetensors", + "language_model.model.layers.34.mlp_res_norm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.mlp_res_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.post_attention_layernorm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.self_attention_res_norm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.self_attention_res_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.34.self_attn.A_log": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.b_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.b_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.b_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.dt_bias": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_a_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_a_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_a_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_b_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_b_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.f_b_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.g_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.g_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.g_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.o_norm.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.o_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.o_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.o_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.qkv_conv.conv.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.qkv_proj.biases": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.qkv_proj.scales": "model-00067-of-00185.safetensors", + "language_model.model.layers.34.self_attn.qkv_proj.weight": "model-00067-of-00185.safetensors", + "language_model.model.layers.35.input_layernorm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.e_score_correction_bias": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.gate.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.gate.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.gate.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_down_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_down_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_down_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_norm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_up_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_up_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.routed_expert_up_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.down_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.down_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.down_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.gate_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.gate_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.gate_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.up_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.up_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.shared_experts.up_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.down_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.down_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight": "model-00070-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.gate_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.gate_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.up_proj.biases": "model-00070-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.up_proj.scales": "model-00070-of-00185.safetensors", + "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight": "model-00070-of-00185.safetensors", + "language_model.model.layers.35.mlp_res_norm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.mlp_res_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.post_attention_layernorm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.self_attention_res_norm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.self_attention_res_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.35.self_attn.embed_q.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.embed_q.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.embed_q.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.g_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.g_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.g_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.kv_a_layernorm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.o_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.o_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.o_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_a_layernorm.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_a_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_a_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_a_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_b_proj.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_b_proj.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.q_b_proj.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.unembed_out.biases": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.unembed_out.scales": "model-00069-of-00185.safetensors", + "language_model.model.layers.35.self_attn.unembed_out.weight": "model-00069-of-00185.safetensors", + "language_model.model.layers.36.input_layernorm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.e_score_correction_bias": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.gate.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.gate.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.gate.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_down_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_down_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_down_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_norm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_up_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_up_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.routed_expert_up_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.down_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.down_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.down_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.gate_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.gate_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.gate_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.up_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.up_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.shared_experts.up_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.down_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.down_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight": "model-00072-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.gate_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.gate_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.up_proj.biases": "model-00072-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.up_proj.scales": "model-00072-of-00185.safetensors", + "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight": "model-00072-of-00185.safetensors", + "language_model.model.layers.36.mlp_res_norm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.mlp_res_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.post_attention_layernorm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.self_attention_res_norm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.self_attention_res_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.36.self_attn.A_log": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.b_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.b_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.b_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.dt_bias": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_a_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_a_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_a_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_b_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_b_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.f_b_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.g_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.g_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.g_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.o_norm.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.o_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.o_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.o_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.qkv_conv.conv.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.qkv_proj.biases": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.qkv_proj.scales": "model-00071-of-00185.safetensors", + "language_model.model.layers.36.self_attn.qkv_proj.weight": "model-00071-of-00185.safetensors", + "language_model.model.layers.37.input_layernorm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.e_score_correction_bias": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.gate.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.gate.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.gate.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_down_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_down_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_down_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_norm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_up_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_up_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.routed_expert_up_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.down_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.down_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.down_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.gate_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.gate_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.gate_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.up_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.up_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.shared_experts.up_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.down_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.down_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight": "model-00074-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.gate_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.gate_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.up_proj.biases": "model-00074-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.up_proj.scales": "model-00074-of-00185.safetensors", + "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight": "model-00074-of-00185.safetensors", + "language_model.model.layers.37.mlp_res_norm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.mlp_res_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.post_attention_layernorm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.self_attention_res_norm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.self_attention_res_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.37.self_attn.A_log": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.b_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.b_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.b_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.dt_bias": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_a_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_a_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_a_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_b_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_b_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.f_b_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.g_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.g_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.g_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.o_norm.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.o_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.o_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.o_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.qkv_conv.conv.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.qkv_proj.biases": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.qkv_proj.scales": "model-00073-of-00185.safetensors", + "language_model.model.layers.37.self_attn.qkv_proj.weight": "model-00073-of-00185.safetensors", + "language_model.model.layers.38.input_layernorm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.e_score_correction_bias": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.gate.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.gate.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.gate.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_down_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_down_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_down_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_norm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_up_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_up_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.routed_expert_up_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.down_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.down_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.down_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.gate_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.gate_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.gate_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.up_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.up_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.shared_experts.up_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.down_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.down_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight": "model-00076-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.gate_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.gate_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.up_proj.biases": "model-00076-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.up_proj.scales": "model-00076-of-00185.safetensors", + "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight": "model-00076-of-00185.safetensors", + "language_model.model.layers.38.mlp_res_norm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.mlp_res_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.post_attention_layernorm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.self_attention_res_norm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.self_attention_res_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.38.self_attn.A_log": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.b_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.b_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.b_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.dt_bias": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_a_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_a_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_a_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_b_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_b_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.f_b_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.g_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.g_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.g_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.o_norm.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.o_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.o_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.o_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.qkv_conv.conv.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.qkv_proj.biases": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.qkv_proj.scales": "model-00075-of-00185.safetensors", + "language_model.model.layers.38.self_attn.qkv_proj.weight": "model-00075-of-00185.safetensors", + "language_model.model.layers.39.input_layernorm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.e_score_correction_bias": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.gate.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.gate.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.gate.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_down_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_down_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_down_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_norm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_up_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_up_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.routed_expert_up_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.down_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.down_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.down_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.gate_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.gate_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.gate_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.up_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.up_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.shared_experts.up_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.down_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.down_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight": "model-00078-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.gate_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.gate_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.up_proj.biases": "model-00078-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.up_proj.scales": "model-00078-of-00185.safetensors", + "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight": "model-00078-of-00185.safetensors", + "language_model.model.layers.39.mlp_res_norm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.mlp_res_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.post_attention_layernorm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.self_attention_res_norm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.self_attention_res_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.39.self_attn.embed_q.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.embed_q.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.embed_q.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.g_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.g_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.g_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.kv_a_layernorm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.o_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.o_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.o_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_a_layernorm.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_a_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_a_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_a_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_b_proj.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_b_proj.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.q_b_proj.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.unembed_out.biases": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.unembed_out.scales": "model-00077-of-00185.safetensors", + "language_model.model.layers.39.self_attn.unembed_out.weight": "model-00077-of-00185.safetensors", + "language_model.model.layers.4.input_layernorm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.e_score_correction_bias": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.gate.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.gate.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.gate.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_down_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_down_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_down_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_norm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_up_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_up_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.routed_expert_up_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.down_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.down_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.down_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.gate_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.gate_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.up_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.up_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.shared_experts.up_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.down_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.down_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight": "model-00008-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.gate_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.gate_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.up_proj.biases": "model-00008-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.up_proj.scales": "model-00008-of-00185.safetensors", + "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight": "model-00008-of-00185.safetensors", + "language_model.model.layers.4.mlp_res_norm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.mlp_res_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.post_attention_layernorm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.self_attention_res_norm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.self_attention_res_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.4.self_attn.A_log": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.b_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.b_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.b_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.dt_bias": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_a_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_a_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_a_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_b_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_b_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.f_b_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.g_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.g_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.g_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.o_norm.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.o_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.o_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.o_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.qkv_conv.conv.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.qkv_proj.biases": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.qkv_proj.scales": "model-00007-of-00185.safetensors", + "language_model.model.layers.4.self_attn.qkv_proj.weight": "model-00007-of-00185.safetensors", + "language_model.model.layers.40.input_layernorm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.e_score_correction_bias": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.gate.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.gate.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.gate.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_down_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_down_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_down_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_norm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_up_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_up_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.routed_expert_up_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.down_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.down_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.down_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.gate_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.gate_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.gate_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.up_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.up_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.shared_experts.up_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.down_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.down_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.down_proj.weight": "model-00080-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.gate_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.gate_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.up_proj.biases": "model-00080-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.up_proj.scales": "model-00080-of-00185.safetensors", + "language_model.model.layers.40.mlp.switch_mlp.up_proj.weight": "model-00080-of-00185.safetensors", + "language_model.model.layers.40.mlp_res_norm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.mlp_res_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.post_attention_layernorm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.self_attention_res_norm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.self_attention_res_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.40.self_attn.A_log": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.b_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.b_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.b_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.dt_bias": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_a_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_a_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_a_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_b_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_b_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.f_b_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.g_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.g_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.g_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.o_norm.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.o_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.o_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.o_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.qkv_conv.conv.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.qkv_proj.biases": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.qkv_proj.scales": "model-00079-of-00185.safetensors", + "language_model.model.layers.40.self_attn.qkv_proj.weight": "model-00079-of-00185.safetensors", + "language_model.model.layers.41.input_layernorm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.e_score_correction_bias": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.gate.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.gate.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.gate.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_down_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_down_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_down_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_norm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_up_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_up_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.routed_expert_up_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.down_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.down_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.down_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.gate_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.gate_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.gate_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.up_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.up_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.shared_experts.up_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.down_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.down_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.down_proj.weight": "model-00082-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.gate_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.gate_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.up_proj.biases": "model-00082-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.up_proj.scales": "model-00082-of-00185.safetensors", + "language_model.model.layers.41.mlp.switch_mlp.up_proj.weight": "model-00082-of-00185.safetensors", + "language_model.model.layers.41.mlp_res_norm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.mlp_res_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.post_attention_layernorm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.self_attention_res_norm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.self_attention_res_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.41.self_attn.A_log": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.b_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.b_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.b_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.dt_bias": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_a_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_a_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_a_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_b_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_b_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.f_b_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.g_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.g_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.g_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.o_norm.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.o_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.o_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.o_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.qkv_conv.conv.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.qkv_proj.biases": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.qkv_proj.scales": "model-00081-of-00185.safetensors", + "language_model.model.layers.41.self_attn.qkv_proj.weight": "model-00081-of-00185.safetensors", + "language_model.model.layers.42.input_layernorm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.e_score_correction_bias": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.gate.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.gate.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.gate.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_down_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_down_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_down_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_norm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_up_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_up_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.routed_expert_up_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.down_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.down_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.down_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.gate_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.gate_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.gate_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.up_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.up_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.shared_experts.up_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.down_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.down_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.down_proj.weight": "model-00084-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.gate_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.gate_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.up_proj.biases": "model-00084-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.up_proj.scales": "model-00084-of-00185.safetensors", + "language_model.model.layers.42.mlp.switch_mlp.up_proj.weight": "model-00084-of-00185.safetensors", + "language_model.model.layers.42.mlp_res_norm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.mlp_res_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.post_attention_layernorm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.self_attention_res_norm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.self_attention_res_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.42.self_attn.A_log": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.b_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.b_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.b_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.dt_bias": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_a_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_a_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_a_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_b_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_b_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.f_b_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.g_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.g_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.g_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.o_norm.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.o_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.o_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.o_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.qkv_conv.conv.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.qkv_proj.biases": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.qkv_proj.scales": "model-00083-of-00185.safetensors", + "language_model.model.layers.42.self_attn.qkv_proj.weight": "model-00083-of-00185.safetensors", + "language_model.model.layers.43.input_layernorm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.e_score_correction_bias": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.gate.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.gate.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.gate.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_down_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_down_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_down_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_norm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_up_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_up_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.routed_expert_up_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.down_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.down_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.down_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.gate_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.gate_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.gate_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.up_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.up_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.shared_experts.up_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.down_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.down_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.down_proj.weight": "model-00086-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.gate_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.gate_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.up_proj.biases": "model-00086-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.up_proj.scales": "model-00086-of-00185.safetensors", + "language_model.model.layers.43.mlp.switch_mlp.up_proj.weight": "model-00086-of-00185.safetensors", + "language_model.model.layers.43.mlp_res_norm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.mlp_res_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.post_attention_layernorm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.self_attention_res_norm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.self_attention_res_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.43.self_attn.embed_q.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.embed_q.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.embed_q.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.g_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.g_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.g_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.kv_a_layernorm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.o_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.o_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.o_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_a_layernorm.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_a_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_a_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_a_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_b_proj.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_b_proj.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.q_b_proj.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.unembed_out.biases": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.unembed_out.scales": "model-00085-of-00185.safetensors", + "language_model.model.layers.43.self_attn.unembed_out.weight": "model-00085-of-00185.safetensors", + "language_model.model.layers.44.input_layernorm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.e_score_correction_bias": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.gate.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.gate.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.gate.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_down_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_down_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_down_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_norm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_up_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_up_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.routed_expert_up_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.down_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.down_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.down_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.gate_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.gate_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.gate_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.up_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.up_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.shared_experts.up_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.down_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.down_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.down_proj.weight": "model-00088-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.gate_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.gate_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.up_proj.biases": "model-00088-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.up_proj.scales": "model-00088-of-00185.safetensors", + "language_model.model.layers.44.mlp.switch_mlp.up_proj.weight": "model-00088-of-00185.safetensors", + "language_model.model.layers.44.mlp_res_norm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.mlp_res_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.post_attention_layernorm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.self_attention_res_norm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.self_attention_res_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.44.self_attn.A_log": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.b_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.b_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.b_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.dt_bias": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_a_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_a_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_a_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_b_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_b_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.f_b_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.g_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.g_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.g_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.o_norm.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.o_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.o_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.o_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.qkv_conv.conv.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.qkv_proj.biases": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.qkv_proj.scales": "model-00087-of-00185.safetensors", + "language_model.model.layers.44.self_attn.qkv_proj.weight": "model-00087-of-00185.safetensors", + "language_model.model.layers.45.input_layernorm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.e_score_correction_bias": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.gate.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.gate.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.gate.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_down_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_down_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_down_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_norm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_up_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_up_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.routed_expert_up_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.down_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.down_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.down_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.gate_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.gate_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.gate_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.up_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.up_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.shared_experts.up_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.down_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.down_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.down_proj.weight": "model-00090-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.gate_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.gate_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.up_proj.biases": "model-00090-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.up_proj.scales": "model-00090-of-00185.safetensors", + "language_model.model.layers.45.mlp.switch_mlp.up_proj.weight": "model-00090-of-00185.safetensors", + "language_model.model.layers.45.mlp_res_norm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.mlp_res_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.post_attention_layernorm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.self_attention_res_norm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.self_attention_res_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.45.self_attn.A_log": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.b_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.b_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.b_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.dt_bias": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_a_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_a_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_a_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_b_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_b_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.f_b_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.g_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.g_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.g_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.o_norm.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.o_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.o_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.o_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.qkv_conv.conv.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.qkv_proj.biases": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.qkv_proj.scales": "model-00089-of-00185.safetensors", + "language_model.model.layers.45.self_attn.qkv_proj.weight": "model-00089-of-00185.safetensors", + "language_model.model.layers.46.input_layernorm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.e_score_correction_bias": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.gate.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.gate.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.gate.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_down_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_down_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_down_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_norm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_up_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_up_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.routed_expert_up_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.down_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.down_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.down_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.gate_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.gate_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.gate_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.up_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.up_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.shared_experts.up_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.down_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.down_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.down_proj.weight": "model-00092-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.gate_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.gate_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.up_proj.biases": "model-00092-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.up_proj.scales": "model-00092-of-00185.safetensors", + "language_model.model.layers.46.mlp.switch_mlp.up_proj.weight": "model-00092-of-00185.safetensors", + "language_model.model.layers.46.mlp_res_norm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.mlp_res_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.post_attention_layernorm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.self_attention_res_norm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.self_attention_res_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.46.self_attn.A_log": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.b_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.b_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.b_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.dt_bias": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_a_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_a_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_a_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_b_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_b_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.f_b_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.g_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.g_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.g_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.o_norm.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.o_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.o_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.o_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.qkv_conv.conv.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.qkv_proj.biases": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.qkv_proj.scales": "model-00091-of-00185.safetensors", + "language_model.model.layers.46.self_attn.qkv_proj.weight": "model-00091-of-00185.safetensors", + "language_model.model.layers.47.input_layernorm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.e_score_correction_bias": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.gate.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.gate.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.gate.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_down_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_down_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_down_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_norm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_up_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_up_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.routed_expert_up_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.down_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.down_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.down_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.gate_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.gate_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.gate_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.up_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.up_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.shared_experts.up_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.down_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.down_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.down_proj.weight": "model-00094-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.gate_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.gate_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.up_proj.biases": "model-00094-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.up_proj.scales": "model-00094-of-00185.safetensors", + "language_model.model.layers.47.mlp.switch_mlp.up_proj.weight": "model-00094-of-00185.safetensors", + "language_model.model.layers.47.mlp_res_norm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.mlp_res_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.post_attention_layernorm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.self_attention_res_norm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.self_attention_res_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.47.self_attn.embed_q.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.embed_q.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.embed_q.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.g_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.g_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.g_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.kv_a_layernorm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.o_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.o_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.o_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_a_layernorm.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_a_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_a_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_a_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_b_proj.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_b_proj.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.q_b_proj.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.unembed_out.biases": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.unembed_out.scales": "model-00093-of-00185.safetensors", + "language_model.model.layers.47.self_attn.unembed_out.weight": "model-00093-of-00185.safetensors", + "language_model.model.layers.48.input_layernorm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.e_score_correction_bias": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.gate.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.gate.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.gate.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_down_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_down_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_down_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_norm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_up_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_up_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.routed_expert_up_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.down_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.down_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.down_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.gate_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.gate_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.gate_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.up_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.up_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.shared_experts.up_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.down_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.down_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.down_proj.weight": "model-00096-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.gate_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.gate_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.gate_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.up_proj.biases": "model-00096-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.up_proj.scales": "model-00096-of-00185.safetensors", + "language_model.model.layers.48.mlp.switch_mlp.up_proj.weight": "model-00096-of-00185.safetensors", + "language_model.model.layers.48.mlp_res_norm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.mlp_res_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.post_attention_layernorm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.self_attention_res_norm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.self_attention_res_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.48.self_attn.A_log": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.b_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.b_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.b_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.dt_bias": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_a_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_a_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_a_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_b_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_b_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.f_b_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.g_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.g_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.g_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.o_norm.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.o_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.o_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.o_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.qkv_conv.conv.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.qkv_proj.biases": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.qkv_proj.scales": "model-00095-of-00185.safetensors", + "language_model.model.layers.48.self_attn.qkv_proj.weight": "model-00095-of-00185.safetensors", + "language_model.model.layers.49.input_layernorm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.e_score_correction_bias": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.gate.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.gate.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.gate.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_down_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_down_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_down_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_norm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_up_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_up_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.routed_expert_up_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.down_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.down_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.down_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.gate_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.gate_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.gate_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.up_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.up_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.shared_experts.up_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.down_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.down_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.down_proj.weight": "model-00098-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.gate_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.gate_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.gate_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.up_proj.biases": "model-00098-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.up_proj.scales": "model-00098-of-00185.safetensors", + "language_model.model.layers.49.mlp.switch_mlp.up_proj.weight": "model-00098-of-00185.safetensors", + "language_model.model.layers.49.mlp_res_norm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.mlp_res_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.post_attention_layernorm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.self_attention_res_norm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.self_attention_res_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.49.self_attn.A_log": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.b_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.b_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.b_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.dt_bias": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_a_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_a_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_a_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_b_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_b_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.f_b_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.g_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.g_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.g_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.o_norm.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.o_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.o_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.o_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.qkv_conv.conv.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.qkv_proj.biases": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.qkv_proj.scales": "model-00097-of-00185.safetensors", + "language_model.model.layers.49.self_attn.qkv_proj.weight": "model-00097-of-00185.safetensors", + "language_model.model.layers.5.input_layernorm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.e_score_correction_bias": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.gate.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.gate.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.gate.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_down_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_down_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_down_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_norm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_up_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_up_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.routed_expert_up_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.down_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.down_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.down_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.gate_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.gate_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.up_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.up_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.shared_experts.up_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.down_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.down_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight": "model-00010-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.gate_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.gate_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.up_proj.biases": "model-00010-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.up_proj.scales": "model-00010-of-00185.safetensors", + "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight": "model-00010-of-00185.safetensors", + "language_model.model.layers.5.mlp_res_norm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.mlp_res_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.post_attention_layernorm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.self_attention_res_norm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.self_attention_res_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.5.self_attn.A_log": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.b_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.b_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.b_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.dt_bias": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_a_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_a_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_a_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_b_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_b_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.f_b_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.g_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.g_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.g_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.o_norm.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.o_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.o_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.o_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.qkv_conv.conv.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.qkv_proj.biases": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.qkv_proj.scales": "model-00009-of-00185.safetensors", + "language_model.model.layers.5.self_attn.qkv_proj.weight": "model-00009-of-00185.safetensors", + "language_model.model.layers.50.input_layernorm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.e_score_correction_bias": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.gate.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.gate.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.gate.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_down_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_down_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_down_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_norm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_up_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_up_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.routed_expert_up_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.down_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.down_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.down_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.gate_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.gate_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.gate_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.up_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.up_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.shared_experts.up_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.down_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.down_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.down_proj.weight": "model-00100-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.gate_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.gate_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.gate_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.up_proj.biases": "model-00100-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.up_proj.scales": "model-00100-of-00185.safetensors", + "language_model.model.layers.50.mlp.switch_mlp.up_proj.weight": "model-00100-of-00185.safetensors", + "language_model.model.layers.50.mlp_res_norm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.mlp_res_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.post_attention_layernorm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.self_attention_res_norm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.self_attention_res_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.50.self_attn.A_log": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.b_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.b_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.b_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.dt_bias": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_a_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_a_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_a_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_b_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_b_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.f_b_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.g_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.g_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.g_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.o_norm.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.o_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.o_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.o_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.qkv_conv.conv.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.qkv_proj.biases": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.qkv_proj.scales": "model-00099-of-00185.safetensors", + "language_model.model.layers.50.self_attn.qkv_proj.weight": "model-00099-of-00185.safetensors", + "language_model.model.layers.51.input_layernorm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.e_score_correction_bias": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.gate.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.gate.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.gate.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_down_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_down_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_down_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_norm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_up_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_up_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.routed_expert_up_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.down_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.down_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.down_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.gate_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.gate_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.gate_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.up_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.up_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.shared_experts.up_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.down_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.down_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.down_proj.weight": "model-00102-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.gate_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.gate_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.gate_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.up_proj.biases": "model-00102-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.up_proj.scales": "model-00102-of-00185.safetensors", + "language_model.model.layers.51.mlp.switch_mlp.up_proj.weight": "model-00102-of-00185.safetensors", + "language_model.model.layers.51.mlp_res_norm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.mlp_res_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.post_attention_layernorm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.self_attention_res_norm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.self_attention_res_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.51.self_attn.embed_q.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.embed_q.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.embed_q.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.g_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.g_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.g_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.kv_a_layernorm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.o_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.o_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.o_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_a_layernorm.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_a_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_a_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_a_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_b_proj.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_b_proj.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.q_b_proj.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.unembed_out.biases": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.unembed_out.scales": "model-00101-of-00185.safetensors", + "language_model.model.layers.51.self_attn.unembed_out.weight": "model-00101-of-00185.safetensors", + "language_model.model.layers.52.input_layernorm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.e_score_correction_bias": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.gate.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.gate.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.gate.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_down_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_down_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_down_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_norm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_up_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_up_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.routed_expert_up_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.down_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.down_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.down_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.gate_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.gate_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.gate_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.up_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.up_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.shared_experts.up_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.down_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.down_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.down_proj.weight": "model-00104-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.gate_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.gate_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.gate_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.up_proj.biases": "model-00104-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.up_proj.scales": "model-00104-of-00185.safetensors", + "language_model.model.layers.52.mlp.switch_mlp.up_proj.weight": "model-00104-of-00185.safetensors", + "language_model.model.layers.52.mlp_res_norm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.mlp_res_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.post_attention_layernorm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.self_attention_res_norm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.self_attention_res_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.52.self_attn.A_log": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.b_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.b_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.b_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.dt_bias": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_a_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_a_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_a_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_b_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_b_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.f_b_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.g_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.g_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.g_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.o_norm.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.o_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.o_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.o_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.qkv_conv.conv.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.qkv_proj.biases": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.qkv_proj.scales": "model-00103-of-00185.safetensors", + "language_model.model.layers.52.self_attn.qkv_proj.weight": "model-00103-of-00185.safetensors", + "language_model.model.layers.53.input_layernorm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.e_score_correction_bias": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.gate.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.gate.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.gate.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_down_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_down_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_down_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_norm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_up_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_up_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.routed_expert_up_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.down_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.down_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.down_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.gate_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.gate_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.gate_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.up_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.up_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.shared_experts.up_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.down_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.down_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.down_proj.weight": "model-00106-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.gate_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.gate_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.gate_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.up_proj.biases": "model-00106-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.up_proj.scales": "model-00106-of-00185.safetensors", + "language_model.model.layers.53.mlp.switch_mlp.up_proj.weight": "model-00106-of-00185.safetensors", + "language_model.model.layers.53.mlp_res_norm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.mlp_res_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.post_attention_layernorm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.self_attention_res_norm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.self_attention_res_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.53.self_attn.A_log": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.b_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.b_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.b_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.dt_bias": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_a_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_a_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_a_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_b_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_b_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.f_b_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.g_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.g_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.g_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.o_norm.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.o_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.o_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.o_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.qkv_conv.conv.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.qkv_proj.biases": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.qkv_proj.scales": "model-00105-of-00185.safetensors", + "language_model.model.layers.53.self_attn.qkv_proj.weight": "model-00105-of-00185.safetensors", + "language_model.model.layers.54.input_layernorm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.e_score_correction_bias": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.gate.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.gate.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.gate.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_down_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_down_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_down_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_norm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_up_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_up_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.routed_expert_up_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.down_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.down_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.down_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.gate_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.gate_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.gate_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.up_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.up_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.shared_experts.up_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.down_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.down_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.down_proj.weight": "model-00108-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.gate_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.gate_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.gate_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.up_proj.biases": "model-00108-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.up_proj.scales": "model-00108-of-00185.safetensors", + "language_model.model.layers.54.mlp.switch_mlp.up_proj.weight": "model-00108-of-00185.safetensors", + "language_model.model.layers.54.mlp_res_norm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.mlp_res_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.post_attention_layernorm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.self_attention_res_norm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.self_attention_res_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.54.self_attn.A_log": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.b_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.b_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.b_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.dt_bias": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_a_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_a_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_a_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_b_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_b_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.f_b_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.g_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.g_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.g_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.o_norm.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.o_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.o_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.o_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.qkv_conv.conv.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.qkv_proj.biases": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.qkv_proj.scales": "model-00107-of-00185.safetensors", + "language_model.model.layers.54.self_attn.qkv_proj.weight": "model-00107-of-00185.safetensors", + "language_model.model.layers.55.input_layernorm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.e_score_correction_bias": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.gate.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.gate.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.gate.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_down_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_down_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_down_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_norm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_up_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_up_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.routed_expert_up_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.down_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.down_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.down_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.gate_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.gate_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.gate_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.up_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.up_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.shared_experts.up_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.down_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.down_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.down_proj.weight": "model-00110-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.gate_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.gate_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.gate_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.up_proj.biases": "model-00110-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.up_proj.scales": "model-00110-of-00185.safetensors", + "language_model.model.layers.55.mlp.switch_mlp.up_proj.weight": "model-00110-of-00185.safetensors", + "language_model.model.layers.55.mlp_res_norm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.mlp_res_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.post_attention_layernorm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.self_attention_res_norm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.self_attention_res_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.55.self_attn.embed_q.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.embed_q.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.embed_q.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.g_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.g_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.g_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.kv_a_layernorm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.o_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.o_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.o_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_a_layernorm.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_a_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_a_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_a_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_b_proj.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_b_proj.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.q_b_proj.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.unembed_out.biases": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.unembed_out.scales": "model-00109-of-00185.safetensors", + "language_model.model.layers.55.self_attn.unembed_out.weight": "model-00109-of-00185.safetensors", + "language_model.model.layers.56.input_layernorm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.e_score_correction_bias": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.gate.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.gate.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.gate.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_down_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_down_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_down_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_norm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_up_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_up_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.routed_expert_up_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.down_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.down_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.down_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.gate_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.gate_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.gate_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.up_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.up_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.shared_experts.up_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.down_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.down_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.down_proj.weight": "model-00112-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.gate_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.gate_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.gate_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.up_proj.biases": "model-00112-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.up_proj.scales": "model-00112-of-00185.safetensors", + "language_model.model.layers.56.mlp.switch_mlp.up_proj.weight": "model-00112-of-00185.safetensors", + "language_model.model.layers.56.mlp_res_norm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.mlp_res_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.post_attention_layernorm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.self_attention_res_norm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.self_attention_res_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.56.self_attn.A_log": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.b_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.b_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.b_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.dt_bias": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_a_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_a_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_a_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_b_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_b_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.f_b_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.g_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.g_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.g_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.o_norm.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.o_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.o_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.o_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.qkv_conv.conv.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.qkv_proj.biases": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.qkv_proj.scales": "model-00111-of-00185.safetensors", + "language_model.model.layers.56.self_attn.qkv_proj.weight": "model-00111-of-00185.safetensors", + "language_model.model.layers.57.input_layernorm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.e_score_correction_bias": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.gate.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.gate.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.gate.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_down_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_down_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_down_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_norm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_up_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_up_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.routed_expert_up_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.down_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.down_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.down_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.gate_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.gate_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.gate_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.up_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.up_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.shared_experts.up_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.down_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.down_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.down_proj.weight": "model-00114-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.gate_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.gate_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.gate_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.up_proj.biases": "model-00114-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.up_proj.scales": "model-00114-of-00185.safetensors", + "language_model.model.layers.57.mlp.switch_mlp.up_proj.weight": "model-00114-of-00185.safetensors", + "language_model.model.layers.57.mlp_res_norm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.mlp_res_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.post_attention_layernorm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.self_attention_res_norm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.self_attention_res_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.57.self_attn.A_log": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.b_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.b_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.b_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.dt_bias": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_a_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_a_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_a_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_b_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_b_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.f_b_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.g_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.g_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.g_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.o_norm.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.o_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.o_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.o_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.qkv_conv.conv.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.qkv_proj.biases": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.qkv_proj.scales": "model-00113-of-00185.safetensors", + "language_model.model.layers.57.self_attn.qkv_proj.weight": "model-00113-of-00185.safetensors", + "language_model.model.layers.58.input_layernorm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.e_score_correction_bias": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.gate.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.gate.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.gate.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_down_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_down_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_down_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_norm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_up_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_up_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.routed_expert_up_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.down_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.down_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.down_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.gate_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.gate_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.gate_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.up_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.up_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.shared_experts.up_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.down_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.down_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.down_proj.weight": "model-00116-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.gate_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.gate_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.gate_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.up_proj.biases": "model-00116-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.up_proj.scales": "model-00116-of-00185.safetensors", + "language_model.model.layers.58.mlp.switch_mlp.up_proj.weight": "model-00116-of-00185.safetensors", + "language_model.model.layers.58.mlp_res_norm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.mlp_res_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.post_attention_layernorm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.self_attention_res_norm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.self_attention_res_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.58.self_attn.A_log": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.b_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.b_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.b_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.dt_bias": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_a_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_a_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_a_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_b_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_b_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.f_b_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.g_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.g_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.g_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.o_norm.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.o_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.o_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.o_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.qkv_conv.conv.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.qkv_proj.biases": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.qkv_proj.scales": "model-00115-of-00185.safetensors", + "language_model.model.layers.58.self_attn.qkv_proj.weight": "model-00115-of-00185.safetensors", + "language_model.model.layers.59.input_layernorm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.e_score_correction_bias": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.gate.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.gate.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.gate.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_down_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_down_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_down_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_norm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_up_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_up_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.routed_expert_up_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.down_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.down_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.down_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.gate_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.gate_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.gate_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.up_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.up_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.shared_experts.up_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.down_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.down_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.down_proj.weight": "model-00118-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.gate_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.gate_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.gate_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.up_proj.biases": "model-00118-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.up_proj.scales": "model-00118-of-00185.safetensors", + "language_model.model.layers.59.mlp.switch_mlp.up_proj.weight": "model-00118-of-00185.safetensors", + "language_model.model.layers.59.mlp_res_norm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.mlp_res_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.post_attention_layernorm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.self_attention_res_norm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.self_attention_res_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.59.self_attn.embed_q.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.embed_q.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.embed_q.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.g_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.g_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.g_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.kv_a_layernorm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.o_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.o_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.o_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_a_layernorm.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_a_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_a_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_a_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_b_proj.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_b_proj.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.q_b_proj.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.unembed_out.biases": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.unembed_out.scales": "model-00117-of-00185.safetensors", + "language_model.model.layers.59.self_attn.unembed_out.weight": "model-00117-of-00185.safetensors", + "language_model.model.layers.6.input_layernorm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.e_score_correction_bias": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.gate.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.gate.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.gate.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_down_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_down_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_down_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_norm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_up_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_up_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.routed_expert_up_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.down_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.down_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.down_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.gate_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.gate_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.up_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.up_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.shared_experts.up_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.down_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.down_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight": "model-00012-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.gate_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.gate_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.up_proj.biases": "model-00012-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.up_proj.scales": "model-00012-of-00185.safetensors", + "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight": "model-00012-of-00185.safetensors", + "language_model.model.layers.6.mlp_res_norm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.mlp_res_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.post_attention_layernorm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.self_attention_res_norm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.self_attention_res_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.6.self_attn.A_log": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.b_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.b_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.b_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.dt_bias": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_a_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_a_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_a_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_b_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_b_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.f_b_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.g_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.g_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.g_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.o_norm.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.o_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.o_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.o_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.qkv_conv.conv.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.qkv_proj.biases": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.qkv_proj.scales": "model-00011-of-00185.safetensors", + "language_model.model.layers.6.self_attn.qkv_proj.weight": "model-00011-of-00185.safetensors", + "language_model.model.layers.60.input_layernorm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.e_score_correction_bias": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.gate.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.gate.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.gate.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_down_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_down_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_down_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_norm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_up_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_up_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.routed_expert_up_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.down_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.down_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.down_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.gate_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.gate_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.gate_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.up_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.up_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.shared_experts.up_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.down_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.down_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.down_proj.weight": "model-00120-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.gate_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.gate_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.gate_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.up_proj.biases": "model-00120-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.up_proj.scales": "model-00120-of-00185.safetensors", + "language_model.model.layers.60.mlp.switch_mlp.up_proj.weight": "model-00120-of-00185.safetensors", + "language_model.model.layers.60.mlp_res_norm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.mlp_res_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.post_attention_layernorm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.self_attention_res_norm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.self_attention_res_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.60.self_attn.A_log": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.b_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.b_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.b_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.dt_bias": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_a_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_a_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_a_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_b_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_b_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.f_b_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.g_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.g_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.g_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.o_norm.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.o_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.o_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.o_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.qkv_conv.conv.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.qkv_proj.biases": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.qkv_proj.scales": "model-00119-of-00185.safetensors", + "language_model.model.layers.60.self_attn.qkv_proj.weight": "model-00119-of-00185.safetensors", + "language_model.model.layers.61.input_layernorm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.e_score_correction_bias": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.gate.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.gate.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.gate.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_down_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_down_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_down_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_norm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_up_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_up_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.routed_expert_up_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.down_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.down_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.down_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.gate_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.gate_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.gate_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.up_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.up_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.shared_experts.up_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.down_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.down_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.down_proj.weight": "model-00122-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.gate_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.gate_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.gate_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.up_proj.biases": "model-00122-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.up_proj.scales": "model-00122-of-00185.safetensors", + "language_model.model.layers.61.mlp.switch_mlp.up_proj.weight": "model-00122-of-00185.safetensors", + "language_model.model.layers.61.mlp_res_norm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.mlp_res_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.post_attention_layernorm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.self_attention_res_norm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.self_attention_res_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.61.self_attn.A_log": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.b_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.b_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.b_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.dt_bias": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_a_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_a_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_a_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_b_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_b_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.f_b_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.g_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.g_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.g_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.o_norm.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.o_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.o_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.o_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.qkv_conv.conv.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.qkv_proj.biases": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.qkv_proj.scales": "model-00121-of-00185.safetensors", + "language_model.model.layers.61.self_attn.qkv_proj.weight": "model-00121-of-00185.safetensors", + "language_model.model.layers.62.input_layernorm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.e_score_correction_bias": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.gate.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.gate.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.gate.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_down_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_down_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_down_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_norm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_up_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_up_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.routed_expert_up_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.down_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.down_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.down_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.gate_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.gate_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.gate_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.up_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.up_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.shared_experts.up_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.down_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.down_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.down_proj.weight": "model-00124-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.gate_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.gate_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.gate_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.up_proj.biases": "model-00124-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.up_proj.scales": "model-00124-of-00185.safetensors", + "language_model.model.layers.62.mlp.switch_mlp.up_proj.weight": "model-00124-of-00185.safetensors", + "language_model.model.layers.62.mlp_res_norm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.mlp_res_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.post_attention_layernorm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.self_attention_res_norm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.self_attention_res_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.62.self_attn.A_log": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.b_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.b_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.b_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.dt_bias": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_a_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_a_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_a_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_b_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_b_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.f_b_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.g_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.g_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.g_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.o_norm.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.o_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.o_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.o_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.qkv_conv.conv.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.qkv_proj.biases": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.qkv_proj.scales": "model-00123-of-00185.safetensors", + "language_model.model.layers.62.self_attn.qkv_proj.weight": "model-00123-of-00185.safetensors", + "language_model.model.layers.63.input_layernorm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.e_score_correction_bias": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.gate.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.gate.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.gate.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_down_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_down_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_down_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_norm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_up_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_up_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.routed_expert_up_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.down_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.down_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.down_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.gate_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.gate_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.gate_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.up_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.up_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.shared_experts.up_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.down_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.down_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.down_proj.weight": "model-00126-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.gate_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.gate_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.gate_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.up_proj.biases": "model-00126-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.up_proj.scales": "model-00126-of-00185.safetensors", + "language_model.model.layers.63.mlp.switch_mlp.up_proj.weight": "model-00126-of-00185.safetensors", + "language_model.model.layers.63.mlp_res_norm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.mlp_res_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.post_attention_layernorm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.self_attention_res_norm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.self_attention_res_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.63.self_attn.embed_q.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.embed_q.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.embed_q.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.g_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.g_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.g_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.kv_a_layernorm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.o_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.o_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.o_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_a_layernorm.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_a_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_a_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_a_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_b_proj.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_b_proj.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.q_b_proj.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.unembed_out.biases": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.unembed_out.scales": "model-00125-of-00185.safetensors", + "language_model.model.layers.63.self_attn.unembed_out.weight": "model-00125-of-00185.safetensors", + "language_model.model.layers.64.input_layernorm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.e_score_correction_bias": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.gate.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.gate.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.gate.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_down_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_down_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_down_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_norm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_up_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_up_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.routed_expert_up_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.down_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.down_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.down_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.gate_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.gate_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.gate_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.up_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.up_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.shared_experts.up_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.down_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.down_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.down_proj.weight": "model-00128-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.gate_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.gate_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.gate_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.up_proj.biases": "model-00128-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.up_proj.scales": "model-00128-of-00185.safetensors", + "language_model.model.layers.64.mlp.switch_mlp.up_proj.weight": "model-00128-of-00185.safetensors", + "language_model.model.layers.64.mlp_res_norm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.mlp_res_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.post_attention_layernorm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.self_attention_res_norm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.self_attention_res_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.64.self_attn.A_log": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.b_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.b_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.b_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.dt_bias": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_a_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_a_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_a_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_b_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_b_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.f_b_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.g_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.g_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.g_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.o_norm.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.o_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.o_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.o_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.qkv_conv.conv.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.qkv_proj.biases": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.qkv_proj.scales": "model-00127-of-00185.safetensors", + "language_model.model.layers.64.self_attn.qkv_proj.weight": "model-00127-of-00185.safetensors", + "language_model.model.layers.65.input_layernorm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.e_score_correction_bias": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.gate.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.gate.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.gate.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_down_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_down_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_down_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_norm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_up_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_up_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.routed_expert_up_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.down_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.down_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.down_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.gate_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.gate_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.gate_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.up_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.up_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.shared_experts.up_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.down_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.down_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.down_proj.weight": "model-00130-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.gate_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.gate_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.gate_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.up_proj.biases": "model-00130-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.up_proj.scales": "model-00130-of-00185.safetensors", + "language_model.model.layers.65.mlp.switch_mlp.up_proj.weight": "model-00130-of-00185.safetensors", + "language_model.model.layers.65.mlp_res_norm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.mlp_res_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.post_attention_layernorm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.self_attention_res_norm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.self_attention_res_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.65.self_attn.A_log": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.b_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.b_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.b_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.dt_bias": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_a_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_a_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_a_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_b_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_b_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.f_b_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.g_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.g_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.g_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.o_norm.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.o_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.o_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.o_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.qkv_conv.conv.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.qkv_proj.biases": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.qkv_proj.scales": "model-00129-of-00185.safetensors", + "language_model.model.layers.65.self_attn.qkv_proj.weight": "model-00129-of-00185.safetensors", + "language_model.model.layers.66.input_layernorm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.e_score_correction_bias": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.gate.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.gate.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.gate.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_down_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_down_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_down_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_norm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_up_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_up_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.routed_expert_up_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.down_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.down_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.down_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.gate_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.gate_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.gate_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.up_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.up_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.shared_experts.up_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.down_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.down_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.down_proj.weight": "model-00132-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.gate_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.gate_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.gate_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.up_proj.biases": "model-00132-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.up_proj.scales": "model-00132-of-00185.safetensors", + "language_model.model.layers.66.mlp.switch_mlp.up_proj.weight": "model-00132-of-00185.safetensors", + "language_model.model.layers.66.mlp_res_norm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.mlp_res_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.post_attention_layernorm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.self_attention_res_norm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.self_attention_res_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.66.self_attn.A_log": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.b_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.b_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.b_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.dt_bias": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_a_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_a_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_a_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_b_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_b_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.f_b_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.g_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.g_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.g_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.o_norm.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.o_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.o_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.o_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.qkv_conv.conv.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.qkv_proj.biases": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.qkv_proj.scales": "model-00131-of-00185.safetensors", + "language_model.model.layers.66.self_attn.qkv_proj.weight": "model-00131-of-00185.safetensors", + "language_model.model.layers.67.input_layernorm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.e_score_correction_bias": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.gate.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.gate.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.gate.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_down_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_down_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_down_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_norm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_up_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_up_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.routed_expert_up_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.down_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.down_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.down_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.gate_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.gate_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.gate_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.up_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.up_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.shared_experts.up_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.down_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.down_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.down_proj.weight": "model-00134-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.gate_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.gate_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.gate_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.up_proj.biases": "model-00134-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.up_proj.scales": "model-00134-of-00185.safetensors", + "language_model.model.layers.67.mlp.switch_mlp.up_proj.weight": "model-00134-of-00185.safetensors", + "language_model.model.layers.67.mlp_res_norm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.mlp_res_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.post_attention_layernorm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.self_attention_res_norm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.self_attention_res_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.67.self_attn.embed_q.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.embed_q.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.embed_q.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.g_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.g_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.g_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.kv_a_layernorm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.o_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.o_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.o_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_a_layernorm.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_a_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_a_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_a_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_b_proj.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_b_proj.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.q_b_proj.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.unembed_out.biases": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.unembed_out.scales": "model-00133-of-00185.safetensors", + "language_model.model.layers.67.self_attn.unembed_out.weight": "model-00133-of-00185.safetensors", + "language_model.model.layers.68.input_layernorm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.e_score_correction_bias": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.gate.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.gate.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.gate.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_down_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_down_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_down_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_norm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_up_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_up_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.routed_expert_up_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.down_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.down_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.down_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.gate_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.gate_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.gate_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.up_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.up_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.shared_experts.up_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.down_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.down_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.down_proj.weight": "model-00136-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.gate_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.gate_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.gate_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.up_proj.biases": "model-00136-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.up_proj.scales": "model-00136-of-00185.safetensors", + "language_model.model.layers.68.mlp.switch_mlp.up_proj.weight": "model-00136-of-00185.safetensors", + "language_model.model.layers.68.mlp_res_norm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.mlp_res_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.post_attention_layernorm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.self_attention_res_norm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.self_attention_res_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.68.self_attn.A_log": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.b_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.b_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.b_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.dt_bias": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_a_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_a_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_a_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_b_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_b_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.f_b_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.g_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.g_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.g_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.o_norm.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.o_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.o_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.o_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.qkv_conv.conv.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.qkv_proj.biases": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.qkv_proj.scales": "model-00135-of-00185.safetensors", + "language_model.model.layers.68.self_attn.qkv_proj.weight": "model-00135-of-00185.safetensors", + "language_model.model.layers.69.input_layernorm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.e_score_correction_bias": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.gate.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.gate.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.gate.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_down_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_down_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_down_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_norm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_up_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_up_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.routed_expert_up_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.down_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.down_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.down_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.gate_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.gate_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.gate_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.up_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.up_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.shared_experts.up_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.down_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.down_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.down_proj.weight": "model-00138-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.gate_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.gate_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.gate_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.up_proj.biases": "model-00138-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.up_proj.scales": "model-00138-of-00185.safetensors", + "language_model.model.layers.69.mlp.switch_mlp.up_proj.weight": "model-00138-of-00185.safetensors", + "language_model.model.layers.69.mlp_res_norm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.mlp_res_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.post_attention_layernorm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.self_attention_res_norm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.self_attention_res_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.69.self_attn.A_log": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.b_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.b_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.b_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.dt_bias": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_a_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_a_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_a_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_b_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_b_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.f_b_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.g_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.g_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.g_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.o_norm.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.o_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.o_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.o_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.qkv_conv.conv.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.qkv_proj.biases": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.qkv_proj.scales": "model-00137-of-00185.safetensors", + "language_model.model.layers.69.self_attn.qkv_proj.weight": "model-00137-of-00185.safetensors", + "language_model.model.layers.7.input_layernorm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.e_score_correction_bias": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.gate.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.gate.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.gate.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_down_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_down_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_down_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_norm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_up_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_up_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.routed_expert_up_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.down_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.down_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.down_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.gate_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.gate_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.up_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.up_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.shared_experts.up_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.down_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.down_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight": "model-00014-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.gate_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.gate_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.up_proj.biases": "model-00014-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.up_proj.scales": "model-00014-of-00185.safetensors", + "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight": "model-00014-of-00185.safetensors", + "language_model.model.layers.7.mlp_res_norm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.mlp_res_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.post_attention_layernorm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.self_attention_res_norm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.self_attention_res_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.7.self_attn.embed_q.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.embed_q.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.embed_q.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.g_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.g_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.g_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.kv_a_layernorm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.o_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.o_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.o_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_a_layernorm.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_a_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_a_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_a_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_b_proj.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_b_proj.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.q_b_proj.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.unembed_out.biases": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.unembed_out.scales": "model-00013-of-00185.safetensors", + "language_model.model.layers.7.self_attn.unembed_out.weight": "model-00013-of-00185.safetensors", + "language_model.model.layers.70.input_layernorm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.e_score_correction_bias": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.gate.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.gate.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.gate.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_down_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_down_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_down_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_norm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_up_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_up_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.routed_expert_up_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.down_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.down_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.down_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.gate_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.gate_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.gate_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.up_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.up_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.shared_experts.up_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.down_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.down_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.down_proj.weight": "model-00140-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.gate_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.gate_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.gate_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.up_proj.biases": "model-00140-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.up_proj.scales": "model-00140-of-00185.safetensors", + "language_model.model.layers.70.mlp.switch_mlp.up_proj.weight": "model-00140-of-00185.safetensors", + "language_model.model.layers.70.mlp_res_norm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.mlp_res_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.post_attention_layernorm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.self_attention_res_norm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.self_attention_res_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.70.self_attn.A_log": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.b_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.b_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.b_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.dt_bias": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_a_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_a_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_a_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_b_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_b_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.f_b_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.g_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.g_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.g_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.o_norm.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.o_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.o_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.o_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.qkv_conv.conv.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.qkv_proj.biases": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.qkv_proj.scales": "model-00139-of-00185.safetensors", + "language_model.model.layers.70.self_attn.qkv_proj.weight": "model-00139-of-00185.safetensors", + "language_model.model.layers.71.input_layernorm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.e_score_correction_bias": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.gate.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.gate.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.gate.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_down_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_down_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_down_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_norm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_up_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_up_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.routed_expert_up_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.down_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.down_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.down_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.gate_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.gate_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.gate_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.up_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.up_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.shared_experts.up_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.down_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.down_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.down_proj.weight": "model-00142-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.gate_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.gate_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.gate_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.up_proj.biases": "model-00142-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.up_proj.scales": "model-00142-of-00185.safetensors", + "language_model.model.layers.71.mlp.switch_mlp.up_proj.weight": "model-00142-of-00185.safetensors", + "language_model.model.layers.71.mlp_res_norm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.mlp_res_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.post_attention_layernorm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.self_attention_res_norm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.self_attention_res_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.71.self_attn.embed_q.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.embed_q.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.embed_q.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.g_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.g_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.g_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.kv_a_layernorm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.o_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.o_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.o_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_a_layernorm.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_a_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_a_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_a_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_b_proj.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_b_proj.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.q_b_proj.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.unembed_out.biases": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.unembed_out.scales": "model-00141-of-00185.safetensors", + "language_model.model.layers.71.self_attn.unembed_out.weight": "model-00141-of-00185.safetensors", + "language_model.model.layers.72.input_layernorm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.e_score_correction_bias": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.gate.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.gate.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.gate.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_down_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_down_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_down_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_norm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_up_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_up_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.routed_expert_up_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.down_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.down_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.down_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.gate_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.gate_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.gate_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.up_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.up_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.shared_experts.up_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.down_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.down_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.down_proj.weight": "model-00144-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.gate_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.gate_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.gate_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.up_proj.biases": "model-00144-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.up_proj.scales": "model-00144-of-00185.safetensors", + "language_model.model.layers.72.mlp.switch_mlp.up_proj.weight": "model-00144-of-00185.safetensors", + "language_model.model.layers.72.mlp_res_norm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.mlp_res_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.post_attention_layernorm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.self_attention_res_norm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.self_attention_res_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.72.self_attn.A_log": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.b_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.b_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.b_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.dt_bias": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_a_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_a_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_a_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_b_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_b_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.f_b_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.g_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.g_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.g_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.o_norm.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.o_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.o_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.o_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.qkv_conv.conv.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.qkv_proj.biases": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.qkv_proj.scales": "model-00143-of-00185.safetensors", + "language_model.model.layers.72.self_attn.qkv_proj.weight": "model-00143-of-00185.safetensors", + "language_model.model.layers.73.input_layernorm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.e_score_correction_bias": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.gate.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.gate.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.gate.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_down_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_down_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_down_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_norm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_up_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_up_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.routed_expert_up_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.down_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.down_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.down_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.gate_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.gate_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.gate_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.up_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.up_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.shared_experts.up_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.down_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.down_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.down_proj.weight": "model-00146-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.gate_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.gate_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.gate_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.up_proj.biases": "model-00146-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.up_proj.scales": "model-00146-of-00185.safetensors", + "language_model.model.layers.73.mlp.switch_mlp.up_proj.weight": "model-00146-of-00185.safetensors", + "language_model.model.layers.73.mlp_res_norm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.mlp_res_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.post_attention_layernorm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.self_attention_res_norm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.self_attention_res_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.73.self_attn.A_log": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.b_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.b_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.b_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.dt_bias": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_a_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_a_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_a_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_b_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_b_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.f_b_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.g_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.g_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.g_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.o_norm.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.o_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.o_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.o_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.qkv_conv.conv.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.qkv_proj.biases": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.qkv_proj.scales": "model-00145-of-00185.safetensors", + "language_model.model.layers.73.self_attn.qkv_proj.weight": "model-00145-of-00185.safetensors", + "language_model.model.layers.74.input_layernorm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.e_score_correction_bias": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.gate.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.gate.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.gate.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_down_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_down_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_down_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_norm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_up_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_up_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.routed_expert_up_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.down_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.down_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.down_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.gate_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.gate_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.gate_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.up_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.up_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.shared_experts.up_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.down_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.down_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.down_proj.weight": "model-00148-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.gate_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.gate_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.gate_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.up_proj.biases": "model-00148-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.up_proj.scales": "model-00148-of-00185.safetensors", + "language_model.model.layers.74.mlp.switch_mlp.up_proj.weight": "model-00148-of-00185.safetensors", + "language_model.model.layers.74.mlp_res_norm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.mlp_res_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.post_attention_layernorm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.self_attention_res_norm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.self_attention_res_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.74.self_attn.A_log": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.b_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.b_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.b_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.dt_bias": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_a_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_a_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_a_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_b_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_b_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.f_b_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.g_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.g_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.g_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.o_norm.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.o_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.o_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.o_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.qkv_conv.conv.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.qkv_proj.biases": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.qkv_proj.scales": "model-00147-of-00185.safetensors", + "language_model.model.layers.74.self_attn.qkv_proj.weight": "model-00147-of-00185.safetensors", + "language_model.model.layers.75.input_layernorm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.e_score_correction_bias": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.gate.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.gate.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.gate.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_down_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_down_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_down_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_norm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_up_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_up_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.routed_expert_up_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.down_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.down_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.down_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.gate_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.gate_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.gate_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.up_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.up_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.shared_experts.up_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.down_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.down_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.down_proj.weight": "model-00150-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.gate_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.gate_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.gate_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.up_proj.biases": "model-00150-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.up_proj.scales": "model-00150-of-00185.safetensors", + "language_model.model.layers.75.mlp.switch_mlp.up_proj.weight": "model-00150-of-00185.safetensors", + "language_model.model.layers.75.mlp_res_norm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.mlp_res_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.post_attention_layernorm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.self_attention_res_norm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.self_attention_res_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.75.self_attn.embed_q.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.embed_q.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.embed_q.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.g_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.g_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.g_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.kv_a_layernorm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.o_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.o_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.o_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_a_layernorm.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_a_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_a_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_a_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_b_proj.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_b_proj.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.q_b_proj.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.unembed_out.biases": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.unembed_out.scales": "model-00149-of-00185.safetensors", + "language_model.model.layers.75.self_attn.unembed_out.weight": "model-00149-of-00185.safetensors", + "language_model.model.layers.76.input_layernorm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.e_score_correction_bias": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.gate.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.gate.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.gate.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_down_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_down_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_down_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_norm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_up_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_up_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.routed_expert_up_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.down_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.down_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.down_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.gate_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.gate_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.gate_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.up_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.up_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.shared_experts.up_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.down_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.down_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.down_proj.weight": "model-00152-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.gate_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.gate_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.gate_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.up_proj.biases": "model-00152-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.up_proj.scales": "model-00152-of-00185.safetensors", + "language_model.model.layers.76.mlp.switch_mlp.up_proj.weight": "model-00152-of-00185.safetensors", + "language_model.model.layers.76.mlp_res_norm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.mlp_res_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.post_attention_layernorm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.self_attention_res_norm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.self_attention_res_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.76.self_attn.A_log": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.b_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.b_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.b_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.dt_bias": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_a_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_a_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_a_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_b_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_b_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.f_b_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.g_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.g_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.g_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.o_norm.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.o_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.o_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.o_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.qkv_conv.conv.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.qkv_proj.biases": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.qkv_proj.scales": "model-00151-of-00185.safetensors", + "language_model.model.layers.76.self_attn.qkv_proj.weight": "model-00151-of-00185.safetensors", + "language_model.model.layers.77.input_layernorm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.e_score_correction_bias": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.gate.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.gate.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.gate.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_down_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_down_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_down_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_norm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_up_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_up_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.routed_expert_up_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.down_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.down_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.down_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.gate_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.gate_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.gate_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.up_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.up_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.shared_experts.up_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.down_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.down_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.down_proj.weight": "model-00154-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.gate_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.gate_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.gate_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.up_proj.biases": "model-00154-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.up_proj.scales": "model-00154-of-00185.safetensors", + "language_model.model.layers.77.mlp.switch_mlp.up_proj.weight": "model-00154-of-00185.safetensors", + "language_model.model.layers.77.mlp_res_norm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.mlp_res_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.post_attention_layernorm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.self_attention_res_norm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.self_attention_res_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.77.self_attn.A_log": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.b_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.b_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.b_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.dt_bias": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_a_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_a_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_a_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_b_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_b_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.f_b_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.g_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.g_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.g_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.o_norm.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.o_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.o_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.o_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.qkv_conv.conv.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.qkv_proj.biases": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.qkv_proj.scales": "model-00153-of-00185.safetensors", + "language_model.model.layers.77.self_attn.qkv_proj.weight": "model-00153-of-00185.safetensors", + "language_model.model.layers.78.input_layernorm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.e_score_correction_bias": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.gate.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.gate.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.gate.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_down_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_down_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_down_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_norm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_up_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_up_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.routed_expert_up_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.down_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.down_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.down_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.gate_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.gate_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.gate_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.up_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.up_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.shared_experts.up_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.down_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.down_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.down_proj.weight": "model-00156-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.gate_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.gate_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.gate_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.up_proj.biases": "model-00156-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.up_proj.scales": "model-00156-of-00185.safetensors", + "language_model.model.layers.78.mlp.switch_mlp.up_proj.weight": "model-00156-of-00185.safetensors", + "language_model.model.layers.78.mlp_res_norm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.mlp_res_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.post_attention_layernorm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.self_attention_res_norm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.self_attention_res_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.78.self_attn.A_log": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.b_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.b_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.b_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.dt_bias": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_a_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_a_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_a_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_b_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_b_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.f_b_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.g_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.g_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.g_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.o_norm.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.o_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.o_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.o_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.qkv_conv.conv.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.qkv_proj.biases": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.qkv_proj.scales": "model-00155-of-00185.safetensors", + "language_model.model.layers.78.self_attn.qkv_proj.weight": "model-00155-of-00185.safetensors", + "language_model.model.layers.79.input_layernorm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.e_score_correction_bias": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.gate.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.gate.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.gate.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_down_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_down_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_down_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_norm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_up_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_up_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.routed_expert_up_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.down_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.down_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.down_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.gate_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.gate_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.gate_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.up_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.up_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.shared_experts.up_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.down_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.down_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.down_proj.weight": "model-00158-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.gate_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.gate_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.gate_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.up_proj.biases": "model-00158-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.up_proj.scales": "model-00158-of-00185.safetensors", + "language_model.model.layers.79.mlp.switch_mlp.up_proj.weight": "model-00158-of-00185.safetensors", + "language_model.model.layers.79.mlp_res_norm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.mlp_res_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.post_attention_layernorm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.self_attention_res_norm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.self_attention_res_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.79.self_attn.embed_q.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.embed_q.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.embed_q.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.g_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.g_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.g_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.kv_a_layernorm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.o_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.o_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.o_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_a_layernorm.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_a_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_a_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_a_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_b_proj.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_b_proj.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.q_b_proj.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.unembed_out.biases": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.unembed_out.scales": "model-00157-of-00185.safetensors", + "language_model.model.layers.79.self_attn.unembed_out.weight": "model-00157-of-00185.safetensors", + "language_model.model.layers.8.input_layernorm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.e_score_correction_bias": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.gate.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.gate.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.gate.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_down_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_down_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_down_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_norm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_up_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_up_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.routed_expert_up_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.down_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.down_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.down_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.gate_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.gate_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.up_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.up_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.shared_experts.up_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.down_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.down_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight": "model-00016-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.gate_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.gate_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.up_proj.biases": "model-00016-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.up_proj.scales": "model-00016-of-00185.safetensors", + "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight": "model-00016-of-00185.safetensors", + "language_model.model.layers.8.mlp_res_norm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.mlp_res_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.post_attention_layernorm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.self_attention_res_norm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.self_attention_res_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.8.self_attn.A_log": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.b_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.b_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.b_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.dt_bias": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_a_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_a_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_a_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_b_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_b_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.f_b_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.g_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.g_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.g_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.o_norm.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.o_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.o_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.o_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.qkv_conv.conv.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.qkv_proj.biases": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.qkv_proj.scales": "model-00015-of-00185.safetensors", + "language_model.model.layers.8.self_attn.qkv_proj.weight": "model-00015-of-00185.safetensors", + "language_model.model.layers.80.input_layernorm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.e_score_correction_bias": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.gate.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.gate.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.gate.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_down_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_down_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_down_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_norm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_up_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_up_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.routed_expert_up_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.down_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.down_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.down_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.gate_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.gate_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.gate_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.up_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.up_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.shared_experts.up_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.down_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.down_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.down_proj.weight": "model-00160-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.gate_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.gate_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.gate_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.up_proj.biases": "model-00160-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.up_proj.scales": "model-00160-of-00185.safetensors", + "language_model.model.layers.80.mlp.switch_mlp.up_proj.weight": "model-00160-of-00185.safetensors", + "language_model.model.layers.80.mlp_res_norm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.mlp_res_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.post_attention_layernorm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.self_attention_res_norm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.self_attention_res_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.80.self_attn.A_log": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.b_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.b_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.b_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.dt_bias": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_a_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_a_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_a_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_b_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_b_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.f_b_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.g_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.g_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.g_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.o_norm.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.o_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.o_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.o_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.qkv_conv.conv.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.qkv_proj.biases": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.qkv_proj.scales": "model-00159-of-00185.safetensors", + "language_model.model.layers.80.self_attn.qkv_proj.weight": "model-00159-of-00185.safetensors", + "language_model.model.layers.81.input_layernorm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.e_score_correction_bias": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.gate.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.gate.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.gate.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_down_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_down_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_down_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_norm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_up_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_up_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.routed_expert_up_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.down_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.down_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.down_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.gate_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.gate_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.gate_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.up_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.up_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.shared_experts.up_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.down_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.down_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.down_proj.weight": "model-00162-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.gate_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.gate_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.gate_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.up_proj.biases": "model-00162-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.up_proj.scales": "model-00162-of-00185.safetensors", + "language_model.model.layers.81.mlp.switch_mlp.up_proj.weight": "model-00162-of-00185.safetensors", + "language_model.model.layers.81.mlp_res_norm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.mlp_res_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.post_attention_layernorm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.self_attention_res_norm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.self_attention_res_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.81.self_attn.A_log": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.b_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.b_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.b_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.dt_bias": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_a_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_a_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_a_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_b_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_b_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.f_b_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.g_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.g_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.g_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.o_norm.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.o_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.o_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.o_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.qkv_conv.conv.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.qkv_proj.biases": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.qkv_proj.scales": "model-00161-of-00185.safetensors", + "language_model.model.layers.81.self_attn.qkv_proj.weight": "model-00161-of-00185.safetensors", + "language_model.model.layers.82.input_layernorm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.e_score_correction_bias": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.gate.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.gate.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.gate.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_down_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_down_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_down_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_norm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_up_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_up_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.routed_expert_up_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.down_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.down_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.down_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.gate_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.gate_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.gate_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.up_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.up_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.shared_experts.up_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.down_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.down_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.down_proj.weight": "model-00164-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.gate_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.gate_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.gate_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.up_proj.biases": "model-00164-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.up_proj.scales": "model-00164-of-00185.safetensors", + "language_model.model.layers.82.mlp.switch_mlp.up_proj.weight": "model-00164-of-00185.safetensors", + "language_model.model.layers.82.mlp_res_norm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.mlp_res_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.post_attention_layernorm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.self_attention_res_norm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.self_attention_res_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.82.self_attn.A_log": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.b_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.b_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.b_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.dt_bias": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_a_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_a_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_a_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_b_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_b_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.f_b_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.g_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.g_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.g_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.o_norm.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.o_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.o_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.o_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.qkv_conv.conv.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.qkv_proj.biases": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.qkv_proj.scales": "model-00163-of-00185.safetensors", + "language_model.model.layers.82.self_attn.qkv_proj.weight": "model-00163-of-00185.safetensors", + "language_model.model.layers.83.input_layernorm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.e_score_correction_bias": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.gate.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.gate.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.gate.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_down_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_down_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_down_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_norm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_up_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_up_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.routed_expert_up_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.down_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.down_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.down_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.gate_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.gate_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.gate_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.up_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.up_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.shared_experts.up_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.down_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.down_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.down_proj.weight": "model-00166-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.gate_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.gate_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.gate_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.up_proj.biases": "model-00166-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.up_proj.scales": "model-00166-of-00185.safetensors", + "language_model.model.layers.83.mlp.switch_mlp.up_proj.weight": "model-00166-of-00185.safetensors", + "language_model.model.layers.83.mlp_res_norm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.mlp_res_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.post_attention_layernorm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.self_attention_res_norm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.self_attention_res_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.83.self_attn.embed_q.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.embed_q.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.embed_q.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.g_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.g_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.g_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.kv_a_layernorm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.o_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.o_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.o_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_a_layernorm.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_a_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_a_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_a_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_b_proj.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_b_proj.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.q_b_proj.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.unembed_out.biases": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.unembed_out.scales": "model-00165-of-00185.safetensors", + "language_model.model.layers.83.self_attn.unembed_out.weight": "model-00165-of-00185.safetensors", + "language_model.model.layers.84.input_layernorm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.e_score_correction_bias": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.gate.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.gate.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.gate.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_down_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_down_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_down_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_norm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_up_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_up_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.routed_expert_up_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.down_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.down_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.down_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.gate_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.gate_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.gate_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.up_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.up_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.shared_experts.up_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.down_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.down_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.down_proj.weight": "model-00168-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.gate_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.gate_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.gate_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.up_proj.biases": "model-00168-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.up_proj.scales": "model-00168-of-00185.safetensors", + "language_model.model.layers.84.mlp.switch_mlp.up_proj.weight": "model-00168-of-00185.safetensors", + "language_model.model.layers.84.mlp_res_norm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.mlp_res_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.post_attention_layernorm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.self_attention_res_norm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.self_attention_res_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.84.self_attn.A_log": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.b_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.b_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.b_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.dt_bias": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_a_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_a_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_a_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_b_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_b_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.f_b_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.g_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.g_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.g_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.o_norm.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.o_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.o_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.o_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.qkv_conv.conv.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.qkv_proj.biases": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.qkv_proj.scales": "model-00167-of-00185.safetensors", + "language_model.model.layers.84.self_attn.qkv_proj.weight": "model-00167-of-00185.safetensors", + "language_model.model.layers.85.input_layernorm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.e_score_correction_bias": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.gate.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.gate.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.gate.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_down_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_down_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_down_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_norm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_up_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_up_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.routed_expert_up_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.down_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.down_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.down_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.gate_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.gate_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.gate_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.up_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.up_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.shared_experts.up_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.down_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.down_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.down_proj.weight": "model-00170-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.gate_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.gate_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.gate_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.up_proj.biases": "model-00170-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.up_proj.scales": "model-00170-of-00185.safetensors", + "language_model.model.layers.85.mlp.switch_mlp.up_proj.weight": "model-00170-of-00185.safetensors", + "language_model.model.layers.85.mlp_res_norm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.mlp_res_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.post_attention_layernorm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.self_attention_res_norm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.self_attention_res_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.85.self_attn.A_log": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.b_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.b_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.b_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.dt_bias": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_a_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_a_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_a_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_b_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_b_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.f_b_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.g_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.g_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.g_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.o_norm.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.o_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.o_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.o_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.qkv_conv.conv.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.qkv_proj.biases": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.qkv_proj.scales": "model-00169-of-00185.safetensors", + "language_model.model.layers.85.self_attn.qkv_proj.weight": "model-00169-of-00185.safetensors", + "language_model.model.layers.86.input_layernorm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.e_score_correction_bias": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.gate.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.gate.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.gate.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_down_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_down_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_down_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_norm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_up_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_up_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.routed_expert_up_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.down_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.down_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.down_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.gate_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.gate_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.gate_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.up_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.up_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.shared_experts.up_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.down_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.down_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.down_proj.weight": "model-00172-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.gate_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.gate_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.gate_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.up_proj.biases": "model-00172-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.up_proj.scales": "model-00172-of-00185.safetensors", + "language_model.model.layers.86.mlp.switch_mlp.up_proj.weight": "model-00172-of-00185.safetensors", + "language_model.model.layers.86.mlp_res_norm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.mlp_res_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.post_attention_layernorm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.self_attention_res_norm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.self_attention_res_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.86.self_attn.A_log": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.b_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.b_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.b_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.dt_bias": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_a_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_a_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_a_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_b_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_b_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.f_b_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.g_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.g_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.g_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.o_norm.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.o_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.o_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.o_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.qkv_conv.conv.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.qkv_proj.biases": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.qkv_proj.scales": "model-00171-of-00185.safetensors", + "language_model.model.layers.86.self_attn.qkv_proj.weight": "model-00171-of-00185.safetensors", + "language_model.model.layers.87.input_layernorm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.e_score_correction_bias": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.gate.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.gate.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.gate.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_down_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_down_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_down_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_norm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_up_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_up_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.routed_expert_up_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.down_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.down_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.down_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.gate_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.gate_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.gate_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.up_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.up_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.shared_experts.up_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.down_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.down_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.down_proj.weight": "model-00174-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.gate_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.gate_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.gate_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.up_proj.biases": "model-00174-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.up_proj.scales": "model-00174-of-00185.safetensors", + "language_model.model.layers.87.mlp.switch_mlp.up_proj.weight": "model-00174-of-00185.safetensors", + "language_model.model.layers.87.mlp_res_norm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.mlp_res_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.post_attention_layernorm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.self_attention_res_norm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.self_attention_res_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.87.self_attn.embed_q.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.embed_q.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.embed_q.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.g_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.g_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.g_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.kv_a_layernorm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.o_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.o_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.o_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_a_layernorm.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_a_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_a_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_a_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_b_proj.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_b_proj.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.q_b_proj.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.unembed_out.biases": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.unembed_out.scales": "model-00173-of-00185.safetensors", + "language_model.model.layers.87.self_attn.unembed_out.weight": "model-00173-of-00185.safetensors", + "language_model.model.layers.88.input_layernorm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.e_score_correction_bias": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.gate.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.gate.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.gate.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_down_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_down_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_down_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_norm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_up_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_up_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.routed_expert_up_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.down_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.down_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.down_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.gate_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.gate_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.gate_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.up_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.up_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.shared_experts.up_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.down_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.down_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.down_proj.weight": "model-00176-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.gate_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.gate_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.gate_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.up_proj.biases": "model-00176-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.up_proj.scales": "model-00176-of-00185.safetensors", + "language_model.model.layers.88.mlp.switch_mlp.up_proj.weight": "model-00176-of-00185.safetensors", + "language_model.model.layers.88.mlp_res_norm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.mlp_res_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.post_attention_layernorm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.self_attention_res_norm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.self_attention_res_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.88.self_attn.A_log": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.b_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.b_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.b_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.dt_bias": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_a_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_a_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_a_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_b_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_b_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.f_b_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.g_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.g_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.g_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.o_norm.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.o_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.o_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.o_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.qkv_conv.conv.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.qkv_proj.biases": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.qkv_proj.scales": "model-00175-of-00185.safetensors", + "language_model.model.layers.88.self_attn.qkv_proj.weight": "model-00175-of-00185.safetensors", + "language_model.model.layers.89.input_layernorm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.e_score_correction_bias": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.gate.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.gate.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.gate.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_down_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_down_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_down_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_norm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_up_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_up_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.routed_expert_up_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.down_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.down_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.down_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.gate_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.gate_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.gate_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.up_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.up_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.shared_experts.up_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.down_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.down_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.down_proj.weight": "model-00178-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.gate_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.gate_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.gate_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.up_proj.biases": "model-00178-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.up_proj.scales": "model-00178-of-00185.safetensors", + "language_model.model.layers.89.mlp.switch_mlp.up_proj.weight": "model-00178-of-00185.safetensors", + "language_model.model.layers.89.mlp_res_norm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.mlp_res_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.post_attention_layernorm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.self_attention_res_norm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.self_attention_res_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.89.self_attn.A_log": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.b_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.b_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.b_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.dt_bias": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_a_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_a_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_a_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_b_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_b_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.f_b_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.g_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.g_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.g_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.o_norm.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.o_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.o_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.o_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.qkv_conv.conv.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.qkv_proj.biases": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.qkv_proj.scales": "model-00177-of-00185.safetensors", + "language_model.model.layers.89.self_attn.qkv_proj.weight": "model-00177-of-00185.safetensors", + "language_model.model.layers.9.input_layernorm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.e_score_correction_bias": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.gate.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.gate.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.gate.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_down_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_down_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_down_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_norm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_up_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_up_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.routed_expert_up_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.down_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.down_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.down_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.gate_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.gate_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.up_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.up_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.shared_experts.up_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.down_proj.biases": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.down_proj.scales": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight": "model-00018-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.gate_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.gate_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.up_proj.biases": "model-00018-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.up_proj.scales": "model-00018-of-00185.safetensors", + "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight": "model-00018-of-00185.safetensors", + "language_model.model.layers.9.mlp_res_norm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.mlp_res_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.post_attention_layernorm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.self_attention_res_norm.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.self_attention_res_proj.weight": "model-00019-of-00185.safetensors", + "language_model.model.layers.9.self_attn.A_log": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.b_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.b_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.b_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.dt_bias": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_a_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_a_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_a_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_b_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_b_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.f_b_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.g_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.g_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.g_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.o_norm.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.o_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.o_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.o_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.qkv_conv.conv.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.qkv_proj.biases": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.qkv_proj.scales": "model-00017-of-00185.safetensors", + "language_model.model.layers.9.self_attn.qkv_proj.weight": "model-00017-of-00185.safetensors", + "language_model.model.layers.90.input_layernorm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.e_score_correction_bias": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.gate.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.gate.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.gate.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_down_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_down_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_down_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_norm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_up_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_up_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.routed_expert_up_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.down_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.down_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.down_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.gate_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.gate_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.gate_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.up_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.up_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.shared_experts.up_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.down_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.down_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.down_proj.weight": "model-00180-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.gate_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.gate_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.gate_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.up_proj.biases": "model-00180-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.up_proj.scales": "model-00180-of-00185.safetensors", + "language_model.model.layers.90.mlp.switch_mlp.up_proj.weight": "model-00180-of-00185.safetensors", + "language_model.model.layers.90.mlp_res_norm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.mlp_res_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.post_attention_layernorm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.self_attention_res_norm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.self_attention_res_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.90.self_attn.A_log": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.b_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.b_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.b_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.dt_bias": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_a_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_a_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_a_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_b_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_b_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.f_b_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.g_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.g_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.g_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.o_norm.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.o_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.o_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.o_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.qkv_conv.conv.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.qkv_proj.biases": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.qkv_proj.scales": "model-00179-of-00185.safetensors", + "language_model.model.layers.90.self_attn.qkv_proj.weight": "model-00179-of-00185.safetensors", + "language_model.model.layers.91.input_layernorm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.e_score_correction_bias": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.gate.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.gate.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.gate.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_down_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_down_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_down_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_norm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_up_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_up_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.routed_expert_up_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.down_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.down_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.down_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.gate_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.gate_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.gate_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.up_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.up_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.shared_experts.up_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.down_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.down_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.down_proj.weight": "model-00182-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.gate_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.gate_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.gate_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.up_proj.biases": "model-00182-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.up_proj.scales": "model-00182-of-00185.safetensors", + "language_model.model.layers.91.mlp.switch_mlp.up_proj.weight": "model-00182-of-00185.safetensors", + "language_model.model.layers.91.mlp_res_norm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.mlp_res_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.post_attention_layernorm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.self_attention_res_norm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.self_attention_res_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.91.self_attn.embed_q.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.embed_q.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.embed_q.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.g_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.g_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.g_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.kv_a_layernorm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.o_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.o_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.o_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_a_layernorm.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_a_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_a_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_a_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_b_proj.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_b_proj.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.q_b_proj.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.unembed_out.biases": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.unembed_out.scales": "model-00181-of-00185.safetensors", + "language_model.model.layers.91.self_attn.unembed_out.weight": "model-00181-of-00185.safetensors", + "language_model.model.layers.92.input_layernorm.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.e_score_correction_bias": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.gate.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.gate.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.gate.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_down_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_down_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_down_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_norm.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_up_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_up_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.routed_expert_up_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.down_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.down_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.down_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.gate_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.gate_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.gate_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.up_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.up_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.shared_experts.up_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.down_proj.biases": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.down_proj.scales": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.down_proj.weight": "model-00184-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.gate_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.gate_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.gate_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.up_proj.biases": "model-00184-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.up_proj.scales": "model-00184-of-00185.safetensors", + "language_model.model.layers.92.mlp.switch_mlp.up_proj.weight": "model-00184-of-00185.safetensors", + "language_model.model.layers.92.mlp_res_norm.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.mlp_res_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.post_attention_layernorm.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.self_attention_res_norm.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.self_attention_res_proj.weight": "model-00185-of-00185.safetensors", + "language_model.model.layers.92.self_attn.embed_q.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.embed_q.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.embed_q.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.g_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.g_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.g_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.kv_a_layernorm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.o_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.o_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.o_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_a_layernorm.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_a_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_a_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_a_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_b_proj.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_b_proj.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.q_b_proj.weight": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.unembed_out.biases": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.unembed_out.scales": "model-00183-of-00185.safetensors", + "language_model.model.layers.92.self_attn.unembed_out.weight": "model-00183-of-00185.safetensors", + "language_model.model.norm.weight": "model-00185-of-00185.safetensors", + "language_model.model.output_attn_res_norm.weight": "model-00185-of-00185.safetensors", + "language_model.model.output_attn_res_proj.weight": "model-00185-of-00185.safetensors", + "mm_projector.proj.2.weight": "model-passthrough-00001.safetensors", + "mm_projector.proj.0.weight": "model-passthrough-00001.safetensors", + "mm_projector.post_norm.weight": "model-passthrough-00001.safetensors", + "vision_tower.patch_embed.proj.weight": "model-passthrough-00001.safetensors", + "vision_tower.patch_embed.pos_emb.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.final_layernorm.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.26.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.24.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.2.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.4.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.9.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.21.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.17.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.14.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.6.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.15.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.19.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.8.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.22.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.25.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.1.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.5.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.10.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.wo.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.18.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.7.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.3.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.mlp.fc1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.13.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.12.mlp.fc0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.23.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.20.norm1.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.11.wqkv.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.16.norm0.weight": "model-passthrough-00001.safetensors", + "vision_tower.encoder.blocks.0.norm1.weight": "model-passthrough-00001.safetensors" + } +} \ No newline at end of file diff --git a/modeling_kimi_k3.py b/modeling_kimi_k3.py new file mode 100644 index 0000000000000000000000000000000000000000..be85612710534c2e41065b67933a9639e7817d7f --- /dev/null +++ b/modeling_kimi_k3.py @@ -0,0 +1,1317 @@ +# coding=utf-8 +# Copyright 2025-2026 The Moonshot AI Team and HuggingFace Inc. team. All rights reserved. +# +# The code is based on llava (llava/modeling_llava.py), but modified for Kimi-K3. +# +# Licensing Information: +# - Code derived from llava (llava/modeling_llava.py) is licensed under the Apache License, Version 2.0. +# - Other parts of the code are licensed under the Kimi K3 License (see the LICENSE file in this repository). +# +# Apache License, Version 2.0: +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# NOTE: Reference implementation for model architecture; see the model card for production deployment. +import math +from collections.abc import Sequence +from copy import deepcopy +from typing import Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import activations + +try: + from transformers.activations import PytorchGELUTanh +except ImportError: + from transformers.activations import GELUTanh + activations.PytorchGELUTanh = GELUTanh + PytorchGELUTanh = GELUTanh +from transformers.activations import PytorchGELUTanh +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_utils import PreTrainedModel +from transformers.models.llava.modeling_llava import \ + LlavaCausalLMOutputWithPast +from transformers.utils import is_flash_attn_2_available + +from .configuration_kimi_k3 import KimiK3Config +from .modeling_kimi_linear import KimiLinearForCausalLM + +# Flash attention imports +if is_flash_attn_2_available(): + from flash_attn import flash_attn_varlen_func +else: + flash_attn_varlen_func = None + + +def multihead_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_cu_seqlens: torch.Tensor | None = None, + k_cu_seqlens: torch.Tensor | None = None, + max_seqlen_q: int | None = None, + max_seqlen_k: int | None = None, + deterministic: bool = False, +): + """Multi-head attention using flash attention 2. + + Args: + q, k, v: tensor of shape (batch_size, seqlen, num_heads, head_dim), + or (tot_seqlens, num_heads, head_dim) if packing. + q_cu_seqlens (torch.Tensor): cumulative sequence lengths of q. + The first element should be 0 and the last element should be q.shape[0]. + k_cu_seqlens (torch.Tensor): cumulative sequence lengths of k. + The first element should be 0 and the last element should be k.shape[0]. + + Returns: + output: shape (batch_size, seqlen, dim) or (tot_seqlens, dim) if packing, + where dim = num_heads * head_dim + """ + attn_out = flash_attn_varlen_func( + q, + k, + v, + q_cu_seqlens, + k_cu_seqlens, + max_seqlen_q, + max_seqlen_k, + causal=False, + deterministic=deterministic, + ) + if isinstance(attn_out, tuple): + attn_out = attn_out[0] + + attn_out = attn_out.flatten(start_dim=-2) + + return attn_out + + +def eager_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_cu_seqlens: Optional[torch.Tensor] = None, + k_cu_seqlens: Optional[torch.Tensor] = None, + **kwargs, +) -> torch.Tensor: + seq_length = q.shape[0] + attention_mask = torch.zeros([1, seq_length, seq_length], + device=q.device, + dtype=torch.bool) + for i in range(1, len(q_cu_seqlens)): + attention_mask[ + ..., + q_cu_seqlens[i - 1]:q_cu_seqlens[i], + q_cu_seqlens[i - 1]:q_cu_seqlens[i], + ] = True + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + + attn_weight = q @ k.transpose(-2, -1) / math.sqrt(q.shape[-1]) + attn_weight = attn_weight.masked_fill( + ~attention_mask, torch.finfo(attn_weight.dtype).min) + attn_weight = torch.softmax(attn_weight, dim=-1, + dtype=torch.float32).to(q.dtype) + + attn_output = attn_weight @ v + attn_output = attn_output.transpose(0, 1) + attn_output = attn_output.reshape(seq_length, -1) + return attn_output + + +VL_VISION_ATTENTION_FUNCTIONS = { + "flash_attention_2": multihead_attention, + "eager": eager_attention, +} + + +def _apply_rope_input_validation(x, freqs_cis): + assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape) + assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape) + assert x.shape[-1] == 2 * freqs_cis.shape[-1], (x.shape, freqs_cis.shape) + assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype + + +def get_rope_shape_decorate(func): + _get_rope_shape_first_call_flag = set() + + def wrapper(org, interpolation_mode, shape): + key = (org.requires_grad, torch.is_grad_enabled(), interpolation_mode) + if key not in _get_rope_shape_first_call_flag: + _get_rope_shape_first_call_flag.add(key) + _ = func(org, interpolation_mode, shape=(64, 64)) + return func(org, interpolation_mode, shape) + + return wrapper + + +@get_rope_shape_decorate +@torch.compile(dynamic=True) +def get_rope_shape(org, interpolation_mode, shape): + return (F.interpolate( + org.permute((2, 0, 1)).unsqueeze(0), + size=shape, + mode=interpolation_mode, + ).squeeze(0).permute((1, 2, 0)).flatten(end_dim=1)) + + +def apply_rope(xq: torch.Tensor, xk: torch.Tensor, + freqs_cis: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Args: (The leading dimensions of all inputs should be the same) + xq: query, tensor of shape (..., num_heads, head_dim) + xk: key, tensor of shape (..., num_heads, head_dim) + freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64. It contains the precomputed cis(freqs) for each position in the 2D grid. + Returns: + xq_out, xk_out: tensors of shape (..., num_heads, head_dim) + """ + _apply_rope_input_validation(xq, freqs_cis) + _apply_rope_input_validation(xk, freqs_cis) + + freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2 + # ..., num_heads, head_dim/2 + xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2)) + xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2)) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten( + -2) # ..., num_heads, head_dim + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten( + -2) # ..., num_heads, head_dim + return xq_out.type_as(xq), xk_out.type_as(xk) + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + From: + https://github.com/OpenGVLab/InternVideo/blob/421f6d2361fc8f61a3394244571f2601a4e99e29/InternVideo2/multi_modality/models/backbones/internvideo2/pos_embed.py#L86 + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float32) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False): + """ + t_size: int of the temporal size + return: + pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token) + """ + grid_t = np.arange(t_size, dtype=np.float32) + pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], + axis=0) + return pos_embed + + +class Learnable2DInterpPosEmbDivided_fixed(nn.Module): + + def __init__(self, + height: int, + width: int, + num_frames: int, + dim: int, + interpolation_mode: str = 'bicubic') -> None: + super().__init__() + self.height = height + self.width = width + self.num_frames = num_frames + self.dim = dim + self.interpolation_mode = interpolation_mode + self.weight = nn.Parameter(torch.empty(height, width, dim)) + self.register_buffer('time_weight', + torch.from_numpy( + get_1d_sincos_pos_embed( + self.dim, + self.num_frames)).float().unsqueeze(1), + persistent=False) + + self.reset_parameters() + + def reset_parameters(self): + nn.init.normal_(self.weight) + + def forward(self, x: torch.Tensor, + grid_thws: torch.Tensor) -> torch.Tensor: + pos_embs = [] + for t, h, w in grid_thws.tolist(): + assert t <= self.num_frames, f't:{t} > self.num_frames:{self.num_frames}' + if (h, w) == self.weight.shape[:-1]: + pos_emb_2d = self.weight.flatten(end_dim=1) + else: + pos_emb_2d = get_rope_shape( + self.weight, + interpolation_mode=self.interpolation_mode, + shape=(h, w), + ) + + if t == 1: + pos_emb_3d = pos_emb_2d + else: + pos_emb_3d = pos_emb_2d.unsqueeze(0).repeat( + t, 1, 1) + self.time_weight[0:t] + + pos_embs.append(pos_emb_3d.reshape(-1, pos_emb_3d.shape[-1])) + + out = x + torch.cat(pos_embs) + return out + + +class MoonVision3dPatchEmbed(nn.Module): + + def __init__(self, + out_dim: int, + in_dim: int = 3, + patch_size: int | tuple[int, int] = (14, 14), + pos_emb_height: int = 14, + pos_emb_width: int = 14, + pos_emb_time: int = 4, + pos_emb_type: str = 'divided_fixed', + patch_embed_proj_bias: bool = True, + pos_emb_interpolation_mode: str = 'bicubic'): + super().__init__() + assert isinstance( + patch_size, + int | Sequence), f'Invalid patch_size type: {type(patch_size)}' + if isinstance(patch_size, int): + patch_size = (patch_size, patch_size) + assert (len(patch_size) == 2 + ), f'Expected patch_size to be a tuple of 2, got {patch_size}' + self.patch_size = patch_size + + self.proj = nn.Conv2d(in_dim, + out_dim, + kernel_size=patch_size, + stride=patch_size, + bias=patch_embed_proj_bias) + + if pos_emb_type == 'divided_fixed': + self.pos_emb = Learnable2DInterpPosEmbDivided_fixed( + height=pos_emb_height, + width=pos_emb_width, + num_frames=pos_emb_time, + dim=out_dim, + interpolation_mode=pos_emb_interpolation_mode) + else: + raise NotImplementedError( + f'Not support pos_emb_type: {pos_emb_type}') + + def forward(self, x: torch.Tensor, + grid_thws: torch.Tensor) -> torch.Tensor: + """ + Args: + x (L, Channels): input tensor + grid_hws (N, 3): temporal, height and width + + Returns: + (L, Cout) tensor + """ + x = self.proj(x).view(x.size(0), -1) + # apply positional embedding + x = self.pos_emb(x, grid_thws) + return x + + +class Rope2DPosEmbRepeated(nn.Module): + """2D rotary position embedding with multi-resolution support. + + This class is intended to be used in the following way: + 1. Before training, create an instance of Rope2DPosEmb. This instance will hold the precomputed cis. + 2. Before each forward pass, call `get_freqs_cis_by_*` to get the `freqs_cis` tensor for this iteration. + 3. During the forward pass, pass the `freqs_cis` tensor to each attention layer, and call `apply` just before each attention operation. + The rope is shared across all attention layers and all heads. + + Refs: + - RoFormer: https://arxiv.org/abs/2104.09864 + - VisionLLaMA: https://arxiv.org/abs/2403.00522 + - https://github.com/Meituan-AutoML/VisionLLaMA/blob/main/dit/models.py + + Args: + dim (int): usually the multi-head attention dimension, should be divisible by 4 (TODO: relax this constraint if needed) + max_height (int): the maximum height of the 2D grid + max_width (int): the maximum width of the 2D grid + theta_base (float): the base of the theta + device (str): the device to store the precomputed cis + """ + + def __init__(self, + dim: int, + max_height: int, + max_width: int, + theta_base=10000): + super().__init__() + self.dim = dim + assert self.dim % 4 == 0, 'dim must be divisible by 4' + self.max_height = max_height + self.max_width = max_width + self.theta_base = theta_base + + def extra_repr(self): + return f'dim={self.dim}, max_height={self.max_height}, max_width={self.max_width}, theta_base={self.theta_base}' + + def _precompute_freqs_cis(self, device: torch.device) -> torch.Tensor: + """Calculate the cis(freqs) for each position in the 2D grid. + + Return: complex tensor of shape (max_height, max_width, dim//2) and value: + height axis: ret[h, w, 2*i] = cis(h * theta_base**(-4*i/dim)) + weight axis: ret[h, w, 2*i+1] = cis(w * theta_base**(-4*i/dim)) with (i in [0, dim//4)) + note: `cis` is a mathematical notation defined by cis x = cos x + i sin x, + """ + N = self.max_height * self.max_width + flat_pos = torch.arange(0, N).float().to(device) + x_pos = flat_pos % self.max_width + y_pos = flat_pos // self.max_width + dim_range = (torch.arange(0, self.dim, + 4)[:(self.dim // 4)].float().to(device) + ) # C/4 + freqs = 1.0 / (self.theta_base**(dim_range / self.dim)) + x_freqs = torch.outer(x_pos, freqs).float() # N, C/4 + y_freqs = torch.outer(y_pos, freqs).float() # N, C/4 + x_cis = torch.polar(torch.ones_like(x_freqs), x_freqs) # N, C/4 + y_cis = torch.polar(torch.ones_like(y_freqs), y_freqs) # N, C/4 + # N, C/4, 2 + freqs_cis = torch.cat( + [x_cis.unsqueeze(dim=-1), + y_cis.unsqueeze(dim=-1)], dim=-1) + # max_height, max_width, C/2 + freqs_cis = freqs_cis.reshape(self.max_height, self.max_width, -1) + return freqs_cis + + def get_freqs_cis(self, grid_thws: torch.Tensor, + device: torch.device) -> torch.Tensor: + """ + Args: + grid_thws (torch.Tensor): grid time, height and width + + Returns: + freqs_cis: tensor of shape (sum(t * height * width), dim//2) + """ + if not hasattr(self, 'freqs_cis'): + self.register_buffer('freqs_cis', + self._precompute_freqs_cis(device), + persistent=False) + + shapes = grid_thws.tolist() + assert all(1 <= h <= self.max_height and 1 <= w <= self.max_width + for t, h, w in shapes), ( + shapes, + self.max_height, + self.max_width, + ) + freqs_cis = torch.cat( + [ + self.freqs_cis[:h, :w].reshape(-1, self.dim // 2).repeat(t, 1) + for t, h, w in shapes + ], + dim=0, + ) + return freqs_cis + + +class MLP2(nn.Module): + """ + Args: + dims: [in_dim, hidden_dim, out_dim] + bias: whether to use bias in linear layer. + """ + + def __init__(self, dims: list[int], activation, bias=True): + super().__init__() + assert len(dims) == 3 + self.fc0 = nn.Linear(dims[0], dims[1], bias=bias) + self.fc1 = nn.Linear(dims[1], dims[2], bias=bias) + self.activation = activation + for m in [self.fc0, self.fc1]: + nn.init.trunc_normal_(m.weight, std=math.sqrt(2 / m.in_features)) + if m.bias is not None: + nn.init.zeros_(m.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.fc0(x) + x = self.activation(x) + return self.fc1(x) + + +class MoonViTEncoderLayer(nn.Module): + + def __init__( + self, + num_heads: int, + hidden_dim: int, + mlp_dim: int, + qkv_hidden_size: int | None = None, + norm_type: str = 'layernorm', + mlp_type: str = 'mlp2', + *, + attn_implementation: str = 'flash_attention_2', + activation=F.gelu, + attn_bias: bool = False, + linear_bias: bool = True, + use_deterministic_attn: bool = False, + ): + super().__init__() + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.qkv_hidden_size = hidden_dim if qkv_hidden_size is None else qkv_hidden_size + self.hidden_size_per_attention_head = self.qkv_hidden_size // self.num_heads + self.attn_implementation = attn_implementation + self.use_deterministic_attn = use_deterministic_attn + + if norm_type == "layernorm": + self.norm0 = nn.LayerNorm(hidden_dim) + self.norm1 = nn.LayerNorm(hidden_dim) + elif norm_type == "rmsnorm": + self.norm0 = nn.RMSNorm(hidden_dim) + self.norm1 = nn.RMSNorm(hidden_dim) + else: + raise NotImplementedError(f"Not support norm_type: {norm_type}") + + if mlp_type == "mlp2": + self.mlp = MLP2([hidden_dim, mlp_dim, hidden_dim], + activation, + bias=linear_bias) + else: + raise NotImplementedError(f"Not support mlp_type: {mlp_type}") + + self.wqkv = nn.Linear(hidden_dim, + self.qkv_hidden_size * 3, + bias=attn_bias) + self.wo = nn.Linear(self.qkv_hidden_size, hidden_dim, bias=attn_bias) + + def attention_qkvpacked( + self, + x: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + rope_freqs_cis: torch.Tensor | None = None, + ): + """ + Args: + x (torch.Tensor): (batch_size, seqlen, hidden_dim) + cu_seqlens (torch.Tensor): + """ + xqkv = self.wqkv(x) + + qkv_shape = xqkv.size()[:-1] + ( + 3, + self.num_heads, + self.hidden_size_per_attention_head, + ) + # xqkv: (batch_size, seqlen, 3, nheads, headdim) + xqkv = xqkv.view(*qkv_shape) + xq, xk, xv = torch.unbind(xqkv, dim=-3) + + xq, xk = apply_rope(xq, xk, rope_freqs_cis) + + attn_func = VL_VISION_ATTENTION_FUNCTIONS[self.attn_implementation] + attn_out = attn_func(xq, + xk, + xv, + q_cu_seqlens=cu_seqlens, + k_cu_seqlens=cu_seqlens, + max_seqlen_k=max_seqlen, + max_seqlen_q=max_seqlen, + deterministic=self.use_deterministic_attn) + + attn_out = self.wo(attn_out) + return attn_out + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: int, + rope_freqs_cis: torch.Tensor | None = None, + ): + residual = hidden_states + hidden_states = self.norm0(hidden_states) + + hidden_states = self.attention_qkvpacked(hidden_states, cu_seqlens, + max_seqlen, rope_freqs_cis) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class MoonViT3dEncoder(nn.Module): + + def __init__(self, + hidden_dim: int, + num_layers: int, + block_cfg: dict, + use_deterministic_attn: bool = False) -> None: + super().__init__() + self.use_deterministic_attn = use_deterministic_attn + + qkv_hidden_size = block_cfg['hidden_dim'] if block_cfg.get( + 'qkv_hidden_size') is None else block_cfg['qkv_hidden_size'] + self.rope_2d = Rope2DPosEmbRepeated( + qkv_hidden_size // block_cfg['num_heads'], 512, 512) + self.blocks = nn.ModuleList([ + MoonViTEncoderLayer( + **block_cfg, + use_deterministic_attn=self.use_deterministic_attn) + for _ in range(num_layers) + ]) + norm_type = block_cfg.get('norm_type', 'layernorm') + if norm_type == "layernorm": + self.final_layernorm = nn.LayerNorm(hidden_dim) + elif norm_type == "rmsnorm": + self.final_layernorm = nn.RMSNorm(hidden_dim) + else: + raise NotImplementedError(f"Not support norm_type: {norm_type}") + + def forward( + self, + hidden_states: torch.Tensor, + grid_thws: torch.Tensor, + ) -> torch.Tensor: + rope_freqs_cis = self.rope_2d.get_freqs_cis( + grid_thws=grid_thws, device=hidden_states.device) + + lengths = torch.cat(( + torch.zeros(1, dtype=grid_thws.dtype, device=grid_thws.device), + grid_thws[:, 0] * grid_thws[:, 1] * grid_thws[:, 2], + )) + + max_seqlen = lengths.max() + cu_seqlens = lengths.to(hidden_states.device).cumsum(dim=0, + dtype=torch.int32) + for block in self.blocks: + hidden_states = block(hidden_states, + cu_seqlens, + max_seqlen, + rope_freqs_cis=rope_freqs_cis) + + hidden_states = self.final_layernorm(hidden_states) + return hidden_states + + +def tpool_patch_merger( + x: torch.Tensor, + grid_thws: torch.Tensor, + merge_kernel_size: tuple[int, int] = (2, 2), +) -> list[torch.Tensor]: + d_model = x.size(-1) + + outputs = [] + pre_sum = 0 + for t, h, w in grid_thws.tolist(): + # Get the current sequence + seq = x[pre_sum:pre_sum + t * h * w] + # Reshape along self.merge_kernel_size and concat to the last dimension + kernel_height, kernel_width = merge_kernel_size + new_height, new_width = h // kernel_height, w // kernel_width + reshaped_seq = seq.view(t, new_height, kernel_height, new_width, + kernel_width, d_model) + reshaped_seq = reshaped_seq.permute(0, 1, + 3, 2, 4, 5).contiguous().mean( + dim=0) # temporal pooling + padded_seq = reshaped_seq.view(new_height * new_width, + kernel_height * kernel_width, -1) + outputs.append(padded_seq) + pre_sum += t * h * w + + return outputs + + +class MoonViT3dPretrainedModel(PreTrainedModel): + config_class = None + model_type = 'moonvit3d' + _no_split_modules = ['MoonViTEncoderLayer'] + _supports_flash_attn_2 = True + _supports_sdpa = True + + def __init__(self, config, *inputs, **kwargs): + super().__init__(config, *inputs, **kwargs) + config = deepcopy(config) + self.merge_kernel_size = config.merge_kernel_size + self.patch_size = config.patch_size + self.merge_type = config.merge_type + + self.patch_embed = MoonVision3dPatchEmbed( + out_dim=config.hidden_size, + patch_size=config.patch_size, + pos_emb_height=config.init_pos_emb_height, + pos_emb_width=config.init_pos_emb_width, + pos_emb_time=config.init_pos_emb_time, + pos_emb_type=config.pos_emb_type, + patch_embed_proj_bias=getattr(config, 'patch_embed_proj_bias', + True), + pos_emb_interpolation_mode=getattr( + config, 'pos_emb_interpolation_mode', 'bicubic'), + ) + + self.encoder = MoonViT3dEncoder( + hidden_dim=config.hidden_size, + num_layers=config.num_hidden_layers, + block_cfg={ + 'num_heads': config.num_attention_heads, + 'hidden_dim': config.hidden_size, + 'qkv_hidden_size': getattr(config, 'qkv_hidden_size', None), + 'mlp_dim': config.intermediate_size, + 'norm_type': getattr(config, 'norm_type', 'layernorm'), + 'mlp_type': getattr(config, 'mlp_type', 'mlp2'), + 'activation': PytorchGELUTanh(), + 'attn_bias': getattr(config, 'attn_bias', True), + 'linear_bias': getattr(config, 'linear_bias', True), + 'attn_implementation': config._attn_implementation, + }, + use_deterministic_attn=getattr(self, 'use_deterministic_attn', + False)) + + def forward(self, pixel_values: torch.Tensor, + grid_thws: torch.Tensor) -> torch.Tensor: + """ + Args: + pixel_values (torch.Tensor): The input pixel values. + grid_thws (torch.Tensor): Temporal, height and width. + + Returns: + torch.Tensor: The output tokens. + """ + # grid_thws = grid_thws.to('cpu') + assert grid_thws.ndim == 2, f'grid_thws should be 2D, got {grid_thws.ndim}' + assert grid_thws.size(1) == 3, f'No support for thw: {grid_thws}' + hidden_states = self.patch_embed(pixel_values, grid_thws) + hidden_states = self.encoder(hidden_states, grid_thws) + if self.merge_type == 'sd2_tpool': # spatial downsampling 2x with temporal pooling all + hidden_states = tpool_patch_merger( + hidden_states, + grid_thws, + merge_kernel_size=self.merge_kernel_size) + else: + raise NotImplementedError(f'Not support {self.merge_type}') + + return hidden_states + + +# ============================================================================ +# MM Projector Helper Classes (from mm_projector/modeling_mm_projectors.py) +# ============================================================================ + + +class IdentityMap(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x, *args, **kwargs): + return x + + +class MLP(nn.Module): + + def __init__(self, config): + super().__init__() + # TODO, use faster LayerNorm + self.pre_norm = nn.LayerNorm(config.mm_hidden_size) + self.proj = nn.Sequential( + nn.Linear(config.mm_hidden_size, config.hidden_size), nn.GELU(), + nn.Linear(config.hidden_size, config.hidden_size)) + + def forward(self, x, *args, **kwargs): + assert isinstance(x, + list | tuple), f'x is not a list or tuple: {type(x)}' + lengths = [item.shape[0] for item in x] + x = torch.cat(x, dim=0) + x = self.pre_norm(x) + x = self.proj(x) + x = torch.split(x, lengths, dim=0) + + return x + + +class PatchMergerMLP(nn.Module): + + def __init__(self, config): + super().__init__() + eps = config.projector_ln_eps + self.hidden_size = config.mm_hidden_size * ( + config.merge_kernel_size[0] * config.merge_kernel_size[1]) + self.pre_norm = nn.LayerNorm(config.mm_hidden_size, eps=eps) + self.proj = nn.Sequential( + nn.Linear(self.hidden_size, self.hidden_size), + nn.GELU(), + nn.Linear(self.hidden_size, config.hidden_size), + ) + + def forward(self, x, *args, **kwargs): + if isinstance(x, list) or isinstance(x, tuple): + x = [ + self.proj(self.pre_norm(item).view(item.shape[0], -1)) + for item in x + ] + else: + # B, N, N_k, C = x.shape + B = x.shape[0] + x = self.proj(self.pre_norm(x).view(B, -1, self.hidden_size)) + return x + + +class PatchMergerMLPV2(nn.Module): + + def __init__(self, config): + super().__init__() + eps = config.projector_ln_eps + self.hidden_size = config.mm_hidden_size * ( + config.merge_kernel_size[0] * config.merge_kernel_size[1]) + self.proj = nn.Sequential( + nn.Linear(self.hidden_size, self.hidden_size, bias=False), + nn.GELU(), + nn.Linear(self.hidden_size, config.hidden_size, bias=False), + ) + self.post_norm = nn.RMSNorm(config.hidden_size, eps=eps) + for m in self.proj.modules(): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, + std=math.sqrt(2 / m.in_features)) + if m.bias is not None: + nn.init.zeros_(m.bias) + + def forward(self, x, *args, **kwargs): + if isinstance(x, list) or isinstance(x, tuple): + lengths = [item.shape[0] for item in x] + x = torch.concat([item.view(item.shape[0], -1) for item in x], + dim=0) + x = self.post_norm(self.proj(x)) + x = torch.split(x, lengths, dim=0) + else: + # B, N, N_k, C = x.shape + B = x.shape[0] + x = self.proj(x.view(B, -1, self.hidden_size)) + x = self.post_norm(x) + return x + + +class KimiK3PreTrainedModel(PreTrainedModel): + config_class = KimiK3Config + base_model_prefix = "model" + _no_split_modules = [ + "MoonViT3dPretrainedModel", + "MoonViTEncoderLayer", + "KimiDecoderLayer", + "PatchMergerMLP", + "PatchMergerMLPV2", + ] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = False + + def _init_weights(self, module): + # important: this ported version of Llava isn't meant for training from scratch - only + # inference and fine-tuning - so the proper init weights code has been removed - the original codebase + # https://github.com/haotian-liu/LLaVA/tree/main/llava should serve for that purpose + std = (self.config.initializer_range if hasattr( + self.config, "initializer_range") else + self.config.text_config.initializer_range) + + if hasattr(module, "class_embedding"): + module.class_embedding.data.normal_(mean=0.0, std=std) + + if isinstance(module, (nn.Linear, nn.Conv2d)): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +class VisionTowerConfig(PretrainedConfig): + model_type = 'moonvit3d' + + def __init__(self, config: KimiK3Config, **kwargs): + super().__init__(**kwargs) + self.patch_size = config.patch_size + self.init_pos_emb_height = config.init_pos_emb_height + self.init_pos_emb_width = config.init_pos_emb_width + self.init_pos_emb_time = config.init_pos_emb_time + self.pos_emb_type = config.pos_emb_type + self.num_attention_heads = config.vt_num_attention_heads + self.num_hidden_layers = config.vt_num_hidden_layers + self.hidden_size = config.vt_hidden_size + self.intermediate_size = config.vt_intermediate_size + self.merge_kernel_size = config.merge_kernel_size + self.merge_type = config.merge_type + self._attn_implementation = config._attn_implementation + self.qkv_hidden_size = getattr(config, 'qkv_hidden_size', None) + self.norm_type = getattr(config, 'norm_type', 'layernorm') + self.attn_bias = getattr(config, 'attn_bias', True) + self.patch_embed_proj_bias = getattr(config, 'patch_embed_proj_bias', + True) + self.mlp_type = getattr(config, 'mlp_type', 'mlp2') + self.linear_bias = getattr(config, 'linear_bias', True) + self.pos_emb_interpolation_mode = getattr( + config, 'pos_emb_interpolation_mode', 'bilinear') + + +class ProjectorConfig: + + def __init__(self, config: KimiK3Config): + self.mm_projector_type = config.mm_projector_type + self.mm_hidden_size = config.mm_hidden_size + self.hidden_size = config.text_hidden_size + self.merge_kernel_size = config.merge_kernel_size + self.projector_hidden_act = config.projector_hidden_act + self.projector_ln_eps = config.projector_ln_eps + + +# ref https://github.com/huggingface/transformers/blob/78b2929c0554b79e0489b451ce4ece14d265ead2/src/transformers/models/llava/modeling_llava.py#L240 +class KimiK3ForConditionalGeneration(KimiK3PreTrainedModel): + + @classmethod + def _supports_default_dynamic_cache(cls) -> bool: + return False + + def __init__(self, config: KimiK3Config): + super().__init__(config) + + vt_config = VisionTowerConfig(config.vision_config) + self.vision_tower = MoonViT3dPretrainedModel(vt_config) + + proj_config = ProjectorConfig(config.vision_config) + if proj_config.mm_projector_type == 'identity': + self.mm_projector = IdentityMap() + elif proj_config.mm_projector_type == 'mlp': + self.mm_projector = MLP(proj_config) + elif proj_config.mm_projector_type == 'patchmerger': + self.mm_projector = PatchMergerMLP(proj_config) + elif proj_config.mm_projector_type == 'patchmergerv2': + self.mm_projector = PatchMergerMLPV2(proj_config) + else: + raise ValueError( + f"Unsupported mm_projector_type: {proj_config.mm_projector_type}" + ) + + self.language_model = KimiLinearForCausalLM(config.text_config) + self.post_init() + + if hasattr(self.language_model, 'dtype'): + target_dtype = self.language_model.dtype + self.vision_tower = self.vision_tower.to(dtype=target_dtype) + self.mm_projector = self.mm_projector.to(dtype=target_dtype) + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def get_output_embeddings(self): + return self.language_model.get_output_embeddings() + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def set_decoder(self, decoder): + self.language_model.set_decoder(decoder) + + def get_decoder(self): + return self.language_model.get_decoder() + + def tie_weights(self): + return self.language_model.tie_weights() + + def resize_token_embeddings(self, + new_num_tokens: int | None = None, + pad_to_multiple_of=None) -> nn.Embedding: + model_embeds = self.language_model.resize_token_embeddings( + new_num_tokens, pad_to_multiple_of) + # update vocab size + self.config.text_config.vocab_size = model_embeds.num_embeddings + self.vocab_size = model_embeds.num_embeddings + return model_embeds + + def _merge_input_ids_with_image_features( + self, + image_features: list[torch.Tensor], + inputs_embeds: torch.Tensor, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + labels: torch.Tensor | None = None, + ): + """ + Args: + image_features (:obj:`torch.Tensor` of shape :obj:`(num_image_tokens, embed_dim)`): + The image features to merge with the input embeddings. + inputs_embeds (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length, embed_dim)`): + The input embeddings. + input_ids (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`): + The input ids. + attention_mask (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`): + The attention mask. + labels (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`, *optional*): + The labels. + """ + _, embed_dim = image_features[0].shape + feature_lengths = [x.shape[0] for x in image_features] + image_features = torch.cat(image_features, dim=0) + + image_token_index: int = self.config.media_placeholder_token_id + pad_token_id: int = self.config.pad_token_id + ignore_index: int = self.config.ignore_index + + batch_size, sequence_length = input_ids.shape + left_padding = not torch.sum( + input_ids[:, -1] == torch.tensor(pad_token_id)) + + # 1. Create a mask to know where special image tokens are + _token_occupation_table = torch.ones_like(input_ids.flatten()) + _token_occupation_table[input_ids.flatten() == + image_token_index] = torch.tensor( + feature_lengths, + dtype=torch.long, + device=input_ids.device) + _token_occupation_table = _token_occupation_table.reshape( + input_ids.shape) + + max_embed_dim = _token_occupation_table.sum(-1).max().item() + assert ( + max_embed_dim >= sequence_length + ), f"The maximum embedding dimension ({max_embed_dim}) is less than the sequence length ({sequence_length})" + batch_indices, non_image_indices = torch.where( + input_ids != image_token_index) + + # 2. Compute the positions where text should be written + # Calculate new positions for text tokens in merged image-text sequence. + new_token_positions = torch.cumsum(_token_occupation_table, -1) - 1 + nb_image_pad = max_embed_dim - 1 - new_token_positions[:, -1] + if left_padding: + new_token_positions += nb_image_pad[:, + None] # offset for left padding + text_to_overwrite = new_token_positions[batch_indices, + non_image_indices] + + # 3. Create the full embedding, already padded to the maximum position + final_embedding = torch.zeros( + batch_size, + max_embed_dim, + embed_dim, + dtype=inputs_embeds.dtype, + device=inputs_embeds.device, + ) + final_attention_mask = torch.zeros(batch_size, + max_embed_dim, + dtype=attention_mask.dtype, + device=inputs_embeds.device) + if labels is not None: + final_labels = torch.full( + (batch_size, max_embed_dim), + ignore_index, + dtype=input_ids.dtype, + device=input_ids.device, + ) + # In case the Vision model or the Language model has been offloaded to CPU, we need to manually + # set the corresponding tensors into their correct target device. + target_device = inputs_embeds.device + batch_indices, non_image_indices, text_to_overwrite = ( + batch_indices.to(target_device), + non_image_indices.to(target_device), + text_to_overwrite.to(target_device), + ) + attention_mask = attention_mask.to(target_device) + + # 4. Fill the embeddings based on the mask. + final_embedding[batch_indices, + text_to_overwrite] = inputs_embeds[batch_indices, + non_image_indices] + final_attention_mask[batch_indices, + text_to_overwrite] = attention_mask[ + batch_indices, non_image_indices] + if labels is not None: + final_labels[batch_indices, + text_to_overwrite] = labels[batch_indices, + non_image_indices] + + # 5. Fill the embeddings corresponding to the images. Anything that is not `text_positions` needs filling (#29835) + image_to_overwrite = torch.full((batch_size, max_embed_dim), + True, + dtype=torch.bool, + device=inputs_embeds.device) + image_to_overwrite[batch_indices, text_to_overwrite] = False + image_to_overwrite &= image_to_overwrite.cumsum( + -1) - 1 >= nb_image_pad[:, None].to(target_device) + + if image_to_overwrite.sum() != image_features.shape[:-1].numel(): + raise ValueError( + f"The input provided to the model are wrong. The number of image tokens is {image_to_overwrite.sum()} while" + f" the number of image features given to the model is {image_features.shape[:-1].numel()}. " + "This prevents correct indexing and breaks batch generation.") + + final_embedding[image_to_overwrite] = ( + image_features.contiguous().reshape(-1, + embed_dim).to(target_device)) + final_attention_mask |= image_to_overwrite + position_ids = (final_attention_mask.cumsum(-1) - 1).masked_fill_( + (final_attention_mask == 0), 1) + + # 6. Mask out the embedding at padding positions, as we later use the past_key_value value to determine the non-attended tokens. + batch_indices, pad_indices = torch.where(input_ids == pad_token_id) + indices_to_mask = new_token_positions[batch_indices, pad_indices] + + final_embedding[batch_indices, indices_to_mask] = 0 + + if labels is None: + final_labels = None + + return final_embedding, final_attention_mask, final_labels, position_ids + + def _extract_image_features(self, pixel_values: torch.Tensor, + grid_thws: torch.Tensor) -> list[torch.Tensor]: + """ + Args: + pixel_values (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_channels, height, width)`): + The pixel values of the images processed by image processor. + grid_thws (:obj:`torch.Tensor` of shape :obj:`(batch_size, 3)`): + The grid, height, width of the images. + + Returns: + selected_image_feature (:obj:`torch.FloatTensor` of shape :obj:`(num_image_tokens, embed_dim)`): + The selected image features to use as input to the projector head. + + """ + + target_dtype = self.vision_tower.patch_embed.proj.weight.dtype + pixel_values = pixel_values.to(target_dtype) + + image_features = self.vision_tower(pixel_values, grid_thws) + return image_features + + def forward( + self, + input_ids: torch.LongTensor | None = None, + pixel_values: torch.FloatTensor | list[torch.FloatTensor] + | None = None, + grid_thws: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: list[torch.FloatTensor] | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + ) -> tuple | LlavaCausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + ```""" + assert self.vision_tower is not None, "vision_tower is not loaded" + output_attentions = (output_attentions if output_attentions is not None + else self.config.output_attentions) + output_hidden_states = (output_hidden_states + if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if inputs_embeds is None: + # 1. Extra the input embeddings + inputs_embeds = self.get_input_embeddings()(input_ids) + + # 2. Merge text and images + if pixel_values is not None and len( + pixel_values) > 0 and input_ids.shape[1] != 1: + image_features = self._extract_image_features( + pixel_values, grid_thws) + if self.mm_projector: + image_features = self.mm_projector(image_features) + + inputs_embeds = inputs_embeds.to( + image_features[0].dtype) # num_tokens, embed_dim + inputs_embeds, attention_mask, labels, position_ids = ( + self._merge_input_ids_with_image_features( + image_features, + inputs_embeds, + input_ids, + attention_mask, + labels, + )) + + # In case input_ids.shape[1] == 1 & pixel_values==None & past_key_values != None, we are in the case of + # generation with cache + elif (past_key_values is not None and pixel_values is not None + and input_ids.shape[1] == 1): + # Retrieve the first layer to inspect the logits and mask out the hidden states + # that are set to 0 + first_layer_past_key_value = past_key_values[0][0][:, :, :, 0] + + # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941 + batch_index, non_attended_tokens = torch.where( + first_layer_past_key_value.float().sum(-2) == 0) + + # Get the target length + target_length = input_ids.shape[1] + past_length = first_layer_past_key_value.shape[-1] + + extended_attention_mask = torch.ones( + (attention_mask.shape[0], past_length), + dtype=attention_mask.dtype, + device=attention_mask.device, + ) + + # Filter out only the tokens that can be un-attended, this can happen + # if one uses Llava + Fused modules where the cache on the + # first iteration is already big enough, or if one passes custom cache + valid_indices = non_attended_tokens < extended_attention_mask.size( + -1) + new_batch_index = batch_index[valid_indices] + new_non_attended_tokens = non_attended_tokens[valid_indices] + + # Zero-out the places where we don't need to attend + extended_attention_mask[new_batch_index, + new_non_attended_tokens] = 0 + + attention_mask = torch.cat( + (extended_attention_mask, attention_mask[:, + -target_length:]), + dim=1) + position_ids = torch.sum(attention_mask, + dim=1).unsqueeze(-1) - 1 + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + logits = outputs[0] + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + if attention_mask is not None: + shift_attention_mask = attention_mask[..., 1:] + shift_logits = logits[..., :-1, :][shift_attention_mask.to( + logits.device) != 0].contiguous() + shift_labels = labels[..., 1:][shift_attention_mask.to( + labels.device) != 0].contiguous() + else: + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1).to(shift_logits.device), + ) + + if not return_dict: + output = (logits, ) + outputs[1:] + return (loss, ) + output if loss is not None else output + + return LlavaCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + inputs_embeds=None, + pixel_values=None, + grid_thws=None, + attention_mask=None, + **kwargs, + ): + if past_key_values is not None: + if hasattr(past_key_values, "get_seq_length"): + cache_length = past_key_values.get_seq_length() + past_length = getattr(past_key_values, 'seen_tokens', + cache_length) + else: + cache_length = past_length = past_key_values[0][0].shape[2] + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[ + 1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - + past_length):] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + elif self.config.media_placeholder_token_id in input_ids: + input_ids = input_ids[:, input_ids.shape[1] - 1:] + # If the cache has seen more tokens than it can hold, then the cache has a size limit. Let's discard the + # older attention values, as their corresponding values are not part of the input. + if cache_length < past_length and attention_mask is not None: + attention_mask = attention_mask[:, -(cache_length + + input_ids.shape[1]):] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1]:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update({ + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "grid_thws": grid_thws, + }) + return model_inputs + + def _reorder_cache(self, *args, **kwargs): + return self.language_model._reorder_cache(*args, **kwargs) diff --git a/modeling_kimi_linear.py b/modeling_kimi_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..b8c41e8bfce768d74d8da3a37e693f5ee43876a0 --- /dev/null +++ b/modeling_kimi_linear.py @@ -0,0 +1,1314 @@ +# coding=utf-8 +# Copyright 2025-2026 The Moonshot AI Team, DeepSeek-AI, and HuggingFace Inc. team. All rights reserved. +# +# The multi-head latent attention, MoE gating and sparse MoE block in this file are +# adapted from DeepSeek-V3 (DeepSeek-V3/modeling_deepseek.py). They have been +# extensively modified and extended for the Kimi-Linear architecture. +# +# Licensing Information: +# - Code adapted from DeepSeek-V3 (DeepSeek-V3/modeling_deepseek.py) is licensed under the Apache License, Version 2.0. +# - Other parts of the code are licensed under the Kimi K3 License (see the LICENSE file in this repository). +# +# Apache License, Version 2.0: +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from collections.abc import Callable +from typing import Any + +import torch +import torch.nn.functional as F +import transformers +from einops import rearrange +from packaging import version +from torch import nn +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache +from transformers.generation import GenerationMixin +from transformers.masking_utils import create_causal_mask +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from transformers.processing_utils import Unpack +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS +from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple, logging +from transformers.utils.generic import OutputRecorder, check_model_inputs + +try: + from fla.modules import FusedRMSNormGated, ShortConvolution + from fla.ops.kda import chunk_kda, fused_recurrent_kda + # from fla.ops.kda.gate import fused_kda_gate # deprecated, gate is now computed inside chunk_kda/fused_recurrent_kda + from fla.ops.utils.index import prepare_cu_seqlens_from_mask, prepare_lens_from_mask + from fla.utils import tensor_cache +except ImportError: + raise ImportError("Plese run `pip install -U fla-core`") + +from .configuration_kimi_k3 import KimiLinearConfig + +assert version.parse(transformers.__version__) >= version.parse("4.56.0"), \ + "Please upgrade transformers to >= 4.56.0" + +logger = logging.get_logger(__name__) + + +# Register Moonshot-specific activation functions +class SituAndMul(nn.Module): + """ + SituAndMul activation: beta * tanh(gate / beta) * sigmoid(gate) * up + When linear_beta is set, up is also transformed by linear_beta * tanh(up / linear_beta). + """ + + def __init__(self, beta: float = 1.0, linear_beta: float | None = None): + super().__init__() + self.beta = beta + self.linear_beta = linear_beta + + def forward(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + gate = x[..., :d].to(torch.float32) + up = x[..., d:].to(torch.float32) + situ_a = self.beta * torch.tanh(gate / self.beta) * torch.sigmoid(gate) + if self.linear_beta is not None: + up = self.linear_beta * torch.tanh(up / self.linear_beta) + return (situ_a * up).to(x.dtype) + + +ACT2FN["situ"] = SituAndMul + + +def _get_situ_activation_params(config: KimiLinearConfig): + beta = getattr(config, "activation_situ_beta", None) + linear_beta = getattr(config, "activation_situ_linear_beta", None) + return beta or 1.0, linear_beta + + +def index_first_axis(x, indices): + return x[indices] + + +@tensor_cache +def get_unpad_data( + attention_mask: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, int]: + lens = prepare_lens_from_mask(attention_mask) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = lens.max().item() + cu_seqlens = prepare_cu_seqlens_from_mask(attention_mask) + return indices, cu_seqlens, max_seqlen_in_batch + + +def pad_input( + hidden_states: torch.Tensor, + indices: torch.LongTensor, + batch_size: int, + seq_len: int, +) -> torch.Tensor: + out = hidden_states.new_zeros((batch_size * seq_len, *hidden_states.shape[1:])) + out[indices] = hidden_states + return out.view(batch_size, seq_len, *hidden_states.shape[1:]) + + +class KimiDynamicCache: + """ + Dynamic cache for Kimi model. + Inspired by Qwen3-Next + """ + is_compileable = False + + def __init__(self, config: KimiLinearConfig): + super().__init__() + self.config = config + + if config.linear_attn_config is not None: + self.layer_types = [] + for i in range(config.num_hidden_layers): + if config.is_kda_layer(i): + self.layer_types.append("linear_attention") + else: + self.layer_types.append("full_attention") + else: + self.layer_types = ["full_attention"] * config.num_hidden_layers + + self.transformer_layers = [ + i for i in range(config.num_hidden_layers) if self.layer_types[i] == "full_attention" + ] + + linear_layers = [i for i in range( + config.num_hidden_layers) if self.layer_types[i] == "linear_attention"] + self.last_linear_layer = linear_layers[-1] if linear_layers else -1 + + self.conv_states = [None for _ in range(config.num_hidden_layers)] + self.recurrent_states = [None for _ in range(config.num_hidden_layers)] + self.key_cache = [None for _ in range(config.num_hidden_layers)] + self.value_cache = [None for _ in range(config.num_hidden_layers)] + + def __len__(self): + return len(self.layer_types) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: dict[str, Any] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.key_cache[layer_idx] is None: + self.key_cache[layer_idx] = key_states + self.value_cache[layer_idx] = value_states + else: + self.key_cache[layer_idx] = torch.cat( + [self.key_cache[layer_idx], key_states], dim=2) + self.value_cache[layer_idx] = torch.cat( + [self.value_cache[layer_idx], value_states], dim=2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + for layer_idx in range(len(self.key_cache)): + if self.key_cache[layer_idx] is not None: + device = self.key_cache[layer_idx].device + beam_idx = beam_idx.to(device) + self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select( + 0, beam_idx) + self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select( + 0, beam_idx) + + if self.conv_states[layer_idx] is not None: + device = self.conv_states[layer_idx][0].device + beam_idx = beam_idx.to(device) + q_conv, k_conv, v_conv = self.conv_states[layer_idx] + self.conv_states[layer_idx] = ( + q_conv.index_select(0, beam_idx), + k_conv.index_select(0, beam_idx), + v_conv.index_select(0, beam_idx), + ) + self.recurrent_states[layer_idx] = self.recurrent_states[layer_idx].index_select( + 0, beam_idx) + + def get_seq_length(self, layer_idx: int | None = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # take any layer that contains cache and not empty tensor + layer_idx = self.transformer_layers[0] if layer_idx not in self.transformer_layers else layer_idx + if len(self.key_cache) <= layer_idx or self.key_cache[layer_idx] is None: + return 0 + return self.key_cache[layer_idx].shape[-2] + + def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + """ + Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for + the given layer at `layer_idx`. + The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. + """ + kv_offset = 0 + query_length = cache_position.shape[0] + past_seen_tokens = self.get_seq_length(layer_idx) + kv_length = query_length + past_seen_tokens + return kv_length, kv_offset + + @property + def has_previous_state(self): + """We have a previous state if the last linear (conv) layer was already updated.""" + if self.last_linear_layer == -1: + return False + return self.conv_states[self.last_linear_layer] is not None + + +class KimiRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + dtype = hidden_states.dtype + x = hidden_states.float() + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) + return self.weight * x.to(dtype) + + +ALL_LAYERNORM_LAYERS.append(KimiRMSNorm) + + +class KimiBlockSparseMLP(nn.Module): + def __init__(self, config: KimiLinearConfig, hidden_size=None, intermediate_size=None): + super().__init__() + self.config = config + self.ffn_dim = config.intermediate_size if intermediate_size is None else intermediate_size + self.hidden_dim = config.hidden_size if hidden_size is None else hidden_size + + self.w1 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False) # gate + self.w2 = nn.Linear(self.ffn_dim, self.hidden_dim, bias=False) # down + self.w3 = nn.Linear(self.hidden_dim, self.ffn_dim, bias=False) # up + + if config.hidden_act == "situ": + beta, linear_beta = _get_situ_activation_params(config) + self.act_fn = SituAndMul( + beta=beta, + linear_beta=linear_beta, + ) + else: + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + if self.config.hidden_act == "situ": + gate_up = torch.cat([self.w1(hidden_states), self.w3(hidden_states)], dim=-1) + current_hidden_states = self.act_fn(gate_up) + else: + current_hidden_states = self.act_fn( + self.w1(hidden_states)) * self.w3(hidden_states) + current_hidden_states = self.w2(current_hidden_states) + return current_hidden_states + + +class KimiMLP(nn.Module): + def __init__(self, config: KimiLinearConfig, hidden_size=None, intermediate_size=None): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size if hidden_size is None else hidden_size + self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size + self.gate_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear( + self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear( + self.intermediate_size, self.hidden_size, bias=False) + if config.hidden_act == "situ": + beta, linear_beta = _get_situ_activation_params(config) + self.act_fn = SituAndMul( + beta=beta, + linear_beta=linear_beta, + ) + else: + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + if self.config.hidden_act == "situ": + gate_up = torch.cat([self.gate_proj(x), self.up_proj(x)], dim=-1) + down_proj = self.down_proj(self.act_fn(gate_up)) + else: + down_proj = self.down_proj(self.act_fn( + self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """Expand the key/value heads from `num_key_value_heads` to `num_attention_heads`.""" + if n_rep == 1: + return hidden_states + return torch.repeat_interleave(hidden_states, dim=1, repeats=n_rep) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key = repeat_kv(key, module.num_key_value_groups) + value = repeat_kv(value, module.num_key_value_groups) + + scores = torch.einsum("bhqd,bhkd->bhqk", query, key) * scaling + if attention_mask is not None: + scores = scores + attention_mask[:, :, :, : key.shape[-2]] + + probs = F.softmax(scores, dim=-1, dtype=torch.float32).to(query.dtype) + probs = F.dropout(probs, p=dropout, training=module.training) + out = torch.einsum("bhqk,bhkd->bhqd", probs, value).transpose(1, 2).contiguous() + + return out, probs + + +class KimiMLAAttention(nn.Module): + """ + Multi-Latent Attention adapted from deepseek-v3 + """ + + def __init__(self, config: KimiLinearConfig, layer_idx: int): + nn.Module.__init__(self) + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + + self.attention_dropout = getattr(config, "attention_dropout", 0.0) + + try: + self.q_lora_rank = config.q_lora_rank + self.qk_rope_head_dim = config.qk_rope_head_dim + self.kv_lora_rank = config.kv_lora_rank + self.v_head_dim = config.v_head_dim + self.qk_nope_head_dim = config.qk_nope_head_dim + self.q_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + self.use_nope = config.mla_use_nope + self.scaling = self.q_head_dim ** (-0.5) + except Exception as e: + raise ValueError( + f"Kimi MLA config is not found or not properly formatted: {e}") + + if self.q_lora_rank is not None: + self.q_a_proj = nn.Linear( + self.hidden_size, self.q_lora_rank, bias=False, + ) + self.q_a_layernorm = KimiRMSNorm(self.q_lora_rank) + self.q_b_proj = nn.Linear( + self.q_lora_rank, + self.num_heads * self.q_head_dim, + bias=False, + ) + else: + self.q_proj = nn.Linear( + self.hidden_size, self.num_heads * self.q_head_dim, bias=False, + ) + self.kv_a_proj_with_mqa = nn.Linear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + ) + self.kv_a_layernorm = KimiRMSNorm(self.kv_lora_rank) + self.kv_b_proj = nn.Linear( + self.kv_lora_rank, + self.num_heads + * (self.q_head_dim - self.qk_rope_head_dim + self.v_head_dim), + bias=False, + ) + self.o_proj = nn.Linear( + self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + ) + self.is_causal = True + assert self.use_nope + + self.use_output_gate = getattr(config, "mla_use_output_gate", False) + if self.use_output_gate: + projection_size = self.num_heads * self.v_head_dim + self.g_proj = nn.Linear(self.hidden_size, projection_size, bias=False) + + self.rotary_emb = None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + batch_size, seq_length = hidden_states.shape[:-1] + query_shape = (batch_size, seq_length, -1, self.q_head_dim) + key_shape = (batch_size, seq_length, -1, + self.qk_nope_head_dim + self.v_head_dim) + + if self.q_lora_rank is not None: + q_states = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states))) + else: + q_states = self.q_proj(hidden_states) + q_states = q_states.view(query_shape).transpose(1, 2) + q_pass, q_rot = torch.split( + q_states, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + + compressed_kv = self.kv_a_proj_with_mqa(hidden_states) + k_pass, k_rot = torch.split( + compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + + k_pass = self.kv_b_proj(self.kv_a_layernorm( + k_pass)).view(key_shape).transpose(1, 2) + k_pass, value_states = torch.split( + k_pass, [self.qk_nope_head_dim, self.v_head_dim], dim=-1) + + k_rot = k_rot.view(batch_size, 1, seq_length, self.qk_rope_head_dim) + + k_rot = k_rot.expand(*k_pass.shape[:-1], -1) + + query_states = torch.cat((q_pass, q_rot), dim=-1) + key_states = torch.cat((k_pass, k_rot), dim=-1) + + if past_key_values is not None: + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx) + + if self.config._attn_implementation == "flash_attention_2" and self.q_head_dim != self.v_head_dim: + value_states = F.pad( + value_states, [0, self.q_head_dim - self.v_head_dim]) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, _ = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + if self.config._attn_implementation == "flash_attention_2" and self.q_head_dim != self.v_head_dim: + attn_output = attn_output[:, :, :, : self.v_head_dim] + + attn_output = attn_output.reshape( + batch_size, seq_length, -1).contiguous() + if self.use_output_gate: + g = self.g_proj(hidden_states).sigmoid() + attn_output = attn_output * g + attn_output = self.o_proj(attn_output) + return attn_output + + +class KimiDeltaAttention(nn.Module): + def __init__(self, config: KimiLinearConfig, layer_idx: int): + super().__init__() + self.config = config + self.mode = "chunk" + + self.hidden_size = config.hidden_size + self.conv_size = config.linear_attn_config["short_conv_kernel_size"] + self.head_dim = config.linear_attn_config["head_dim"] + self.num_heads = config.linear_attn_config["num_heads"] + self.head_k_dim = self.head_dim + self.num_k_heads = self.num_heads + + self.layer_idx = layer_idx + + assert self.mode in [ + 'chunk', 'fused_recurrent'], f"Not supported mode `{self.mode}`." + + projection_k_size = self.head_k_dim * self.num_k_heads + projection_size = self.head_dim * self.num_heads + + self.q_proj = nn.Linear( + self.hidden_size, projection_k_size, bias=False) + self.k_proj = nn.Linear( + self.hidden_size, projection_k_size, bias=False) + self.v_proj = nn.Linear(self.hidden_size, projection_size, bias=False) + + self.q_conv1d = ShortConvolution( + hidden_size=projection_k_size, + kernel_size=self.conv_size, + activation='silu', + ) + self.k_conv1d = ShortConvolution( + hidden_size=projection_k_size, + kernel_size=self.conv_size, + activation='silu', + ) + self.v_conv1d = ShortConvolution( + hidden_size=projection_size, + kernel_size=self.conv_size, + activation='silu', + ) + + self.A_log = torch.nn.Parameter(torch.log(torch.empty( + self.num_heads, dtype=torch.float32).uniform_(1, 16))) + + self.f_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False) + self.f_b_proj = nn.Linear(self.head_dim, projection_size, bias=False) + + self.dt_bias = nn.Parameter( + torch.empty(projection_size, dtype=torch.float32)) + + self.b_proj = nn.Linear(self.hidden_size, self.num_heads, bias=False) + + self.use_full_rank_gate = config.linear_attn_config.get("use_full_rank_gate", False) + self.gate_lower_bound = config.linear_attn_config.get("gate_lower_bound", None) + if self.use_full_rank_gate: + self.g_proj = nn.Linear(self.hidden_size, projection_size, bias=False) + else: + self.g_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False) + self.g_b_proj = nn.Linear(self.head_dim, projection_size, bias=False) + + self.o_norm = FusedRMSNormGated( + self.head_dim, eps=config.rms_norm_eps, activation='sigmoid') + self.o_proj = nn.Linear(projection_size, self.hidden_size, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + cache_params: KimiDynamicCache | None = None, + **kwargs: Unpack[dict], + ) -> tuple[torch.Tensor, torch.Tensor | None, Cache | None]: + if attention_mask is not None: + if attention_mask.dim() != 2: + attention_mask = kwargs.get("padding_mask") + + if attention_mask is not None and attention_mask.dim() != 2: + raise ValueError( + "attention_mask must be a 0-1 matrix of shape [batch_size, seq_len] " + "(0 = padding). 3D masks are not supported here.", + ) + use_cache = cache_params is not None + batch_size, q_len, _ = hidden_states.shape + mode = 'fused_recurrent' if use_cache and q_len == 1 else self.mode + if self.training: + assert mode == 'chunk', "Only chunk mode is supported in training." + + cu_seqlens = kwargs.get('cu_seqlens') + indices = None + if attention_mask is not None: + indices, cu_seqlens, _ = get_unpad_data(attention_mask[:, -q_len:]) + hidden_states = index_first_axis( + rearrange(hidden_states, "b s ... -> (b s) ..."), indices).unsqueeze(0) + + conv_state_q, conv_state_k, conv_state_v = None, None, None + recurrent_state = None + if cache_params is not None: + if cache_params.conv_states[self.layer_idx] is not None: + conv_state_q, conv_state_k, conv_state_v = cache_params.conv_states[ + self.layer_idx] + recurrent_state = cache_params.recurrent_states[self.layer_idx] + + q_proj_states = self.q_proj(hidden_states) + k_proj_states = self.k_proj(hidden_states) + v_proj_states = self.v_proj(hidden_states) + q, conv_state_q = self.q_conv1d( + x=q_proj_states, + cache=conv_state_q, + output_final_state=use_cache, + cu_seqlens=cu_seqlens, + ) + k, conv_state_k = self.k_conv1d( + x=k_proj_states, + cache=conv_state_k, + output_final_state=use_cache, + cu_seqlens=cu_seqlens, + ) + v, conv_state_v = self.v_conv1d( + x=v_proj_states, + cache=conv_state_v, + output_final_state=use_cache, + cu_seqlens=cu_seqlens, + ) + g = self.f_b_proj(self.f_a_proj(hidden_states)) + g = rearrange(g, '... (h d) -> ... h d', d=self.head_dim) + beta = self.b_proj(hidden_states).float() + + q, k = map(lambda x: rearrange( + x, '... (h d) -> ... h d', d=self.head_k_dim), (q, k)) + v = rearrange(v, '... (h d) -> ... h d', d=self.head_dim) + + if mode == 'chunk': + o, recurrent_state = chunk_kda( + q=q, + k=k, + v=v, + g=g, + beta=beta, + A_log=self.A_log, + dt_bias=self.dt_bias, + initial_state=recurrent_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + use_gate_in_kernel=True, + use_beta_sigmoid_in_kernel=True, + safe_gate=self.gate_lower_bound is not None, + lower_bound=self.gate_lower_bound, + transpose_state_layout=True, + cu_seqlens=cu_seqlens, + ) + else: + o, recurrent_state = fused_recurrent_kda( + q=q, + k=k, + v=v, + g=g, + beta=beta, + A_log=self.A_log, + dt_bias=self.dt_bias, + initial_state=recurrent_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + use_gate_in_kernel=True, + use_beta_sigmoid_in_kernel=True, + lower_bound=self.gate_lower_bound, + transpose_state_layout=True, + cu_seqlens=cu_seqlens, + ) + if cache_params is not None: + cache_params.recurrent_states[self.layer_idx] = recurrent_state + cache_params.conv_states[self.layer_idx] = ( + conv_state_q, conv_state_k, conv_state_v) + + if self.use_full_rank_gate: + g = self.g_proj(hidden_states) + else: + g = self.g_b_proj(self.g_a_proj(hidden_states)) + g = rearrange(g, '... (h d) -> ... h d', d=self.head_dim) + o = self.o_norm(o, g) + + o = rearrange(o, 'b t h d -> b t (h d)') + o = self.o_proj(o) + if attention_mask is not None: + o = pad_input(o.squeeze(0), indices, batch_size, q_len) + + return o + + +class KimiMoEGate(nn.Module): + """ + MoEGate adapted from Deepseek-V3. + Parameter correspondences: + num_experts -> n_routed_experts + num_experts_per_token -> num_experts_per_tok + num_expert_group -> n_group + moe_router_activation_func -> scoring_func + """ + + def __init__(self, config: KimiLinearConfig): + super().__init__() + self.config = config + self.top_k = config.num_experts_per_token + self.num_experts = config.num_experts + self.routed_scaling_factor = config.routed_scaling_factor + self.moe_router_activation_func = config.moe_router_activation_func + self.num_expert_group = getattr(config, "num_expert_group", 1) + self.topk_group = getattr(config, "topk_group", 1) + + # topk selection algorithm + self.moe_renormalize = config.moe_renormalize + self.gating_dim = config.hidden_size + self.weight = nn.Parameter( + torch.empty((self.num_experts, self.gating_dim)), + ) + + self.e_score_correction_bias = nn.Parameter( + torch.empty(self.num_experts), + ) + self.reset_parameters() + + def reset_parameters(self) -> None: + import torch.nn.init as init + + init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + + def forward(self, hidden_states): + bsz, seq_len, h = hidden_states.shape + # compute gating score + hidden_states = hidden_states.view(-1, h) + logits = F.linear( + hidden_states.type(torch.float32), self.weight.type( + torch.float32), None, + ) + if self.moe_router_activation_func == "sigmoid": + scores = logits.sigmoid() + elif self.moe_router_activation_func == "softmax": + scores = logits.softmax(dim=1) + else: + raise NotImplementedError( + f"insupportable scoring function for MoE gating: {self.moe_router_activation_func}", + ) + + # select top-k experts + assert not self.training + scores = scores.view(bsz * seq_len, -1) + scores_for_choice = scores + self.e_score_correction_bias.unsqueeze(0) + if self.num_expert_group > 1 and self.num_expert_group > self.topk_group: + group_scores = ( + scores_for_choice.view( + bsz * seq_len, self.num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1) + ) # [n, num_expert_group] + group_idx = torch.topk( + group_scores, k=self.topk_group, dim=-1, sorted=False, + )[ + 1 + ] # [n, top_k_group] + group_mask = torch.zeros_like(group_scores) # [n, num_expert_group] + group_mask.scatter_(1, group_idx, 1) # [n, num_expert_group] + score_mask = ( + group_mask.unsqueeze(-1) + .expand( + bsz * seq_len, self.num_expert_group, self.num_experts // self.num_expert_group, + ) + .reshape(bsz * seq_len, -1) + ) # [n, e] + tmp_scores = scores_for_choice.masked_fill( + ~score_mask.bool(), float("-inf")) # [n, e] + else: + tmp_scores = scores_for_choice + _, topk_idx = torch.topk( + tmp_scores, k=self.top_k, dim=-1, sorted=False, + ) + topk_weight = scores.gather(1, topk_idx) + + # norm gate to sum 1 + if self.top_k > 1 and self.moe_renormalize: + denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20 + topk_weight = topk_weight / denominator + # must multiply the scaling factor + topk_weight = topk_weight * self.routed_scaling_factor + + return topk_idx, topk_weight + + +class KimiSparseMoeBlock(nn.Module): + """ + Adapted from Deepseek-V3's MOE implementation + The namings are consistent with Kimi's version. + """ + + def __init__(self, config: KimiLinearConfig): + super().__init__() + self.config = config + self.hidden_dim = config.hidden_size + self.num_experts = config.num_experts + self.top_k = config.num_experts_per_token + self.moe_renormalize = config.moe_renormalize + + self.use_latent_moe = getattr(config, "routed_expert_hidden_size", None) is not None + self.moe_hidden_size = ( + config.routed_expert_hidden_size + if self.use_latent_moe else config.hidden_size + ) + self.latent_moe_use_norm = getattr(config, "latent_moe_use_norm", False) + + self.ep_size = 1 + self.experts_per_rank = config.num_experts + self.ep_rank = 0 + self.experts = nn.ModuleList( + [ + KimiBlockSparseMLP( + config, + hidden_size=self.moe_hidden_size, + intermediate_size=config.moe_intermediate_size, + ) + for _ in range(config.num_experts) + ], + ) + self.gate = KimiMoEGate(config) + if config.num_shared_experts is not None: + intermediate_size = config.moe_intermediate_size * config.num_shared_experts + self.shared_experts = KimiMLP( + config=config, intermediate_size=intermediate_size, + ) + + if self.use_latent_moe: + self.routed_expert_down_proj = nn.Linear( + config.hidden_size, self.moe_hidden_size, bias=False, + ) + self.routed_expert_up_proj = nn.Linear( + self.moe_hidden_size, config.hidden_size, bias=False, + ) + if self.latent_moe_use_norm: + self.routed_expert_norm = KimiRMSNorm( + self.moe_hidden_size, eps=config.rms_norm_eps, + ) + + def forward(self, hidden_states): + identity = hidden_states + orig_shape = hidden_states.shape + topk_idx, topk_weight = self.gate(hidden_states) + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + + if self.use_latent_moe: + hidden_states = self.routed_expert_down_proj(hidden_states) + + if not self.training: + y = self.moe_infer(hidden_states, topk_idx, topk_weight) + else: + raise NotImplementedError("Training mode is not supported in KimiSparseMoeBlock") + + if self.use_latent_moe: + if self.latent_moe_use_norm: + y = self.routed_expert_norm(y) + y = self.routed_expert_up_proj(y) + + y = y.view(*orig_shape) + + if self.config.num_shared_experts is not None: + y = y + self.shared_experts(identity) + return y + + @torch.no_grad() + def moe_infer(self, x, topk_ids, topk_weight): + cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts))) + cnts.scatter_(1, topk_ids, 1) + tokens_per_expert = cnts.sum(dim=0) + idxs = topk_ids.view(-1).argsort() + sorted_tokens = x[idxs // topk_ids.shape[1]] + + tokens_per_expert = tokens_per_expert.cpu().numpy() + + outputs = [] + start_idx = 0 + for i, num_tokens in enumerate(tokens_per_expert): + end_idx = start_idx + num_tokens + if num_tokens == 0: + continue + expert = self.experts[i + self.ep_rank * self.experts_per_rank] + tokens_for_this_expert = sorted_tokens[start_idx:end_idx] + expert_out = expert(tokens_for_this_expert) + outputs.append(expert_out) + start_idx = end_idx + + outs = torch.cat(outputs, dim=0) if len( + outputs) else sorted_tokens.new_empty(0) + + new_x = torch.empty_like(outs) + new_x[idxs] = outs + final_out = ( + new_x.view(*topk_ids.shape, -1) + .type(topk_weight.dtype) + .mul_(topk_weight.unsqueeze(dim=-1)) + .sum(dim=1) + .type(new_x.dtype) + ) + return final_out + + +class KimiDecoderLayer(nn.Module): + def __init__(self, config: KimiLinearConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.config = config + self.layer_idx = layer_idx + if config.is_kda_layer(layer_idx): + self.is_linear_attn = True + self.self_attn = KimiDeltaAttention( + config=config, layer_idx=layer_idx) + elif config.is_mla: + self.is_linear_attn = False + self.self_attn = KimiMLAAttention( + config=config, layer_idx=layer_idx) + else: + raise NotImplementedError + if ( + config.num_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % getattr(config, "moe_layer_freq", 1) == 0 + ): + self.block_sparse_moe = KimiSparseMoeBlock(config) + else: + self.mlp = KimiMLP(config) + self.input_layernorm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + + # Attention residual + self.use_attn_residuals = getattr(config, "attn_res_block_size", None) is not None + if self.use_attn_residuals: + self.attn_res_block_size = config.attn_res_block_size + self.self_attention_res_norm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + self.mlp_res_norm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + self.self_attention_res_proj = nn.Linear( + config.hidden_size, 1, bias=False) + self.mlp_res_proj = nn.Linear( + config.hidden_size, 1, bias=False) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: tuple[torch.Tensor] | None = None, + output_attentions: bool | None = False, + use_cache: bool | None = False, + block_residual: torch.Tensor | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ): + if self.use_attn_residuals: + return self._forward_attn_residual( + hidden_states, attention_mask, position_ids, + past_key_values, output_attentions, use_cache, + block_residual, **kwargs) + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + if self.is_linear_attn is False: + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + **kwargs, + ) + else: + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + cache_params=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + if hasattr(self, "block_sparse_moe"): + hidden_states = self.block_sparse_moe(hidden_states) + else: + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + def _forward_attn_residual( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: tuple[torch.Tensor] | None = None, + output_attentions: bool | None = False, + use_cache: bool | None = False, + block_residual: torch.Tensor | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ): + batch_size, seq_len, hidden_size = hidden_states.shape + prefix_sum = hidden_states + + if block_residual is not None and block_residual.shape[1] > 0: + hidden_states = _apply_attn_res( + prefix_sum.view(-1, hidden_size), + block_residual, + self.self_attention_res_proj, + self.self_attention_res_norm, + ).view(batch_size, seq_len, hidden_size) + + if self.layer_idx % self.attn_res_block_size == 0: + block_residual = torch.cat( + [block_residual, prefix_sum.view(-1, hidden_size).unsqueeze(1)], dim=1) + prefix_sum = None + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + if self.is_linear_attn is False: + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + **kwargs, + ) + else: + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + cache_params=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + **kwargs, + ) + + if prefix_sum is not None: + prefix_sum = prefix_sum + hidden_states + else: + prefix_sum = hidden_states + + hidden_states = _apply_attn_res( + prefix_sum.view(-1, hidden_size), + block_residual, + self.mlp_res_proj, + self.mlp_res_norm, + ).view(batch_size, seq_len, hidden_size) + + hidden_states = self.post_attention_layernorm(hidden_states) + if hasattr(self, "block_sparse_moe"): + hidden_states = self.block_sparse_moe(hidden_states) + else: + hidden_states = self.mlp(hidden_states) + + if prefix_sum is None: + prefix_sum = hidden_states + else: + prefix_sum = prefix_sum + hidden_states + + return prefix_sum, block_residual + + +class KimiPreTrainedModel(PreTrainedModel): + config_class = KimiLinearConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["KimiDecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _can_record_outputs = { + "router_logits": OutputRecorder(KimiBlockSparseMLP, index=1), + "hidden_states": KimiDecoderLayer, + "attentions": KimiMLAAttention, + } + _is_stateful = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +def _apply_attn_res(prefix_sum, block_residual, proj, norm): + """ + prefix_sum: (num_tokens, hidden_size) + block_residual: (num_tokens, num_blocks, hidden_size) + """ + v = torch.cat((block_residual, prefix_sum.unsqueeze(1)), dim=1) + v_float = v.float() + variance = v_float.pow(2).mean(-1, keepdim=True) + k = v_float * torch.rsqrt(variance + norm.variance_epsilon) + score_weight = norm.weight.float() * proj.weight.squeeze(0).float() + scores = (k * score_weight).sum(-1) + probs = scores.softmax(-1).unsqueeze(1) + hidden_states = torch.matmul(probs, v_float).squeeze(1) + return hidden_states.to(v.dtype) + +class KimiLinearModel(KimiPreTrainedModel): + def __init__(self, config: KimiLinearConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding( + config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList([KimiDecoderLayer( + config, layer_idx) for layer_idx in range(config.num_hidden_layers)]) + self.norm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + + self.use_attn_residuals = getattr(config, "attn_res_block_size", None) is not None + if self.use_attn_residuals: + self.output_attn_res_norm = KimiRMSNorm( + config.hidden_size, eps=config.rms_norm_eps) + self.output_attn_res_proj = nn.Linear( + config.hidden_size, 1, bias=False) + + if getattr(config, "_attn_implementation", None) is not None: + if config._attn_implementation != "flash_attention_2": + logger.warning_once( + f"Ignoring the provided attention implementation {config._attn_implementation}") + logger.warning_once("Using flash_attention_2 backend instead.") + config._attn_implementation = "flash_attention_2" + else: + config._attn_implementation = "flash_attention_2" + + self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def _update_linear_attn_mask(self, attention_mask, cache_position): + """ + NOTE: Left-padding is used for linear attention mask. + No need for zeroing states when + 1. Cached forward + 2. Attending to all inputs + """ + linear_attn_mask = attention_mask + if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + linear_attn_mask = None + return linear_attn_mask + + @check_model_inputs + # @auto_docstring + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + cache_position: torch.LongTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPast: + + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if (input_ids is None) and (inputs_embeds is None): + raise ValueError( + "You must specify exactly one of input_ids or inputs_embeds") + + # Get inputs_embeds + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = KimiDynamicCache(config=self.config) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length( + ) if past_key_values is not None else 0 + cache_position: torch.Tensor = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device, + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + input_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) + linear_attn_mask = self._update_linear_attn_mask( + attention_mask, cache_position) + + hidden_states = inputs_embeds + if past_key_values is not None: + assert isinstance(past_key_values, KimiDynamicCache) + + block_residual = None + if self.use_attn_residuals: + block_residual = hidden_states.new_zeros( + hidden_states.shape[0] * hidden_states.shape[1], 0, + hidden_states.shape[2]) + + for decoder_layer in self.layers: + layer_mask = linear_attn_mask if decoder_layer.is_linear_attn else causal_mask + + if self.use_attn_residuals: + hidden_states, block_residual = decoder_layer( + hidden_states, + attention_mask=layer_mask, + past_key_values=past_key_values, + cache_position=cache_position, + block_residual=block_residual, + **kwargs, + ) + else: + hidden_states = decoder_layer( + hidden_states, + attention_mask=layer_mask, + past_key_values=past_key_values, + cache_position=cache_position, + **kwargs, + ) + + if self.use_attn_residuals: + hidden_states = self._apply_output_attn_res( + hidden_states, block_residual) + + hidden_states = self.norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + def _apply_output_attn_res(self, hidden_states, block_residual): + batch_size, seq_len, hidden_size = hidden_states.shape + return _apply_attn_res( + hidden_states.view(-1, hidden_size), + block_residual, + self.output_attn_res_proj, + self.output_attn_res_norm, + ).view(batch_size, seq_len, hidden_size) + + +class KimiLinearForCausalLM(KimiPreTrainedModel, GenerationMixin): + @classmethod + def _supports_default_dynamic_cache(cls) -> bool: + return False + + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = KimiLinearModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + # @auto_docstring + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: list[torch.FloatTensor] | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + generation_mode: bool | None = None, + return_dict: bool | None = None, + cache_position: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | CausalLMOutputWithPast: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + logits = outputs[0] + if generation_mode: + logits = logits[:, -1:] + logits = self.lm_head(logits) + + loss = None + if labels is not None: + loss = self.loss_function( + logits, labels, self.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8be28c13c93fb8982f81928d3f4fe80568102286 --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,38 @@ +{ + "media_proc_cfg": { + "in_patch_limit": 65536, + "patch_size": 14, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "merge_kernel_size": 2, + "fixed_output_tokens": null, + "patch_limit_on_one_side": 512, + "in_patch_limit_each_frame": 16384, + "in_patch_limit_video": 655360, + "sample_fps": 8.0, + "max_num_frames_each_video": null, + "temporal_merge_kernel_size": 4, + "timestamp_mode": "hh:mm:ss.fff", + "transparent_bg_config": { + "pattern": "chessboard", + "chessboard_square_size": 8, + "chessboard_square_on_top_left": true, + "chessboard_white_value": 255, + "chessboard_gray_value": 180 + }, + "transparent_bg_fill_stage": "after_resize", + "config_type": "media_proc.processors.moonvit.MoonViTMediaProcessorConfig" + }, + "auto_map": { + "AutoProcessor": "kimi_k3_processor.KimiK3Processor", + "AutoImageProcessor": "kimi_k3_vision_processing.KimiK3VisionProcessor" + } +} \ No newline at end of file diff --git a/tiktoken.model b/tiktoken.model new file mode 100644 index 0000000000000000000000000000000000000000..b4149a6e17a01b6442187f39890f89bc2fe8d309 --- /dev/null +++ b/tiktoken.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c497a7469b33ced9c38afb1ad6e47f03f5e5dc05f15930799210ec050c5103 +size 2795286 diff --git a/tokenization_kimi.py b/tokenization_kimi.py new file mode 100644 index 0000000000000000000000000000000000000000..5df9d462b43295245084fbab46fd956020d4b84e --- /dev/null +++ b/tokenization_kimi.py @@ -0,0 +1,408 @@ +import os +from logging import getLogger +from pathlib import Path +from shutil import copyfile +from typing import Dict, Iterator, List, Optional, Tuple, Union, cast + +import tiktoken +from tiktoken.load import load_tiktoken_bpe +from tokenizers import AddedToken +from transformers.convert_slow_tokenizer import bytes_to_unicode +from transformers.tokenization_utils import PreTrainedTokenizer + +try: + from .encoding_k3 import build_chat_segments, is_batched_conversation +except ImportError: # pragma: no cover - supports direct file execution/import. + from encoding_k3 import build_chat_segments, is_batched_conversation + +logger = getLogger(__name__) +VOCAB_FILES_NAMES = {"vocab_file": "tiktoken.model"} + + +class TikTokenTokenizer(PreTrainedTokenizer): + """ + Tokenizing and encoding/decoding text using the Tiktoken tokenizer. See megatron/tokenizer/tiktoken_tokenizer.py. + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + The path to the Tiktoken model file. + bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|begin_of_text|>",`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|end_of_text|>"`): + The end of sequence token. + unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|reserved_special_token_249|>"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. The second to last item in special_tokens. + pad_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<|reserved_special_token_250|>"`): + The token used for padding, for example when batching sequences of different lengths. + additional_special_tokens (list of `str`, *optional*): + A tuple or a list of additional tokens, which will be marked as `special`, meaning that they will be + skipped when decoding if `skip_special_tokens` is set to `True`. + """ + + vocab_files_names = VOCAB_FILES_NAMES + + model_input_names = ["input_ids", "attention_mask"] + + special_tokens: Dict[str, int] + + num_reserved_special_tokens = 256 + + pat_str = "|".join([ + r"""[\p{Han}]+""", + r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", + r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""", + r"""\p{N}{1,3}""", + r""" ?[^\s\p{L}\p{N}]+[\r\n]*""", + r"""\s*[\r\n]+""", + r"""\s+(?!\S)""", + r"""\s+""", + ]) + + def __init__( + self, + vocab_file, + bos_token: Union[str, AddedToken] = "[BOS]", + eos_token: Union[str, AddedToken] = "[EOS]", + unk_token: Union[str, AddedToken, None] = None, + pad_token: Union[str, AddedToken, None] = None, + additional_special_tokens: List[str] = None, + added_tokens_decoder: Optional[dict] = None, + **kwargs, + ): + assert os.path.isfile(vocab_file), vocab_file + + if additional_special_tokens is None: + additional_special_tokens = [ + "<|im_end|>", + "<|im_user|>", + "<|im_assistant|>", + "<|start_header_id|>", + "<|end_header_id|>", + "[EOT]", + "<|im_system|>", + "<|im_middle|>", + ] + + if added_tokens_decoder: + special_tokens_mapping = { + i: added_tokens_decoder[i].content + for i in added_tokens_decoder + } + else: + special_tokens_mapping = {} + + self.vocab_file = vocab_file + mergeable_ranks = load_tiktoken_bpe(vocab_file) + num_base_tokens = len(mergeable_ranks) + self.special_tokens = { + special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i + for i in range(num_base_tokens, num_base_tokens + + self.num_reserved_special_tokens) + } + + self.model = tiktoken.Encoding( + name=Path(vocab_file).name, + pat_str=self.pat_str, + mergeable_ranks=mergeable_ranks, + special_tokens=self.special_tokens, + ) + logger.info(f"Reloaded tiktoken model from {vocab_file}") + + self.n_words: int = self.model.n_vocab + # BOS / EOS token IDs + self.bos_id: int = self.special_tokens[str(bos_token)] + self.eos_id: int = self.special_tokens[str(eos_token)] + logger.info( + f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}" + ) + + self.pad_id: int = self.special_tokens[str(pad_token)] + self.unk_id: int = self.special_tokens[str(unk_token)] + + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + + self.decoder = {} + for i in range(self.n_words): + # Taken from https://gist.github.com/xenova/a452a6474428de0182b17605a98631ee + decoding = ''.join([ + self.byte_encoder[ord(char)] for char in + self.model.decode_single_token_bytes(i).decode('latin-1') + ]) + self.decoder[i] = decoding + + self.encoder = {} + for i in range(self.n_words): + if i in self.decoder: + self.encoder[self.decoder[i]] = i + + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + additional_special_tokens=additional_special_tokens, + added_tokens_decoder=added_tokens_decoder, + **kwargs, + ) + self.all_special_ids_set = set(self.all_special_ids) + + def _encode_text_piece(self, text: str, + allow_special_tokens: bool = True) -> List[int]: + # The tiktoken tokenizer can handle <=400k chars without + # pyo3_runtime.PanicException. + TIKTOKEN_MAX_ENCODE_CHARS = 400_000 + + # https://github.com/openai/tiktoken/issues/195 + # Here we iterate over subsequences and split if we exceed the limit + # of max consecutive non-whitespace or whitespace characters. + MAX_NO_WHITESPACES_CHARS = 25_000 + + t: List[int] = [] + for i in range(0, len(text), TIKTOKEN_MAX_ENCODE_CHARS): + for substr in self._split_whitespaces_or_nonwhitespaces( + text[i:i + TIKTOKEN_MAX_ENCODE_CHARS], + MAX_NO_WHITESPACES_CHARS, + ): + if allow_special_tokens: + t.extend( + # structural markers: encode <|...|> as their special token IDs + self.model.encode( + substr, + allowed_special="all", + )) + else: + t.extend( + # user/tool text: encode any <|...|> as ordinary BPE tokens (never as control tokens) + self.model.encode( + substr, + disallowed_special=(), + )) + + return t + + def encode(self, + text: str, + allow_special_tokens: bool = True, + **kwargs) -> List[int]: + """ + Encodes a string into a list of token IDs. + + Args: + text (str): The input string to be encoded. + + Returns: + list[int]: A list of token IDs. + """ + # If there are other args, we should call super().encode because there are a lot of code + # to handle those args. supper().encode finally will call _tokenize and _convert_token_to_id. + # NOTE: our encode method is not compatible with the super().encode method, + # e.g. split_special_tokens' default is True in our encode method. + if len(kwargs) > 0: + logger.warning(f"Calling super().encode with {kwargs}") + return super().encode(text, **kwargs) + + assert type(text) is str + return self._encode_text_piece(text, + allow_special_tokens=allow_special_tokens) + + def decode(self, token_ids: Union[int, List[int]], **kwargs) -> str: + """ + Decodes a list of token IDs into a string. + + Args: + token_ids (List[int]): The list of token IDs to be decoded. + + Returns: + str: The decoded string. + """ + # If there are other args, we should call super().decode because there are a lot of code + # to handle those args. supper().encode finally will call convert_tokens_to_string and _convert_id_to_token. + if len(kwargs) > 0: + return super().decode(token_ids, **kwargs) + + if type(token_ids) is int: + token_ids = [token_ids] + + return self.model.decode(cast(List[int], token_ids)) + + @staticmethod + def _split_whitespaces_or_nonwhitespaces( + s: str, max_consecutive_slice_len: int) -> Iterator[str]: + """ + Splits the string `s` so that each substring contains no more than `max_consecutive_slice_len` + consecutive whitespaces or consecutive non-whitespaces. + """ + current_slice_len = 0 + current_slice_is_space = s[0].isspace() if len(s) > 0 else False + slice_start = 0 + + for i in range(len(s)): + is_now_space = s[i].isspace() + + if current_slice_is_space ^ is_now_space: + current_slice_len = 1 + current_slice_is_space = is_now_space + else: + current_slice_len += 1 + if current_slice_len > max_consecutive_slice_len: + yield s[slice_start:i] + slice_start = i + current_slice_len = 1 + yield s[slice_start:] + + def _encode_chat_segments(self, segments) -> List[int]: + token_ids: List[int] = [] + for segment in segments: + token_ids.extend( + self._encode_text_piece( + segment.text, + allow_special_tokens=segment.allow_special, + )) + return token_ids + + @staticmethod + def _truncate(ids: List[int], + truncation: bool = False, + max_length: Optional[int] = None) -> List[int]: + if truncation and max_length is not None: + return ids[:max_length] + return ids + + def _format_chat_token_output(self, + encoded_inputs: List[List[int]], + *, + is_batched: bool, + padding=False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors=None, + return_dict: bool = False): + encoded_inputs = [ + self._truncate(ids, truncation=truncation, max_length=max_length) + for ids in encoded_inputs + ] + + needs_batch_encoding = ( + is_batched or padding or return_tensors is not None or return_dict) + if not needs_batch_encoding: + return encoded_inputs[0] + + features = [{ + "input_ids": ids, + "attention_mask": [1] * len(ids) + } for ids in encoded_inputs] + batch = self.pad(features, + padding=padding, + max_length=max_length if padding else None, + return_attention_mask=True, + return_tensors=return_tensors) + + if return_dict: + return batch + if is_batched: + return batch["input_ids"] + return batch["input_ids"][0] if return_tensors is None else batch[ + "input_ids"] + + """ ----- Below are the abstract methods required by PreTrainedTokenizer ----- """ + + @property + def vocab_size(self) -> int: + return self.n_words + + def get_vocab(self) -> Dict[str, int]: + return self.encoder + + def _tokenize(self, text: str, **kwargs) -> List[str]: + return [self.decoder[t] for t in self.encode(text)] + + def _convert_token_to_id(self, token: str) -> int: + return self.encoder.get(token, self.unk_id) + + def _convert_id_to_token(self, index: int) -> str: + return self.decoder.get(index) + + @staticmethod + def clean_up_tokenization(out_string: str) -> str: + return out_string + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + text = ''.join(tokens) + text = bytearray([self.byte_decoder[c] + for c in text]).decode('utf-8', 'replace') + return text + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + if not os.path.isdir(save_directory): + raise ValueError( + f"vocabulary path ({save_directory}) should be a directory") + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"]) + + if os.path.abspath(self.vocab_file) != os.path.abspath( + out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file, ) + + def apply_chat_template(self, + conversation, + tools: Optional[list[dict]] = None, + tokenize: bool = False, + add_generation_prompt: bool = True, + thinking: bool = True, + padding=False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors=None, + return_dict: bool = False, + **kwargs): + # Tokenizer-level rendering reorders tool result messages to match + # assistant tool_calls, normalizes per-call arguments and response + # schema, then encodes the resulting XTML structure segment-by-segment. + is_batched = is_batched_conversation(conversation) + conversations = conversation if is_batched else [conversation] + image_prompts = kwargs.pop("image_prompts", None) + if is_batched and image_prompts is not None: + raise ValueError("image_prompts is only supported for one chat.") + + # by default set thinking effort to max + kwargs.setdefault("thinking_effort", "max") + + segment_batches = [ + build_chat_segments( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + thinking=thinking, + image_prompts=image_prompts, + **kwargs, + ) for messages in conversations + ] + + if not tokenize: + rendered = ["".join(segment.text for segment in segments) + for segments in segment_batches] + return rendered if is_batched else rendered[0] + + encoded_inputs = [ + self._encode_chat_segments(segments) for segments in segment_batches + ] + return self._format_chat_token_output( + encoded_inputs, + is_batched=is_batched, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=return_dict, + ) diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9fad8611e641403de7d6d5c48d0f2e92c20006 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,215 @@ +{ + "added_tokens_decoder": { + "163584": { + "content": "[BOS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163585": { + "content": "[EOS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163586": { + "content": "<|end_of_msg|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163587": { + "content": "<|open|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163588": { + "content": "<|close|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163589": { + "content": "<|sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163590": { + "content": "[start_header_id]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163591": { + "content": "[end_header_id]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163593": { + "content": "[EOT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163602": { + "content": "<|media_begin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163603": { + "content": "<|media_content|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163604": { + "content": "<|media_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163605": { + "content": "<|media_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163838": { + "content": "[UNK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163839": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163840": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163841": { + "content": "<|im_user|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163842": { + "content": "<|im_assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163843": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163844": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163845": { + "content": "<|im_system|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163846": { + "content": "<|im_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "auto_map": { + "AutoTokenizer": [ + "tokenization_kimi.TikTokenTokenizer", + null + ] + }, + "backend": "custom", + "bos_token": "[BOS]", + "clean_up_tokenization_spaces": false, + "eos_token": "[EOS]", + "extra_special_tokens": [ + "<|im_end|>", + "<|im_user|>", + "<|im_assistant|>", + "<|start_header_id|>", + "<|end_header_id|>", + "[EOT]", + "<|im_system|>", + "<|im_middle|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "[PAD]", + "tokenizer_class": "TikTokenTokenizer", + "tool_parser_type": "kimi_k3", + "unk_token": "[UNK]" +}