diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a09db2ea4d1bd1fc1c09f6fca45e8e4054953535 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e14ce68b0a559601dc936eba5e7c4d1c3d8ffc84 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Zhipu AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b2f24214ed04ea0044bc1e4e371c8b2e99e7a64 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +--- +license: mit +base_model: zai-org/GLM-5.2 +pipeline_tag: text-generation +tags: + - moe + - reap + - pruning + - router-kd + - knowledge + - nvfp4 + - glm + - glm-5.2 +--- + +# GLM-5.2-504B-K — knowledge-augmented REAP keep-168 (full-data Router-KD, NVFP4) + +The **"K-cut"** sibling of [`0xSero/GLM-5.2-504B`](https://huggingface.co/0xSero/GLM-5.2-504B): the +same ~504B / keep-168 budget, but the expert selection is **biased toward knowledge & reasoning** — +the winning top-160 core (kept bit-for-bit) **plus the 8 highest-priority knowledge-exclusive experts +per layer** that coding-saliency pruning drops. Recovered with gate-only **Router-KD trained on the +FULL calibration set (~18.6k real traces)** — 6x the data of the first-pass cuts. + +## Sponsor +8x NVIDIA B200 sponsored by [Lambda](https://lambda.ai). Thank you. + +## Why this variant exists +REAP saliency computed from coding traces under-weights experts that fire mainly on +reasoning/knowledge. The K-cut deliberately re-includes them — trading a sliver of coding-saliency +coverage for broader knowledge coverage. Reach for this on knowledge/reasoning-heavy workloads; use +the plain [`GLM-5.2-504B`](https://huggingface.co/0xSero/GLM-5.2-504B) otherwise. + +## Eval (n=2000 held-out real prompts, raw, no max_tokens / no timeout) +| metric | GLM-5.2-504B-K (this) | GLM-5.2-504B (plain floor) | +|---|---|---| +| attractor / loop rate | **0.078** | 0.072 | +| natural-EOS rate | 0.923 | 0.928 | +| distinct-4 | 0.881 | 0.880 | +| median tokens | 1232 | 1267 | + +On the loop metric this is **at parity with (or better than) the plain-cut floor**. The residual loops are inherent to GLM-5.2 itself (the unpruned teacher +loops on the same prompts), so neither cut "fixes" them — this one buys knowledge-expert coverage. + +## Serving (vLLM) +```bash +vllm serve 0xSero/GLM-5.2-504B-K --tensor-parallel-size 8 \ + --quantization modelopt_fp4 --kv-cache-dtype fp8 --trust-remote-code --max-model-len 262144 +``` + +--- +*REAP knowledge-augmented cut + full-data Router-KD. Compute sponsored by [Lambda](https://lambda.ai).* diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..672034e3a1196d8b9fc0116ddc037cb554227f89 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,119 @@ +[gMASK] +{%- set effective_reasoning_effort = 'high' if reasoning_effort is defined and reasoning_effort == 'high' else 'max' -%} +{%- if (enable_thinking is not defined or enable_thinking) and effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%} +{%- if tools -%} +{%- macro tool_to_json(tool) -%} + {%- set ns_tool = namespace(first=true) -%} + {{ '{' -}} + {%- for k, v in tool.items() -%} + {%- if k != 'defer_loading' and k != 'strict' -%} + {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%} + {%- set ns_tool.first = false -%} + "{{ k }}": {{ v | tojson(ensure_ascii=False) }} + {%- endif -%} + {%- endfor -%} + {{- '}' -}} +{%- endmacro -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} +{%- endif -%} +{% if tool.defer_loading is not defined or not tool.defer_loading %} +{{ tool_to_json(tool) }} +{% endif %} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- elif item is mapping and item.type in ['image', 'image_url', 'video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%} + {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%} + {{- "You are unable to process this " ~ media_type ~ " because you don't have multi-modal input ability. Try different methods." }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) -%} +{%- for m in messages %} + {%- if m.role == 'user' %} + {%- set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{%- for m in messages -%} +{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- elif '' in content %} + {%- set reasoning_content = content.split('')[0].split('')[-1] %} + {%- set content = content.split('')[-1] %} +{%- endif %} +{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%} +{{ '' + reasoning_content + ''}} +{%- else -%} +{{ '' }} +{%- endif -%} +{%- if content.strip() -%} +{{ content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{- '' + tc.name -}} +{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' -}} +{%- endif %} +{%- if m.content is string -%} + {{- '' + m.content + '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%} + {{- '\n' -}} + {% for tr in m.content %} + {%- for tool in tools -%} + {%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} + {%- endif -%} + {%- if tool.name == tr.name -%} + {{- tool_to_json(tool) + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0 is mapping and m.content.0.output is defined -%} + {%- for tr in m.content -%} + {{- '' + tr.output + '' -}} + {%- endfor -%} +{%- else -%} + {{- '' + visible_text(m.content) + '' -}} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|>{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c94efb8a16efa6d82146e659782503cbe782ca52 --- /dev/null +++ b/config.json @@ -0,0 +1,418 @@ +{ + "architectures": [ + "GlmMoeDsaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "ep_size": 1, + "first_k_dense_replace": 3, + "head_dim": 192, + "hidden_act": "silu", + "hidden_size": 6144, + "index_head_dim": 128, + "index_n_heads": 32, + "index_share_for_mtp_iteration": true, + "index_skip_topk_offset": 3, + "index_topk": 2048, + "index_topk_freq": 4, + "index_topk_pattern": null, + "indexer_rope_interleave": true, + "indexer_types": [ + "full", + "full", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared" + ], + "initializer_range": 0.02, + "intermediate_size": 12288, + "kv_lora_rank": 512, + "max_position_embeddings": 1048576, + "mlp_layer_types": [ + "dense", + "dense", + "dense", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse" + ], + "model_type": "glm_moe_dsa", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 1, + "n_routed_experts": 168, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 8, + "num_hidden_layers": 78, + "num_key_value_heads": 64, + "num_nextn_predict_layers": 1, + "pad_token_id": 154820, + "pretraining_tp": 1, + "q_lora_rank": 2048, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-05, + "rope_interleave": true, + "rope_parameters": { + "rope_theta": 8000000, + "rope_type": "default" + }, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 1, + "topk_method": "noaux_tc", + "transformers_version": "5.12.0", + "use_cache": true, + "v_head_dim": 256, + "vocab_size": 154880, + "quantization_config": { + "config_groups": { + "group_0": { + "input_activations": { + "dynamic": false, + "num_bits": 4, + "type": "float", + "group_size": 16 + }, + "weights": { + "dynamic": false, + "num_bits": 4, + "type": "float", + "group_size": 16 + }, + "targets": [ + "Linear" + ] + } + }, + "ignore": [ + "lm_head", + "model.layers.0.mlp*", + "model.layers.0.self_attn*", + "model.layers.0.self_attn.indexer*", + "model.layers.1.mlp*", + "model.layers.1.self_attn*", + "model.layers.1.self_attn.indexer*", + "*shared_experts*", + "model.layers.10.self_attn*", + "model.layers.10.self_attn.indexer*", + "model.layers.11.self_attn*", + "model.layers.11.self_attn.indexer*", + "model.layers.12.self_attn*", + "model.layers.12.self_attn.indexer*", + "model.layers.13.self_attn*", + "model.layers.13.self_attn.indexer*", + "model.layers.14.self_attn*", + "model.layers.14.self_attn.indexer*", + "model.layers.15.self_attn*", + "model.layers.15.self_attn.indexer*", + "model.layers.16.self_attn*", + "model.layers.16.self_attn.indexer*", + "model.layers.17.self_attn*", + "model.layers.17.self_attn.indexer*", + "model.layers.18.self_attn*", + "model.layers.18.self_attn.indexer*", + "model.layers.19.self_attn*", + "model.layers.19.self_attn.indexer*", + "model.layers.2.mlp*", + "model.layers.2.self_attn*", + "model.layers.2.self_attn.indexer*", + "model.layers.20.self_attn*", + "model.layers.20.self_attn.indexer*", + "model.layers.21.self_attn*", + "model.layers.21.self_attn.indexer*", + "model.layers.22.self_attn*", + "model.layers.22.self_attn.indexer*", + "model.layers.23.self_attn*", + "model.layers.23.self_attn.indexer*", + "model.layers.24.self_attn*", + "model.layers.24.self_attn.indexer*", + "model.layers.25.self_attn*", + "model.layers.25.self_attn.indexer*", + "model.layers.26.self_attn*", + "model.layers.26.self_attn.indexer*", + "model.layers.27.self_attn*", + "model.layers.27.self_attn.indexer*", + "model.layers.28.self_attn*", + "model.layers.28.self_attn.indexer*", + "model.layers.29.self_attn*", + "model.layers.29.self_attn.indexer*", + "model.layers.3.self_attn*", + "model.layers.3.self_attn.indexer*", + "model.layers.30.self_attn*", + "model.layers.30.self_attn.indexer*", + "model.layers.31.self_attn*", + "model.layers.31.self_attn.indexer*", + "model.layers.32.self_attn*", + "model.layers.32.self_attn.indexer*", + "model.layers.33.self_attn*", + "model.layers.33.self_attn.indexer*", + "model.layers.34.self_attn*", + "model.layers.34.self_attn.indexer*", + "model.layers.35.self_attn*", + "model.layers.35.self_attn.indexer*", + "model.layers.36.self_attn*", + "model.layers.36.self_attn.indexer*", + "model.layers.37.self_attn*", + "model.layers.37.self_attn.indexer*", + "model.layers.38.self_attn*", + "model.layers.38.self_attn.indexer*", + "model.layers.39.self_attn*", + "model.layers.39.self_attn.indexer*", + "model.layers.4.self_attn*", + "model.layers.4.self_attn.indexer*", + "model.layers.40.self_attn*", + "model.layers.40.self_attn.indexer*", + "model.layers.41.self_attn*", + "model.layers.41.self_attn.indexer*", + "model.layers.42.self_attn*", + "model.layers.42.self_attn.indexer*", + "model.layers.43.self_attn*", + "model.layers.43.self_attn.indexer*", + "model.layers.44.self_attn*", + "model.layers.44.self_attn.indexer*", + "model.layers.45.self_attn*", + "model.layers.45.self_attn.indexer*", + "model.layers.46.self_attn*", + "model.layers.46.self_attn.indexer*", + "model.layers.47.self_attn*", + "model.layers.47.self_attn.indexer*", + "model.layers.48.self_attn*", + "model.layers.48.self_attn.indexer*", + "model.layers.49.self_attn*", + "model.layers.49.self_attn.indexer*", + "model.layers.5.self_attn*", + "model.layers.5.self_attn.indexer*", + "model.layers.50.self_attn*", + "model.layers.50.self_attn.indexer*", + "model.layers.51.self_attn*", + "model.layers.51.self_attn.indexer*", + "model.layers.52.self_attn*", + "model.layers.52.self_attn.indexer*", + "model.layers.53.self_attn*", + "model.layers.53.self_attn.indexer*", + "model.layers.54.self_attn*", + "model.layers.54.self_attn.indexer*", + "model.layers.55.self_attn*", + "model.layers.55.self_attn.indexer*", + "model.layers.56.self_attn*", + "model.layers.56.self_attn.indexer*", + "model.layers.57.self_attn*", + "model.layers.57.self_attn.indexer*", + "model.layers.58.self_attn*", + "model.layers.58.self_attn.indexer*", + "model.layers.59.self_attn*", + "model.layers.59.self_attn.indexer*", + "model.layers.6.self_attn*", + "model.layers.6.self_attn.indexer*", + "model.layers.60.self_attn*", + "model.layers.60.self_attn.indexer*", + "model.layers.61.self_attn*", + "model.layers.61.self_attn.indexer*", + "model.layers.62.self_attn*", + "model.layers.62.self_attn.indexer*", + "model.layers.63.self_attn*", + "model.layers.63.self_attn.indexer*", + "model.layers.64.self_attn*", + "model.layers.64.self_attn.indexer*", + "model.layers.65.self_attn*", + "model.layers.65.self_attn.indexer*", + "model.layers.66.self_attn*", + "model.layers.66.self_attn.indexer*", + "model.layers.67.self_attn*", + "model.layers.67.self_attn.indexer*", + "model.layers.68.self_attn*", + "model.layers.68.self_attn.indexer*", + "model.layers.69.self_attn*", + "model.layers.69.self_attn.indexer*", + "model.layers.7.self_attn*", + "model.layers.7.self_attn.indexer*", + "model.layers.70.self_attn*", + "model.layers.70.self_attn.indexer*", + "model.layers.71.self_attn*", + "model.layers.71.self_attn.indexer*", + "model.layers.72.self_attn*", + "model.layers.72.self_attn.indexer*", + "model.layers.73.self_attn*", + "model.layers.73.self_attn.indexer*", + "model.layers.74.self_attn*", + "model.layers.74.self_attn.indexer*", + "model.layers.75.self_attn*", + "model.layers.75.self_attn.indexer*", + "model.layers.76.self_attn*", + "model.layers.76.self_attn.indexer*", + "model.layers.77.self_attn*", + "model.layers.77.self_attn.indexer*", + "model.layers.8.self_attn*", + "model.layers.8.self_attn.indexer*", + "model.layers.9.self_attn*", + "model.layers.9.self_attn.indexer*", + "model.layers.78.eh_proj*", + "model.layers.78.self_attn*", + "model.layers.78.self_attn.indexer*", + "model.layers.78.mlp.gate*", + "model.layers.78.mlp.shared_experts*" + ], + "quant_algo": "NVFP4", + "producer": { + "name": "modelopt", + "version": "0.39.0.dev290+gf9d9a71de.d20260214" + }, + "quant_method": "modelopt" + } +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..216bbb092d0df0dda1dc4cedd789b92286c4c001 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "pad_token_id": 154820, + "temperature": 1.0, + "top_p": 0.95, + "transformers_version": "5.12.0" +} diff --git a/model-00001.safetensors b/model-00001.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2146c90dba1234dc628580452c5e2558b6d6f108 --- /dev/null +++ b/model-00001.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e32eb17744ce168464db1e52af4fb3466a0b2f0d25e930f64bf097ed064e5562 +size 5000912520 diff --git a/model-00002.safetensors b/model-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abbe2e3a06db8d6b88c8365ad9007621da442203 --- /dev/null +++ b/model-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926df1b551c1d19fe56730b5d705298cf859e18ce718fdf41e04ffdd3ebba3ec +size 5000636364 diff --git a/model-00004.safetensors b/model-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ecfccfe20ff4e54ad27770c328be8df47a2bc3eb --- /dev/null +++ b/model-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1367004c73763aaf5309bd7e24e387e59f254a593857c07973875a03f625452 +size 5166417984 diff --git a/model-00005.safetensors b/model-00005.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b31c49c1cfe4b2a0ead085dedcff6c4d9e693b1 --- /dev/null +++ b/model-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc4a23a2ef1259e4b8f72cdaf98308d56d88f4b0c041ea795678ec2fda02b246 +size 5000910116 diff --git a/model-00009.safetensors b/model-00009.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..20c36cdd4a25e55c63e00cbe33e76d5474208731 --- /dev/null +++ b/model-00009.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3236d6d8f76afef822f4d5c7e04add890a92503f5a87b4c682e980d4f865d6d0 +size 5000241880 diff --git a/model-00011.safetensors b/model-00011.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bf006384416fbedfdd921df19eb28b32ce4a9e25 --- /dev/null +++ b/model-00011.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84aeb90f97ed617c77f2cf1d24ecddef737717db14b043988cbc341f4ac40e17 +size 5006010796 diff --git a/model-00012.safetensors b/model-00012.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a573ae9a0b903b9af1bff03628f76383883ee497 --- /dev/null +++ b/model-00012.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1c499a02258f7a02cdb9c74957dd5c147b408a2a56990a6dc2cc6e793bf149 +size 5001422756 diff --git a/model-00013.safetensors b/model-00013.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5cbc8b5be103d0ef8d8624cc85d7fd9683955ae9 --- /dev/null +++ b/model-00013.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaad58ef5b6d8201a3226c483a0ffc517a8abc6ec18fe098719126fc57de777c +size 5005596112 diff --git a/model-00014.safetensors b/model-00014.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..adb6733ce684c9baa192c3d74bcfb11079694b4c --- /dev/null +++ b/model-00014.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522f8001146b46ff930a805ba9c29be328cbb229efc0ab945be3b260fdf4dce8 +size 5006010788 diff --git a/model-00016.safetensors b/model-00016.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3989eb7c7751ca8b1648c821f80e24a6d85eefd7 --- /dev/null +++ b/model-00016.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519be3732637e34abde9b962b80c14debee206eb90918734b9057ab02beedc20 +size 5001422748 diff --git a/model-00020.safetensors b/model-00020.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..07649f5758fa232d804c5fa29d320126cedbf277 --- /dev/null +++ b/model-00020.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350ed3d8ee410f0cd9822950b8e3080565eb5cbd8131449b588fc356d44f37a4 +size 5002972344 diff --git a/model-00021.safetensors b/model-00021.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..339d224dae9407dd5ae1ee078cbc73cdfbc9ba40 --- /dev/null +++ b/model-00021.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72f51dc0ad2322b37b1b11c726219b444841a3d7464a5bf63da31e3ec99cb0e9 +size 5001555720 diff --git a/model-00022.safetensors b/model-00022.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..082e77c151fd3199bf312cef726dda2f81b12c6f --- /dev/null +++ b/model-00022.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3d4d4cfadd955335d7079060a795defd2e367831849f50ef1435897ac748866 +size 5001422892 diff --git a/model-00025.safetensors b/model-00025.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3401d2359aeb241bb94fb8df9225e1b1c9e520 --- /dev/null +++ b/model-00025.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eee33e55fc294ed188c11c53d06e4a15f3811112c53c3932df1d2b666c7a6cb +size 5000909684 diff --git a/model-00027.safetensors b/model-00027.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c2177b10821e30258a204cef046f16f2bd41c3b1 --- /dev/null +++ b/model-00027.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2379e2df9b56ca8f296204b78c653291b92867a453aa8a88e659a66a51756d2f +size 5001422804 diff --git a/model-00028.safetensors b/model-00028.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b341f2e44d7517323ec31c90836dfca7d0207804 --- /dev/null +++ b/model-00028.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62a9ea869d167e5f9343140203d50a042d25c4704e037ca41c5297e340c0a8f9 +size 5003105772 diff --git a/model-00030.safetensors b/model-00030.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..169852f769879e1535edfc4307546b686b930a8b --- /dev/null +++ b/model-00030.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:826e8eb1951b0fd4790218484fb22dd676eb8a880a3e6c53f659064c0ed26ea1 +size 5001422948 diff --git a/model-00031.safetensors b/model-00031.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5dc617ec6c47b7001f67a8333aa0c8ffc5a880ce --- /dev/null +++ b/model-00031.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95bc65aa72d7c3cdaa61d2e1453030cedcbf4a288434db02176841c16a79b76c +size 5001422788 diff --git a/model-00032.safetensors b/model-00032.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..55c9cf58f50afd8003ca904212f2d997522b9d18 --- /dev/null +++ b/model-00032.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:910f4c775e05c9d12104670b5d95b8b004212c10992ffebaa572d84b1a902e2c +size 5003105764 diff --git a/model-00033.safetensors b/model-00033.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a09ebfe8a0ffa8942394f64a06d22d2003b19aa8 --- /dev/null +++ b/model-00033.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e3a5af36aafe686ef4395e282f9a346e71ae86d77ec702907674a546126f31 +size 5001422684 diff --git a/model-00034.safetensors b/model-00034.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b343b0e65163cfcf3666e605abcbdda16f198e1 --- /dev/null +++ b/model-00034.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d076e85b7bd7f30318fd54dffa0e9611c50a45df7c048d98f7c17c6239d2814 +size 5001422052 diff --git a/model-00035.safetensors b/model-00035.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7647bb1a76688cd4fe4f4f2704136f5d9941c144 --- /dev/null +++ b/model-00035.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d598c9c665a1bb56fe7aa7777a78be550ec88123783c3771f138b2c0f975c7 +size 5001422148 diff --git a/model-00036.safetensors b/model-00036.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a9d99088125efe3c14607f99ddeab7e29151af1d --- /dev/null +++ b/model-00036.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0499790d617a3f20ca82fdea339e63bf380d68c9c6e8313594d250b6c8066f9 +size 5003105788 diff --git a/model-00037.safetensors b/model-00037.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..00eeec68460c9d6df75ff085b07b28bcd505fded --- /dev/null +++ b/model-00037.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b83490bc9bd65e903e97cd6d08bc85e5c6349ce68e65bb60eb31f6bc7cedf8c0 +size 5001422740 diff --git a/model-00038.safetensors b/model-00038.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb3bec3aba5dac957841072325907748e4e72b83 --- /dev/null +++ b/model-00038.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a51b916cc7308e9a8f3f5d84369e5a0555c994953dee587f137f2a17de5ccd +size 5001422964 diff --git a/model-00039.safetensors b/model-00039.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..973d694ab60fbdd59b7b31d3b2e50cd7c72705e9 --- /dev/null +++ b/model-00039.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e5665442cb7786318bc751a80b0f8207abeb0acec84a6c983511481b87dc04 +size 5006010644 diff --git a/model-00042.safetensors b/model-00042.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c40db31e9525ae2604718f38883563e1f2d1872d --- /dev/null +++ b/model-00042.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c74d7e0a6fb088f94c94a403402de2a953b41c230953966f7747eefb59326a4 +size 5006010764 diff --git a/model-00043.safetensors b/model-00043.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..86deab7ebe268ce0eda82e784a4426820a674e81 --- /dev/null +++ b/model-00043.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3bd55fd96ba81cd04212f471de12931167f51368b0131b7521c55d5ef8e54f3 +size 5001421292 diff --git a/model-00044.safetensors b/model-00044.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..edb637ecb75ad190e0d962e44a4aac5e64a64aca --- /dev/null +++ b/model-00044.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0833692890268a635153700d3ce669aa847c09f4ac09a26971cf5d322f9cec3b +size 5003105716 diff --git a/model-00046.safetensors b/model-00046.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bd25dae5883b2fcb0e1a2f414ad106bd0a6673f0 --- /dev/null +++ b/model-00046.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f5a3286732329777d8e6851545a64d1caf9d12bb918c87d61b638617862f5b +size 5006010812 diff --git a/model-00047.safetensors b/model-00047.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8caf8dbfc31de12bb7e37b0924ec6ec814205011 --- /dev/null +++ b/model-00047.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d4270ec45e070d7fd7151c4e78d746d2e772727749f8f8b966ca8d9f58360e +size 5001422764 diff --git a/model-00049.safetensors b/model-00049.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70539a8f6cb80f70fbc40b30dcd7dee3e8519f70 --- /dev/null +++ b/model-00049.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6164122cb7e3e90faf09e0457b2aa92717d195c1fe295cc1ef97e3aa25aa576 +size 5006010708 diff --git a/model-00050.safetensors b/model-00050.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7ae469170a5d2e7aaf7c1fb63a33222124c35c5 --- /dev/null +++ b/model-00050.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406f00ee2be46b9a50ba023e9cf35d66ce50890081f276719d0a477f825473d5 +size 5001422948 diff --git a/model-00052.safetensors b/model-00052.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a768db4e9d8cb510c0b7d5d5e2c6a3b822824a97 --- /dev/null +++ b/model-00052.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d59986622f67249c501100c5df41f36c047b042d93ba7342cd8483f1ca30146c +size 5005091788 diff --git a/model-00054.safetensors b/model-00054.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cabba3b1f12e818a7e0bc8fc98874322ad9de9b --- /dev/null +++ b/model-00054.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d491b893b39b7e133228e8fb7f6efc7d401f3d92c008545c1bf2721fca79daa3 +size 5001422940 diff --git a/model-00055.safetensors b/model-00055.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7551a4b68f39c8abeed22c62df75ec59b16115b1 --- /dev/null +++ b/model-00055.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d910b6456af7f9b1520f1d05b3af90cd7715db3f666d8ae634453f5392a78e +size 5112282228 diff --git a/model-00056.safetensors b/model-00056.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..edd37e4021c16e561592a2fb291e8f6c355897e7 --- /dev/null +++ b/model-00056.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1137a7c2f03d8692df34c197a8fdeb1a674b8d3ba999261b432c6410d897367c +size 5005498036 diff --git a/model-00057.safetensors b/model-00057.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..920fa381bb2c849d87eab31c832bb26060f1ca72 --- /dev/null +++ b/model-00057.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab5b10eccbb5de97322f8895636d0c6892d5c0ec8fa07089c612423d437bbaed +size 5001422988 diff --git a/model-00058.safetensors b/model-00058.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6bd026e07b34d4bb78d8214f47139e30fa5f7dff --- /dev/null +++ b/model-00058.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4d1b6ae4c738eb485ce2c6e3e8a8c9e764361a93d443e870bc1117e657efa1 +size 5001422780 diff --git a/model-00059.safetensors b/model-00059.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4d04ceff4c94ec013c53b7a750f39303eae5378 --- /dev/null +++ b/model-00059.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d4321b5dd0ab4124e08b71813467cfa085ecd2d1bd1e9ab73045d190feab58 +size 5004367132 diff --git a/model-00062.safetensors b/model-00062.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4bdbf5aaa2554343ace7de4ab4bdc06101b2edd8 --- /dev/null +++ b/model-00062.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1335ab687f82bc394a095c591449547daee1fddd6447dac0215b36e7ed74a03b +size 5008224104 diff --git a/model-00063.safetensors b/model-00063.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c143fa4a502d5bbbd7ecd9ca841bdbdb6d9ddf8f --- /dev/null +++ b/model-00063.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07c8d923bbba0bb61e2afb8bf23a0ad0917a01dd574e9347c98c4d8425572d8 +size 2515249728 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..a016ac4ebd692963e173c9d34d6d790ce84654e1 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe40b894556e2aa4b0580ddd0d9fc1fd638f64aa04ecc43989c9030b056e8a3 +size 12706968 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..aba40197a4cdb5607f4ab7a05fb0a4ee8054fd6d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d +size 20217442 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e375fa0a4e16660ce0e2026e39374d35dad4c079 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,33 @@ +{ + "backend": "tokenizers", + "clean_up_tokenization_spaces": false, + "do_lower_case": false, + "eos_token": "<|endoftext|>", + "extra_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>", + "<|begin_of_audio|>", + "<|end_of_audio|>", + "<|begin_of_transcription|>", + "<|end_of_transcription|>" + ], + "is_local": true, + "model_max_length": 1048576, + "model_specific_special_tokens": {}, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "remove_space": false, + "tokenizer_class": "TokenizersBackend" +}