Add files using upload-large-folder tool

Browse files

Files changed (50) hide show

THIRD_PARTY_NOTICES.md +7 -0
assembly-status.json +39 -0
chat_template.jinja +213 -0
glm53_exl3_tp4.py +250 -0
layer-06-part-tail-0.json +0 -0
layer-06-part-tail-1.json +0 -0
layer-08-part-tail-1.json +0 -0
layer-09-part-tail-1.json +0 -0
layer-11-part-tail-0.json +0 -0
layer-11-part-tail-1.json +0 -0
layer-12-part-tail-0.json +0 -0
layer-15-part-tail-1.json +0 -0
layer-16-part-tail-1.json +0 -0
layer-17-part-tail-1.json +0 -0
layer-20-part-tail-0.json +0 -0
layer-21-part-tail-0.json +0 -0
layer-23-part-tail-0.json +0 -0
layer-26-part-tail-0.json +0 -0
layer-28-part-tail-0.json +0 -0
layer-29-part-tail-0.json +0 -0
layer-33-part-tail-0.json +0 -0
layer-36-part-tail-0.json +0 -0
layer-37-part-tail-0.json +0 -0
layer-38-part-tail-0.json +0 -0
layer-42-part-tail-0.json +0 -0
layer-42-part-tail-1.json +0 -0
layer-43-part-tail-0.json +0 -0
processor_config.json +44 -0
quantization_config.json +22 -0
retained-00015-of-00120.safetensors.json +73 -0
retained-00023-of-00120.safetensors.json +73 -0
retained-00029-of-00120.safetensors.json +123 -0
retained-00043-of-00120.safetensors.json +73 -0
retained-00048-of-00120.safetensors.json +73 -0
retained-00056-of-00120.safetensors.json +49 -0
retained-00059-of-00120.safetensors.json +71 -0
retained-00062-of-00120.safetensors.json +73 -0
retained-00065-of-00120.safetensors.json +71 -0
retained-00068-of-00120.safetensors.json +73 -0
retained-00076-of-00120.safetensors.json +71 -0
retained-00081-of-00120.safetensors.json +73 -0
retained-00087-of-00120.safetensors.json +71 -0
retained-00092-of-00120.safetensors.json +73 -0
retained-00095-of-00120.safetensors.json +73 -0
retained-00104-of-00120.safetensors.json +657 -0
retained-00109-of-00120.safetensors.json +73 -0
retained-00114-of-00120.safetensors.json +47 -0
retained-00117-of-00120.safetensors.json +73 -0
retained-00120-of-00120.safetensors.json +755 -0
tokenizer_config.json +33 -0

THIRD_PARTY_NOTICES.md ADDED Viewed

	@@ -0,0 +1,7 @@

+# Third-party notices and lineage
+Z.AI created the MIT-licensed source model. TurboDerp and ExLlamaV3
+contributors created EXL3/Trellis. Brandon M. Music is credited for the
+MIT-licensed GLM-5.2 TR3 v3.1 implementation pinned at `f79c9167690ca705e877ae4dc55a841d1aae1247` and its
+documented 64-expert retained-tier structure. This is an independent GLM-5.3
+BF16 adaptation; no BrandonMusic GLM-5.3 weights were used.

assembly-status.json ADDED Viewed

	@@ -0,0 +1,39 @@

+{
+  "artifact_file_bytes": 111500623215,
+  "completed_at": "2026-09-05T21:56:00.065455+00:00",
+  "effective_routed_tier_bpw": 2.0,
+  "indexed_tensor_count": 583090,
+  "k3_source_audit_manifest_sha256": null,
+  "k4_audit_manifest_sha256": null,
+  "k4_encoded_status_sha256": null,
+  "layout": "uniform-k2-routed-experts",
+  "manifest_sha256": "501641b947fa56afc9ed098bdc034e59c2bd229d4fa1a1d7b80e3727f10c8ba3",
+  "publication_hygiene": {
+    "checked_text_bytes": 148596759,
+    "checked_text_files": 203,
+    "excluded_binary_suffixes": [
+      ".bin",
+      ".npy",
+      ".npz",
+      ".pt",
+      ".pth",
+      ".safetensors"
+    ],
+    "forbidden_private_artifact_names": [
+      "private-session-calibration-manifest.json",
+      "private-session-calibration.jsonl",
+      "token_rows.safetensors"
+    ],
+    "state": "PASS"
+  },
+  "quality_claim": false,
+  "runtime_claim": false,
+  "selection_audit_manifest_sha256": "daf1d1993d207be215b291c2a3be0be93b8faab8fb17a8ca5fd224fb68878537",
+  "selection_manifest_sha256": "58240b2d7d71feae843e8d7c6a135f02e33b130bfd062e5d5024a699196ca2e8",
+  "state": "COMPLETE",
+  "structural_evidence_manifest_sha256": "e14841c8a84fcc06493b0aeed1fdbcae286ecc3b691ca42acc984edaeaa78b88",
+  "structural_pass": true,
+  "tail_encoded_status_sha256": "73336df2eb9ac226fcd346372b9eaa64f7d8cfd6b2e1029da1d79c0e609bc90f",
+  "tail_source_audit_manifest_sha256": "65e44fc8b022b378d0df57d5eba20c91436b7d24a433a2c101607778983a5283",
+  "target_bpw": 2.0
+}

chat_template.jinja ADDED Viewed

	@@ -0,0 +1,213 @@

+[gMASK]<sop>
+{%- set effective_reasoning_effort = reasoning_effort if reasoning_effort is defined and reasoning_effort in ['low', 'high'] else 'max' -%}
+{%- if effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%}
+{%- if tools -%}
+{%- macro tool_to_json(tool) -%}
+    {%- set ns_tool = namespace(first=true) -%}
+    {{ '{' -}}
+    {%- for k, v in tool.items() -%}
+        {%- if k != 'defer_loading' and k != 'strict' -%}
+            {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}
+            {%- set ns_tool.first = false -%}
+            "{{ k }}": {{ v | tojson(ensure_ascii=False) }}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- '}' -}}
+{%- endmacro -%}
+{%- macro tool_references_to_response(refs) -%}
+    {{- '<tool_response><tools>\n' -}}
+    {%- for tr in refs -%}
+        {%- for tool in tools -%}
+            {%- if 'function' in tool -%}
+                {%- set tool = tool['function'] -%}
+            {%- endif -%}
+            {%- if tool.name == tr.name -%}
+                {{- tool_to_json(tool) + '\n' -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endfor -%}
+    {{- '</tools></tool_response>' -}}
+{%- endmacro -%}
+<|system|>
+# Tools
+You may call one or more functions to assist with the user query.
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{%- if 'function' in tool -%}
+    {%- set tool = tool['function'] -%}
+{%- endif -%}
+{% if tool.defer_loading is not defined or not tool.defer_loading %}
+{{ tool_to_json(tool) }}
+{% endif %}
+{% endfor %}
+</tools>
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- elif item is mapping and item.type in ['image', 'image_url'] -%}
+                {{- '<|begin_of_image|><|image|><|end_of_image|>' }}
+            {%- elif item is mapping and item.type in ['video', 'video_url'] -%}
+                {{- '<|begin_of_video|><|video|><|end_of_video|>' }}
+            {%- elif item is mapping and item.type in ['audio', 'audio_url', 'input_audio'] -%}
+                {{- "<reminder>You are unable to process this audio because you don't have audio input ability. Try different methods.</reminder>" }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro tool_response(text) -%}
+{{- '<tool_response>' + text + '</tool_response>' -}}
+{%- endmacro -%}
+{%- macro render_tool_response(m) -%}
+{%- if m.content is string -%}
+    {{- tool_response(m.content) -}}
+{%- elif m.content and m.content is not mapping and m.content.0.type == "tool_reference" -%}
+    {{- tool_references_to_response(m.content) -}}
+{%- elif is_list_of_outputs(m) -%}
+    {%- for tr in m.content -%}
+        {%- if tr.output is iterable and tr.output is not string and tr.output is not mapping and tr.output and tr.output.0.type == "tool_reference" -%}
+            {{- tool_references_to_response(tr.output) -}}
+        {%- else -%}
+            {{- tool_response(visible_text(tr.output)) -}}
+        {%- endif -%}
+    {%- endfor -%}
+{%- else -%}
+    {{- tool_response(visible_text(m.content)) -}}
+{%- endif -%}
+{%- endmacro -%}
+{%- macro id_of(obj) -%}
+    {%- if obj.tool_call_id -%}
+        {{- obj.tool_call_id -}}
+    {%- elif obj.id -%}
+        {{- obj.id -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro is_list_of_outputs(m) -%}
+    {%- if m.content and m.content.0.output is defined -%}1{%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1) -%}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{%- for m in messages -%}
+{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- elif '</think>' in content %}
+    {%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
+    {%- set content = content.split('</think>')[-1] %}
+{%- endif %}
+{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%}
+{{ '<think>' + reasoning_content +  '</think>'}}
+{%- else -%}
+{{ '<think></think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{- '<tool_call>' + tc.name -}}
+{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' -}}
+    {%- set block_start = loop.index0 -%}
+    {%- set ns_blk = namespace(end=block_start) -%}
+    {%- for j in range(block_start, messages|length) -%}
+        {%- if messages[j].role == 'tool' -%}
+            {%- set ns_blk.end = j -%}
+        {%- else -%}
+            {%- break -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- set ns_a = namespace(tool_calls=none) -%}
+    {%- if block_start > 0 and messages[block_start - 1].role == 'assistant' and messages[block_start - 1].tool_calls -%}
+        {%- set ns_a.tool_calls = messages[block_start - 1].tool_calls -%}
+    {%- endif -%}
+    {%- set ns_chk = namespace(can_sort=true) -%}
+    {%- if not ns_a.tool_calls -%}
+        {%- set ns_chk.can_sort = false -%}
+    {%- else -%}
+        {%- for k in range(block_start, ns_blk.end + 1) -%}
+            {%- set m = messages[k] -%}
+            {%- if is_list_of_outputs(m) -%}
+                {%- for entry in m.content -%}
+                    {%- set eid = id_of(entry) -%}
+                    {%- if not eid -%}
+                        {%- set ns_chk.can_sort = false -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- else -%}
+                {%- set tk_id = id_of(m) -%}
+                {%- if not tk_id -%}
+                    {%- set ns_chk.can_sort = false -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {%- for tc in ns_a.tool_calls -%}
+            {%- set tc_id = id_of(tc) -%}
+            {%- if not tc_id -%}
+                {%- set ns_chk.can_sort = false -%}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+    {%- if ns_chk.can_sort -%}
+        {%- for tc in ns_a.tool_calls -%}
+            {%- set tc_id = id_of(tc) -%}
+            {%- for k in range(block_start, ns_blk.end + 1) -%}
+                {%- set m = messages[k] -%}
+                {%- if is_list_of_outputs(m) -%}
+                    {%- for entry in m.content -%}
+                        {%- set eid = id_of(entry) -%}
+                        {%- if eid == tc_id -%}
+                            {%- if entry.output is iterable and entry.output is not string and entry.output is not mapping and entry.output and entry.output.0.type == "tool_reference" -%}
+                                {{- tool_references_to_response(entry.output) -}}
+                            {%- else -%}
+                                {{- tool_response(visible_text(entry.output)) -}}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                {%- else -%}
+                    {%- set tk_id = id_of(m) -%}
+                    {%- if tk_id == tc_id -%}
+                        {{- render_tool_response(m) -}}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endfor -%}
+    {%- else -%}
+        {%- for k in range(block_start, ns_blk.end + 1) -%}
+            {{- render_tool_response(messages[k]) -}}
+        {%- endfor -%}
+    {%- endif -%}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '<think>' -}}
+{%- endif -%}

glm53_exl3_tp4.py ADDED Viewed

	@@ -0,0 +1,250 @@

+"""Runtime primitive for Brandon-style GLM-5.3 K2/K3/K4 EXL3 hybrids."""
+from __future__ import annotations
+import json
+from pathlib import Path
+from typing import Any
+import torch
+import torch.distributed as dist
+from torch import nn
+import torch.nn.functional as F
+from safetensors import safe_open
+from exllamav3.modules.quant import LinearEXL3
+HIDDEN = 4096
+LOCAL_INTERMEDIATE = 512
+EXPERTS = 288
+TP = 4
+SWIGLU_LIMIT = 10.0
+def _packed_prefix(layer: int, expert: int, projection: str, rank: int) -> str:
+    return (
+        f"model.language_model.layers.{layer}.mlp.experts.{expert}."
+        f"{projection}.rank{rank}"
+    )
+def _load_linear(
+    handle: Any,
+    prefix: str,
+    in_features: int,
+    out_features: int,
+    out_dtype: torch.dtype,
+) -> LinearEXL3:
+    tensors = {
+        suffix: handle.get_tensor(f"{prefix}.{suffix}")
+        for suffix in ("suh", "svh", "trellis", "mcg")
+    }
+    return LinearEXL3(
+        config=None,
+        in_features=in_features,
+        out_features=out_features,
+        suh=tensors["suh"],
+        svh=tensors["svh"],
+        trellis=tensors["trellis"],
+        mcg=tensors["mcg"],
+        out_dtype=out_dtype,
+        key=prefix,
+    )
+class TPRankEXL3Experts(nn.Module):
+    """One TP rank of a manifest-bound 224-tail/64-K4 routed-expert union."""
+    def __init__(
+        self,
+        artifact_root: str | Path,
+        layer: int,
+        tp_rank: int,
+        device: torch.device | str,
+        process_group: Any | None = None,
+        out_dtype: torch.dtype = torch.float16,
+        swiglu_limit: float = SWIGLU_LIMIT,
+    ) -> None:
+        super().__init__()
+        if not 3 <= layer <= 44:
+            raise ValueError(f"layer outside routed scope: {layer}")
+        if not 0 <= tp_rank < TP:
+            raise ValueError(f"TP rank outside [0, {TP}): {tp_rank}")
+        self.artifact_root = Path(artifact_root)
+        self.layer = layer
+        self.tp_rank = tp_rank
+        self.device = torch.device(device)
+        self.process_group = process_group
+        self.out_dtype = out_dtype
+        self.swiglu_limit = float(swiglu_limit)
+        self.num_experts = EXPERTS
+        self.gate: list[LinearEXL3 | None] = [None] * EXPERTS
+        self.up: list[LinearEXL3 | None] = [None] * EXPERTS
+        self.down: list[LinearEXL3 | None] = [None] * EXPERTS
+        self.expert_integer_k: list[int | None] = [None] * EXPERTS
+        self._load()
+    def _expected_integer_k(self) -> dict[int, int]:
+        bitmap_path = self.artifact_root / "tier_bitmap.json"
+        bitmap = json.loads(bitmap_path.read_text())
+        if bitmap.get("state") != "PASS":
+            raise RuntimeError("hybrid tier bitmap is not PASS")
+        layer = bitmap.get("layers", {}).get(str(self.layer), {})
+        declared = layer.get("expert_k")
+        if declared is not None:
+            expected = {int(expert): int(k) for expert, k in declared.items()}
+        else:
+            expected = {}
+            for key, integer_k in (
+                ("tail_k2", 2),
+                ("tail_k3", 3),
+                ("keep_k4", 4),
+            ):
+                for expert in layer.get(key, []):
+                    expert = int(expert)
+                    if expert in expected:
+                        raise RuntimeError(
+                            f"duplicate tier-bitmap expert at layer {self.layer}: {expert}"
+                        )
+                    expected[expert] = integer_k
+        if set(expected) != set(range(EXPERTS)) or not set(expected.values()).issubset(
+            {2, 3, 4}
+        ):
+            raise RuntimeError(
+                f"invalid tier-bitmap coverage at layer {self.layer}: {len(expected)}/{EXPERTS}"
+            )
+        if sum(k == 4 for k in expected.values()) != 64:
+            raise RuntimeError(
+                f"tier bitmap does not retain exactly 64 K4 experts at layer {self.layer}"
+            )
+        return expected
+    def _load(self) -> None:
+        expected_integer_k = self._expected_integer_k()
+        observed = set()
+        sidecars = sorted(
+            (self.artifact_root / "layers").glob(f"layer-{self.layer:02d}-part-*.json")
+        )
+        if not sidecars:
+            raise RuntimeError(f"no hybrid packed sidecars for layer {self.layer}")
+        for sidecar_path in sidecars:
+            sidecar = json.loads(sidecar_path.read_text())
+            experts = [int(value) for value in sidecar.get("experts", [])]
+            duplicate = observed.intersection(experts)
+            if duplicate:
+                raise RuntimeError(
+                    f"duplicate hybrid expert assignment at layer {self.layer}: {sorted(duplicate)}"
+                )
+            observed.update(experts)
+            path = sidecar_path.with_suffix(".safetensors")
+            with safe_open(path, framework="pt", device=str(self.device)) as handle:
+                for expert in experts:
+                    gate_prefix = _packed_prefix(
+                        self.layer, expert, "gate_proj", self.tp_rank
+                    )
+                    trellis_shape = handle.get_slice(f"{gate_prefix}.trellis").get_shape()
+                    integer_k = int(trellis_shape[-1]) // 16
+                    if integer_k not in (2, 3, 4):
+                        raise RuntimeError(
+                            f"unexpected hybrid K={integer_k} at layer={self.layer} expert={expert}"
+                        )
+                    if integer_k != expected_integer_k[expert]:
+                        raise RuntimeError(
+                            f"hybrid K disagrees with tier bitmap at layer={self.layer} "
+                            f"expert={expert}: packed={integer_k} expected={expected_integer_k[expert]}"
+                        )
+                    self.expert_integer_k[expert] = integer_k
+                    self.gate[expert] = _load_linear(
+                        handle,
+                        gate_prefix,
+                        HIDDEN,
+                        LOCAL_INTERMEDIATE,
+                        self.out_dtype,
+                    )
+                    self.up[expert] = _load_linear(
+                        handle,
+                        _packed_prefix(self.layer, expert, "up_proj", self.tp_rank),
+                        HIDDEN,
+                        LOCAL_INTERMEDIATE,
+                        self.out_dtype,
+                    )
+                    self.down[expert] = _load_linear(
+                        handle,
+                        _packed_prefix(self.layer, expert, "down_proj", self.tp_rank),
+                        LOCAL_INTERMEDIATE,
+                        HIDDEN,
+                        self.out_dtype,
+                    )
+        if observed != set(range(EXPERTS)):
+            raise RuntimeError(
+                f"incomplete hybrid expert union at layer {self.layer}: {len(observed)}/{EXPERTS}"
+            )
+        if any(
+            module is None
+            for modules in (self.gate, self.up, self.down)
+            for module in modules
+        ):
+            raise RuntimeError(
+                f"incomplete hybrid EXL3 module coverage for layer {self.layer} rank {self.tp_rank}"
+            )
+        observed_counts = {
+            k: self.expert_integer_k.count(k) for k in (2, 3, 4)
+        }
+        expected_counts = {
+            k: sum(value == k for value in expected_integer_k.values()) for k in (2, 3, 4)
+        }
+        if observed_counts != expected_counts:
+            raise RuntimeError(
+                f"hybrid tier cardinality mismatch at layer {self.layer}: "
+                f"observed={observed_counts} expected={expected_counts}"
+            )
+    @torch.inference_mode()
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        top_k_index: torch.Tensor,
+        top_k_weights: torch.Tensor,
+        params: dict | None = None,
+    ) -> torch.Tensor:
+        if hidden_states.ndim != 2 or hidden_states.shape[-1] != HIDDEN:
+            raise ValueError(
+                f"expected [tokens, {HIDDEN}] hidden states, got {tuple(hidden_states.shape)}"
+            )
+        if (
+            top_k_index.shape != top_k_weights.shape
+            or top_k_index.shape[0] != hidden_states.shape[0]
+        ):
+            raise ValueError("top-k routing tensors do not match hidden states")
+        params = {} if params is None else params
+        final = torch.zeros_like(hidden_states, dtype=self.out_dtype)
+        mask = F.one_hot(top_k_index, num_classes=EXPERTS).permute(2, 1, 0)
+        hit = torch.greater(mask.sum(dim=(-1, -2)), 0).nonzero().flatten().tolist()
+        for expert in hit:
+            top_k_pos, token_idx = torch.where(mask[expert])
+            x = hidden_states[token_idx].to(self.out_dtype).contiguous()
+            gate = self.gate[expert].forward(x, params).clamp(max=self.swiglu_limit)
+            up = self.up[expert].forward(x, params).clamp(
+                min=-self.swiglu_limit, max=self.swiglu_limit
+            )
+            activated = (F.silu(gate) * up).contiguous()
+            current = self.down[expert].forward(activated, params)
+            current = current * top_k_weights[token_idx, top_k_pos, None].to(
+                current.dtype
+            )
+            final.index_add_(0, token_idx, current)
+        if self.process_group is not None:
+            if not dist.is_initialized():
+                raise RuntimeError(
+                    "a process group was provided but torch.distributed is not initialized"
+                )
+            dist.all_reduce(final, op=dist.ReduceOp.SUM, group=self.process_group)
+        return final
+    def unload(self) -> None:
+        for modules in (self.gate, self.up, self.down):
+            for module in modules:
+                if module is not None:
+                    module.unload()
+            modules.clear()

layer-06-part-tail-0.json ADDED Viewed