kquant source (vectorized packers) for reproducibility

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+src/agiws_neural_quant/kquant/__pycache__/_iq_tables.cpython-313.pyc filter=lfs diff=lfs merge=lfs -text

src/agiws_neural_quant/kquant/__init__.py

@@ -0,0 +1,330 @@
+"""kquant — GGUF k-quant block-wise quantization formats (llama.cpp).
+
+Block-wise quantization with super-block structure. Each format packs N weights
+per block with a shared scale (and optionally min/d-scale).
+
+Formats:
+  Q4_0  — block_size=32, fp16 scale, 4-bit values. ~4.5 bpw.
+  Q4_1  — block_size=32, fp16 scale + fp16 min, 4-bit values. ~5 bpw.
+  Q4_K  — super-block 256, 8 sub-blocks of 32. 6-bit scale + 6-bit d-scale + 4-bit
+           values. ~4.5 bpw. _S/_M/_L variants differ in d-scale precision.
+  Q5_K  — super-block 256, 5-bit values + 6-bit scale + 6-bit d-scale. ~5.5 bpw.
+  Q6_K  — super-block 256, 6-bit values + 8-bit d-scale + 6-bit scale. ~6.5 bpw.
+  Q8_0  — block_size=32, fp16 scale, 8-bit values. ~8.5 bpw. Near-lossless.
+  Q2_K  — super-block 256, 4-bit Q2-quants + 4-bit d-scale. ~2.6 bpw.
+  Q3_K  — super-block 256, 3-bit values + 6-bit scale. ~3.5 bpw.
+
+For NeuralQuant we implement the math (quantize/dequantize per block); packing
+into GGUF binary layout is handled by the GGUF writer (stage 22.8 converters).
+Here we store block data as tensors and dequantize on-the-fly for inference.
+"""
+
+from __future__ import annotations
+
+import torch
+
+
+# ---------------------------------------------------------------------------
+# Q8_0 — block_size=32, fp16 scale, int8 values. Near-lossless int8.
+# ---------------------------------------------------------------------------
+
+def quantize_q8_0_block(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    """Quantize a block of 32 values to Q8_0.
+
+    Returns dict with 'scale' (fp32 scalar) and 'values' (int8 [32]).
+    scale = max(abs(w)) / 127; values = round(w / scale).
+    """
+    assert w.numel() == 32, f"Q8_0 block must be 32 elements, got {w.numel()}"
+    max_abs = w.abs().amax().clamp(min=1e-8)
+    scale = max_abs / 127.0
+    values = torch.clamp(torch.round(w / scale), min=-127, max=127).to(torch.int8)
+    return {"scale": scale.to(torch.float32), "values": values}
+
+
+def dequantize_q8_0_block(scale: torch.Tensor, values: torch.Tensor) -> torch.Tensor:
+    """Reconstruct 32 values from Q8_0 block."""
+    return values.to(torch.float32) * scale.to(torch.float32)
+
+
+# ---------------------------------------------------------------------------
+# Q4_0 — block_size=32, fp16 scale, 4-bit values [-8, 7].
+# ---------------------------------------------------------------------------
+
+def quantize_q4_0_block(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    assert w.numel() == 32, f"Q4_0 block must be 32 elements, got {w.numel()}"
+    max_abs = w.abs().amax().clamp(min=1e-8)
+    scale = max_abs / 7.0  # 4-bit symmetric: [-8, 7], use 7 for scale
+    values = torch.clamp(torch.round(w / scale), min=-8, max=7).to(torch.int8)
+    return {"scale": scale.to(torch.float32), "values": values}
+
+
+def dequantize_q4_0_block(scale: torch.Tensor, values: torch.Tensor) -> torch.Tensor:
+    return values.to(torch.float32) * scale.to(torch.float32)
+
+
+# ---------------------------------------------------------------------------
+# Q4_K — super-block 256 = 8 sub-blocks of 32. 6-bit packed scale + 6-bit
+# d-scale per sub-block + 4-bit values.
+# ---------------------------------------------------------------------------
+
+def quantize_q4_k_superblock(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    """Quantize 256 values to Q4_K super-block.
+
+    Structure:
+      - 8 sub-blocks of 32 values, each 4-bit.
+      - super-block scale (fp32, derived from sub-block scales).
+      - per-sub-block d-scale (fp32, ratio sub-scale / super-scale).
+
+    For NeuralQuant inference we store the actual sub-block scales (not the
+    6-bit packed GGUF representation); the GGUF writer (stage 22.8) handles
+    bit-packing.
+    """
+    assert w.numel() == 256, f"Q4_K super-block must be 256 elements, got {w.numel()}"
+    sub = w.reshape(8, 32)
+    # Per-sub-block scale: absmax / 7 (4-bit symmetric).
+    sub_scales = sub.abs().amax(dim=1).clamp(min=1e-8) / 7.0  # [8]
+    values = torch.clamp(torch.round(sub / sub_scales.unsqueeze(1)), min=-8, max=7).to(torch.int8)
+    # Super-block scale = max(sub_scales). d-scale = sub_scale / super_scale.
+    super_scale = sub_scales.amax().clamp(min=1e-8)
+    d_scales = sub_scales / super_scale  # [8], in (0, 1]
+    return {
+        "super_scale": super_scale.to(torch.float32),
+        "d_scales": d_scales.to(torch.float32),
+        "values": values.reshape(256),  # int8 [256]
+    }
+
+
+def dequantize_q4_k_superblock(super_scale, d_scales, values) -> torch.Tensor:
+    """Reconstruct 256 values from Q4_K super-block."""
+    vals = values.to(torch.int8).reshape(8, 32)
+    sub_scales = super_scale.to(torch.float32) * d_scales.to(torch.float32)  # [8]
+    return (vals.to(torch.float32) * sub_scales.unsqueeze(1)).reshape(256)
+
+
+# ---------------------------------------------------------------------------
+# Q6_K — super-block 256, 6-bit values + 8-bit d-scale + 6-bit super-scale.
+# ---------------------------------------------------------------------------
+
+_Q6_LEVELS = 31  # 6-bit symmetric [-32, 31], use 31 for scale
+
+
+def quantize_q6_k_superblock(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    assert w.numel() == 256, f"Q6_K super-block must be 256 elements, got {w.numel()}"
+    sub = w.reshape(8, 32)
+    sub_scales = sub.abs().amax(dim=1).clamp(min=1e-8) / _Q6_LEVELS  # [8]
+    values = torch.clamp(torch.round(sub / sub_scales.unsqueeze(1)), min=-32, max=31).to(torch.int8)
+    super_scale = sub_scales.amax().clamp(min=1e-8)
+    d_scales = sub_scales / super_scale  # [8]
+    return {
+        "super_scale": super_scale.to(torch.float32),
+        "d_scales": d_scales.to(torch.float32),
+        "values": values.reshape(256),
+    }
+
+
+def dequantize_q6_k_superblock(super_scale, d_scales, values) -> torch.Tensor:
+    vals = values.to(torch.int8).reshape(8, 32)
+    sub_scales = super_scale.to(torch.float32) * d_scales.to(torch.float32)
+    return (vals.to(torch.float32) * sub_scales.unsqueeze(1)).reshape(256)
+
+
+# ---------------------------------------------------------------------------
+# Q5_K — super-block 256, 5-bit values + 6-bit d-scale.
+# ---------------------------------------------------------------------------
+
+_Q5_LEVELS = 15  # 5-bit symmetric [-16, 15], use 15
+
+
+def quantize_q5_k_superblock(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    assert w.numel() == 256
+    sub = w.reshape(8, 32)
+    sub_scales = sub.abs().amax(dim=1).clamp(min=1e-8) / _Q5_LEVELS
+    values = torch.clamp(torch.round(sub / sub_scales.unsqueeze(1)), min=-16, max=15).to(torch.int8)
+    super_scale = sub_scales.amax().clamp(min=1e-8)
+    d_scales = sub_scales / super_scale
+    return {
+        "super_scale": super_scale.to(torch.float32),
+        "d_scales": d_scales.to(torch.float32),
+        "values": values.reshape(256),
+    }
+
+
+def dequantize_q5_k_superblock(super_scale, d_scales, values) -> torch.Tensor:
+    vals = values.to(torch.int8).reshape(8, 32)
+    sub_scales = super_scale.to(torch.float32) * d_scales.to(torch.float32)
+    return (vals.to(torch.float32) * sub_scales.unsqueeze(1)).reshape(256)
+
+
+# ---------------------------------------------------------------------------
+# Q2_K — super-block 256, 2-bit values + 4-bit d-scale. Extreme compression.
+# ---------------------------------------------------------------------------
+
+_Q2_LEVELS = 1  # 2-bit symmetric [-2, 1], use 1 for scale (coarse)
+
+
+def quantize_q2_k_superblock(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    assert w.numel() == 256
+    sub = w.reshape(8, 32)
+    sub_scales = sub.abs().amax(dim=1).clamp(min=1e-8) / _Q2_LEVELS
+    values = torch.clamp(torch.round(sub / sub_scales.unsqueeze(1)), min=-2, max=1).to(torch.int8)
+    super_scale = sub_scales.amax().clamp(min=1e-8)
+    d_scales = sub_scales / super_scale
+    return {
+        "super_scale": super_scale.to(torch.float32),
+        "d_scales": d_scales.to(torch.float32),
+        "values": values.reshape(256),
+    }
+
+
+def dequantize_q2_k_superblock(super_scale, d_scales, values) -> torch.Tensor:
+    vals = values.to(torch.int8).reshape(8, 32)
+    sub_scales = super_scale.to(torch.float32) * d_scales.to(torch.float32)
+    return (vals.to(torch.float32) * sub_scales.unsqueeze(1)).reshape(256)
+
+
+# ---------------------------------------------------------------------------
+# Q3_K — super-block 256, 3-bit values + 6-bit d-scale.
+# ---------------------------------------------------------------------------
+
+_Q3_LEVELS = 3  # 3-bit symmetric [-4, 3], use 3
+
+
+def quantize_q3_k_superblock(w: torch.Tensor) -> dict[str, torch.Tensor]:
+    assert w.numel() == 256
+    sub = w.reshape(8, 32)
+    sub_scales = sub.abs().amax(dim=1).clamp(min=1e-8) / _Q3_LEVELS
+    values = torch.clamp(torch.round(sub / sub_scales.unsqueeze(1)), min=-4, max=3).to(torch.int8)
+    super_scale = sub_scales.amax().clamp(min=1e-8)
+    d_scales = sub_scales / super_scale
+    return {
+        "super_scale": super_scale.to(torch.float32),
+        "d_scales": d_scales.to(torch.float32),
+        "values": values.reshape(256),
+    }
+
+
+def dequantize_q3_k_superblock(super_scale, d_scales, values) -> torch.Tensor:
+    vals = values.to(torch.int8).reshape(8, 32)
+    sub_scales = super_scale.to(torch.float32) * d_scales.to(torch.float32)
+    return (vals.to(torch.float32) * sub_scales.unsqueeze(1)).reshape(256)
+
+
+# ---------------------------------------------------------------------------
+# Format registry: format string -> (block quantize, block dequantize, block_size)
+# ---------------------------------------------------------------------------
+
+BLOCK_SIZE_32 = 32
+BLOCK_SIZE_256 = 256
+
+FORMATS = {
+    "q8_0": (quantize_q8_0_block, dequantize_q8_0_block, BLOCK_SIZE_32),
+    "q4_0": (quantize_q4_0_block, dequantize_q4_0_block, BLOCK_SIZE_32),
+    "q4_k": (quantize_q4_k_superblock, dequantize_q4_k_superblock, BLOCK_SIZE_256),
+    "q5_k": (quantize_q5_k_superblock, dequantize_q5_k_superblock, BLOCK_SIZE_256),
+    "q6_k": (quantize_q6_k_superblock, dequantize_q6_k_superblock, BLOCK_SIZE_256),
+    "q2_k": (quantize_q2_k_superblock, dequantize_q2_k_superblock, BLOCK_SIZE_256),
+    "q3_k": (quantize_q3_k_superblock, dequantize_q3_k_superblock, BLOCK_SIZE_256),
+}
+
+
+def quantize_blocks(w: torch.Tensor, fmt: str) -> dict[str, torch.Tensor]:
+    """Quantize a flat weight tensor into blocks of the given format.
+
+    Pads the last dim to be divisible by block_size.
+
+    Returns dict with stacked block tensors.
+    """
+    quant_fn, _, block_size = FORMATS[fmt]
+    out_features = w.shape[0]
+    in_features = w.shape[1] if w.dim() > 1 else w.numel()
+    if w.dim() > 1:
+        flat = w
+    else:
+        flat = w.reshape(1, -1)
+        out_features = 1
+    # Pad in_features to be divisible by block_size.
+    pad = (block_size - (flat.shape[1] % block_size)) % block_size
+    if pad > 0:
+        flat = torch.nn.functional.pad(flat, (0, pad))
+    in_padded = flat.shape[1]
+    num_blocks = in_padded // block_size
+
+    # Reshape to [out, num_blocks, block_size].
+    blocks = flat.reshape(out_features, num_blocks, block_size)
+
+    if fmt in ("q8_0", "q4_0"):
+        # Simple block: per-block absmax scale + int values.
+        n_levels = 127 if fmt == "q8_0" else 7
+        max_val = n_levels if fmt == "q8_0" else 7
+        min_val = -n_levels if fmt == "q8_0" else -8
+        scales = blocks.abs().amax(dim=2).clamp(min=1e-8) / n_levels  # [out, num_blocks]
+        values = torch.clamp(
+            torch.round(blocks / scales.unsqueeze(2)), min=min_val, max=max_val
+        ).to(torch.int8)
+        return {
+            "scales": scales.to(torch.float32),
+            "values": values,
+            "in_features": in_features,
+            "in_padded": in_padded,
+            "out_features": out_features,
+            "block_size": block_size,
+        }
+    else:
+        # K-formats: super-block 256 = 8 sub-blocks of 32.
+        # blocks already [out, num_blocks, 256]; reshape to [out, num_blocks, 8, 32].
+        n_levels_map = {"q4_k": 7, "q5_k": 15, "q6_k": 31, "q2_k": 1, "q3_k": 3}
+        min_val_map = {"q4_k": -8, "q5_k": -16, "q6_k": -32, "q2_k": -2, "q3_k": -4}
+        n_levels = n_levels_map[fmt]
+        min_val = min_val_map[fmt]
+        sub = blocks.reshape(out_features, num_blocks, 8, 32)
+        sub_scales = sub.abs().amax(dim=3).clamp(min=1e-8) / n_levels  # [out, num_blocks, 8]
+        values = torch.clamp(
+            torch.round(sub / sub_scales.unsqueeze(3)), min=min_val, max=n_levels
+        ).to(torch.int8)
+        super_scales = sub_scales.amax(dim=2).clamp(min=1e-8)  # [out, num_blocks]
+        d_scales = sub_scales / super_scales.unsqueeze(2)  # [out, num_blocks, 8]
+        return {
+            "super_scales": super_scales.to(torch.float32),
+            "d_scales": d_scales.to(torch.float32),
+            "values": values.reshape(out_features, num_blocks, 256),
+            "in_features": in_features,
+            "in_padded": in_padded,
+            "out_features": out_features,
+            "block_size": 256,
+            "num_super": num_blocks,
+        }
+
+
+def dequantize_blocks(qd: dict[str, torch.Tensor], fmt: str) -> torch.Tensor:
+    """Reconstruct the weight tensor from block-quantized data.
+
+    Supports both symmetric (our quantize_blocks output) and asymmetric
+    (GGUF Q4_K/Q5_K with min offset) formats. Detection is by key presence:
+      - If 'mins' and 'super_min_scales' are in qd -> asymmetric:
+          x = super_scale * scale[i] * q[i] + super_min_scale * min[i]
+      - Otherwise -> symmetric:
+          x = super_scale * d_scale[i] * q[i]
+    """
+    in_features = qd["in_features"]
+    in_padded = qd["in_padded"]
+    out_features = qd["out_features"]
+    if fmt in ("q8_0", "q4_0"):
+        scales = qd["scales"]  # [out, num_blocks]
+        values = qd["values"]  # [out, num_blocks, block_size]
+        w = values.to(torch.float32) * scales.to(torch.float32).unsqueeze(2)
+        w = w.reshape(out_features, in_padded)[:, :in_features]
+    else:
+        super_scales = qd["super_scales"]  # [out, num_blocks]
+        d_scales = qd["d_scales"]  # [out, num_blocks, 8]
+        values = qd["values"]  # [out, num_blocks, 256]
+        sub_scales = super_scales.to(torch.float32).unsqueeze(2) * d_scales.to(torch.float32)
+        sub = values.reshape(out_features, -1, 8, 32)
+        w = sub.to(torch.float32) * sub_scales.unsqueeze(3)
+        # Asymmetric: add min offset (GGUF Q4_K/Q5_K).
+        if "mins" in qd and "super_min_scales" in qd:
+            mins = qd["mins"]  # [out, num_blocks, 8]
+            super_min_scales = qd["super_min_scales"]  # [out, num_blocks]
+            sub_mins = super_min_scales.to(torch.float32).unsqueeze(2) * mins.to(torch.float32)
+            w = w + sub_mins.unsqueeze(3)
+        w = w.reshape(out_features, -1)[:, :in_features]
+    return w

src/agiws_neural_quant/kquant/__pycache__/_iq_tables.cpython-313.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e64273922ab23df733b43120821c6c80c72efecb8fb95eafd8f4a4193928b96
+size 183305

src/agiws_neural_quant/kquant/_gguf_bits.py

@@ -0,0 +1,29 @@
+"""Shared bit-twiddling helpers for GGUF k-quant packed dequantization."""
+
+from __future__ import annotations
+
+import torch
+
+
+def f16_view_to_f32(u8: torch.Tensor) -> torch.Tensor:
+    """View uint8 bytes as float16 then cast to float32. u8: [..., 2] uint8."""
+    flat = u8.reshape(-1, 2).to(torch.int16)
+    val = flat[:, 0] | (flat[:, 1].to(torch.int16) << 8)
+    f16 = val.view(torch.float16)
+    return f16.to(torch.float32).reshape(u8.shape[:-1])
+
+
+def unpack_scales_k4(scales: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    """Unpack 8 (scale, min) 6-bit pairs from 12 bytes. scales: [..., 12] uint8.
+
+    Returns (sc, m) each [..., 8] (6-bit values as uint8).
+    """
+    q = scales.to(torch.int32)
+    sh = q.shape[:-1]
+    sc = torch.zeros(*sh, 8, dtype=torch.int32, device=q.device)
+    m = torch.zeros(*sh, 8, dtype=torch.int32, device=q.device)
+    sc[..., 0:4] = q[..., 0:4] & 0x3F
+    m[..., 0:4] = q[..., 4:8] & 0x3F
+    sc[..., 4:8] = (q[..., 8:12] & 0x0F) | ((q[..., 0:4] >> 6) << 4)
+    m[..., 4:8] = (q[..., 8:12] >> 4) | ((q[..., 4:8] >> 6) << 4)
+    return sc.to(torch.uint8), m.to(torch.uint8)

src/agiws_neural_quant/kquant/gguf_pack.py

@@ -0,0 +1,455 @@
+"""GGUF k-quant packers - torch vectorized, float32 -> packed GGUF bytes.
+
+Reverse of dequant_gguf_slice (gguf_packed.py). Each packer is a direct
+translation of llama.cpp ggml-quants.c quantize_row_*_ref:
+
+  Q8_0 (34 B):  d(f16) + qs[32] (int8)              y = d*q8              block 32
+  Q2_K (84 B):  scales[16] + qs[64] + d(f16) + dmin(f16)
+                y = d*sc*q - dmin*m  (q 0..3)       16 sub-blocks of 16
+  Q3_K (110 B): hmask[32] + qs[64] + scales[12] + d(f16)
+                y = d*sc*(q - (hm ? 0 : 4))         16 sub-blocks of 16
+  Q4_K (144 B): d(f16) + dmin(f16) + scales[12] + qs[128]
+                y = d*sc*q - dmin*m  (q 0..15)      8 sub-blocks of 32
+  Q5_K (176 B): d(f16) + dmin(f16) + scales[12] + qh[32] + qs[128]
+                y = d*sc*(q+16*bit5) - dmin*m       8 sub-blocks of 32
+  Q6_K (210 B): ql[128] + qh[64] + scales[16] + d(f16)
+                y = d*sc*(q6-32)                    16 sub-blocks of 16
+
+i-quants are NOT packed here - kquant/iq.py quantize_iq*_s() already returns
+packed GGUF bytes.
+"""
+
+from __future__ import annotations
+
+import torch
+
+QK_K = 256
+EPS = 1e-15
+
+
+def _nearest_int(x: torch.Tensor) -> torch.Tensor:
+    """Round half away from zero (C nearest_int / llm_rintf)."""
+    return torch.where(x >= 0, torch.floor(x + 0.5), torch.ceil(x - 0.5))
+
+
+def _f16_bytes(t: torch.Tensor) -> torch.Tensor:
+    """fp32 [..., n] -> uint8 [..., n, 2] little-endian f16 bytes."""
+    u = (t.half().view(torch.int16).to(torch.int64)) & 0xFFFF
+    return torch.stack([u & 0xFF, (u >> 8) & 0xFF], dim=-1).to(torch.uint8)
+
+
+def _cat_blocks(segments: list[torch.Tensor], out_f: int, nblk: int,
+                bpb: int) -> torch.Tensor:
+    """Concatenate per-block byte segments into one [out_f, nblk*bpb] row buffer.
+
+    Each segment is [out_f, nblk, seg_len] uint8 in the GGUF block layout;
+    the result is row-major [out_f, nblk*bpb] (no Python row loop).
+    """
+    return torch.cat(segments, dim=-1).reshape(out_f, nblk * bpb)
+
+
+def _make_qkx2(nmax: int, x: torch.Tensor, weights: torch.Tensor, rmin: float,
+               rdelta: float, nstep: int, use_mad: bool
+               ) -> tuple[torch.Tensor, torch.Tensor]:
+    """Vectorized make_qkx2_quants (ggml-quants.c, line 799).
+
+    Args:
+        nmax: max level (15 Q4_K, 31 Q5_K, 3 Q2_K).
+        x: [..., n] values.
+        weights: [..., n] per-value weights.
+    Returns:
+        (scale, the_min) each [..., 1].
+    """
+    n = x.shape[-1]
+    min_v = x.min(dim=-1, keepdim=True).values
+    max_v = x.max(dim=-1, keepdim=True).values
+    min_v = torch.where(min_v > 0, torch.zeros_like(min_v), min_v)  # if min > 0: min = 0
+
+    sum_w = weights.sum(dim=-1, keepdim=True)
+    sum_x = (weights * x).sum(dim=-1, keepdim=True)
+
+    # Initial quantization with iscale = nmax/(max-min).
+    iscale = nmax / (max_v - min_v).clamp_min(EPS)
+    scale = 1.0 / iscale
+    l0 = _nearest_int(iscale * (x - min_v)).clamp(0, nmax)
+    diff = scale * l0 + min_v - x
+    best_error = (weights * (diff.abs() if use_mad else diff * diff)).sum(dim=-1, keepdim=True)
+
+    # Candidate sweep: is in 0..nstep, iscale = (rmin + rdelta*is + nmax)/(max-min).
+    is_grid = rmin + rdelta * torch.arange(nstep + 1, device=x.device, dtype=x.dtype) + nmax
+    isc_all = is_grid.view(-1, *([1] * x.dim())) / (max_v - min_v).clamp_min(EPS).unsqueeze(0)
+    l_all = _nearest_int(isc_all * (x - min_v).unsqueeze(0)).clamp(0, nmax)  # [S, ..., n]
+    wl = weights.unsqueeze(0) * l_all
+    sum_l = wl.sum(dim=-1, keepdim=True)
+    sum_l2 = (wl * l_all).sum(dim=-1, keepdim=True)
+    sum_xl = (weights.unsqueeze(0) * l_all * x.unsqueeze(0)).sum(dim=-1, keepdim=True)
+
+    D = sum_w.unsqueeze(0) * sum_l2 - sum_l * sum_l
+    this_scale = (sum_w.unsqueeze(0) * sum_xl - sum_x.unsqueeze(0) * sum_l) / D.clamp_min(EPS)
+    this_min = (sum_l2 * sum_x.unsqueeze(0) - sum_l * sum_xl) / D.clamp_min(EPS)
+    # if this_min > 0: this_min = 0; this_scale = sum_xl / sum_l2
+    pos_min = this_min > 0
+    this_scale = torch.where(pos_min, sum_xl / sum_l2.clamp_min(EPS), this_scale)
+    this_min = torch.where(pos_min, torch.zeros_like(this_min), this_min)
+
+    diff2 = this_scale * l_all + this_min - x.unsqueeze(0)
+    cur_error = (weights.unsqueeze(0) * (diff2.abs() if use_mad else diff2 * diff2)).sum(dim=-1, keepdim=True)
+
+    better = cur_error < best_error                       # [S, ..., 1]
+    any_better = better.any(dim=0)
+    best_idx = better.int().argmax(dim=0)                 # [..., 1]
+    scale = torch.where(any_better, this_scale.gather(0, best_idx.unsqueeze(0)).squeeze(0), scale)
+    min_f = torch.where(any_better, this_min.gather(0, best_idx.unsqueeze(0)).squeeze(0), min_v)
+    return scale, -min_f
+
+
+def _pack_q2_q3(l: torch.Tensor) -> torch.Tensor:
+    """Pack 256 x 2-bit levels into 64 bytes (llama.cpp q2_K/q3_K qs layout).
+
+    qs[j/4 + l] = L[j+l] | L[j+l+32]<<2 | L[j+l+64]<<4 | L[j+l+96]<<6, j = 0, 128.
+    l: [..., 256] uint8 in [0, 3].
+    """
+    v = l.to(torch.int64)
+    lo = v[..., 0:32]; q1 = v[..., 32:64]; q2 = v[..., 64:96]; q3 = v[..., 96:128]
+    lo2 = v[..., 128:160]; q12 = v[..., 160:192]; q22 = v[..., 192:224]; q32 = v[..., 224:256]
+    first = lo | (q1 << 2) | (q2 << 4) | (q3 << 6)      # [..., 32]
+    second = lo2 | (q12 << 2) | (q22 << 4) | (q32 << 6)
+    return torch.cat([first, second], dim=-1).to(torch.uint8)  # [..., 64]
+
+
+def _pack_scales_min_k4(sc: torch.Tensor, m: torch.Tensor) -> torch.Tensor:
+    """Pack 8 (scale, min) 6-bit pairs into 12 bytes (get_scale_min_k4 layout).
+
+    j<4: q[j]=sc[j], q[j+4]=m[j]; j>=4: q[j+4]=(sc&0xF)|((m&0xF)<<4),
+         q[j-4] |= (sc>>4)<<6, q[j] |= (m>>4)<<6.
+    """
+    sc = sc.clamp(0, 63).to(torch.int64)
+    m = m.clamp(0, 63).to(torch.int64)
+    q = torch.zeros(*sc.shape[:-1], 12, dtype=torch.int64, device=sc.device)
+    q[..., 0:4] = sc[..., 0:4]
+    q[..., 4:8] = m[..., 0:4]
+    q[..., 8:12] = (sc[..., 4:8] & 0xF) | ((m[..., 4:8] & 0xF) << 4)
+    q[..., 0:4] |= (sc[..., 4:8] >> 4) << 6
+    q[..., 4:8] |= (m[..., 4:8] >> 4) << 6
+    return q.to(torch.uint8)
+
+
+def _unpack_scales_min_k4(q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    """Mirror of _gguf_bits.unpack_scales_k4 for packed [..., 12] int64 bytes."""
+    q = q.to(torch.int64)
+    sc = torch.zeros(*q.shape[:-1], 8, dtype=torch.int64, device=q.device)
+    m = torch.zeros_like(sc)
+    sc[..., 0:4] = q[..., 0:4] & 0x3F
+    m[..., 0:4] = q[..., 4:8] & 0x3F
+    sc[..., 4:8] = (q[..., 8:12] & 0x0F) | ((q[..., 0:4] >> 6) << 4)
+    m[..., 4:8] = (q[..., 8:12] >> 4) | ((q[..., 4:8] >> 6) << 4)
+    return sc, m
+
+
+def _pad(w: torch.Tensor, block: int) -> tuple[torch.Tensor, int]:
+    out_f, in_f = w.shape
+    nblk = (in_f + block - 1) // block
+    wp = torch.zeros(out_f, nblk * block, dtype=torch.float32, device=w.device)
+    wp[:, :in_f] = w
+    return wp, nblk
+
+
+# ---------------------------------------------------------------------------
+# Q8_0 (llama.cpp:276) - block 32.
+# ---------------------------------------------------------------------------
+
+def pack_q8_0(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, 32)
+    x = wp.view(out_f, nblk, 32)
+    amax = x.abs().amax(dim=-1).clamp_min(EPS)
+    d = amax / 127.0
+    qs = _nearest_int(x / d.unsqueeze(-1)).clamp(-127, 127).to(torch.int8).view(torch.uint8)
+    return _cat_blocks([_f16_bytes(d), qs], out_f, nblk, 34)
+
+
+# ---------------------------------------------------------------------------
+# Q2_K (llama.cpp:891) - 16 sub-blocks of 16.
+# ---------------------------------------------------------------------------
+
+def pack_q2_k(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, QK_K)
+    x = wp.view(out_f, nblk, 16, 16)
+    scale, m = _make_qkx2(3, x, x.abs(), -0.5, 0.1, 15, True)   # [.., 16, 1]
+    scale = scale.squeeze(-1)
+    m = m.squeeze(-1)
+    max_scale = scale.amax(dim=-1).clamp_min(EPS)
+    max_min = m.amax(dim=-1).clamp_min(EPS)
+
+    ls = _nearest_int((15.0 / max_scale).unsqueeze(-1) * scale).clamp(0, 15).to(torch.int64)
+    lm = _nearest_int((15.0 / max_min).unsqueeze(-1) * m).clamp(0, 15).to(torch.int64)
+    scales = (ls & 0xF) | ((lm & 0xF) << 4)                    # [out, nb, 16]
+    d = max_scale / 15.0
+    dmin = max_min / 15.0
+
+    # Re-quantize with stored d/dmin/scales.
+    d_f = d.unsqueeze(-1).unsqueeze(-1)
+    dm_f = dmin.unsqueeze(-1).unsqueeze(-1)
+    sc16 = (scales & 0xF).to(torch.float32).unsqueeze(-1)
+    m16 = (scales >> 4).to(torch.float32).unsqueeze(-1)
+    l = _nearest_int((x + dm_f * m16) / (d_f * sc16).clamp_min(EPS)).clamp(0, 3).to(torch.int64)
+    qs = _pack_q2_q3(l.view(out_f, nblk, QK_K))
+
+    return _cat_blocks([scales.to(torch.uint8), qs,
+                        _f16_bytes(d), _f16_bytes(dmin)],
+                       out_f, nblk, 84)
+
+
+# ---------------------------------------------------------------------------
+# Q3_K (llama.cpp:1229) - 16 sub-blocks of 16.
+# ---------------------------------------------------------------------------
+
+def _make_q3_quants(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    """Vectorized make_q3_quants(16, 4, x, L, do_rmse=true).
+
+    x: [..., 16]. Returns (scale, L) with L in [-4, 3] (int), scale [..., 1].
+    """
+    amax = x.abs().amax(dim=-1)
+    max_v = x.gather(-1, x.abs().argmax(dim=-1, keepdim=True)).squeeze(-1)  # signed max|x| element
+    max_s = torch.where(amax < EPS, torch.zeros_like(max_v), max_v)
+    iscale = -4.0 / max_s.clamp_min(EPS)
+    l = _nearest_int(iscale.unsqueeze(-1) * x).clamp(-4, 3).to(torch.float32)
+    w_ = x * x
+    sumlx = (w_ * x * l).sum(dim=-1, keepdim=True)
+    suml2 = (w_ * l * l).sum(dim=-1, keepdim=True)
+    for _ in range(5):
+        slx = sumlx - w_ * x * l                     # [..., 16] per-element
+        sl2 = suml2 - w_ * l * l                     # [..., 16] per-element
+        new_l = _nearest_int(x * sl2 / torch.where(slx > 0, slx, torch.ones_like(slx))).clamp(-4, 3)
+        slx2 = slx + w_ * x * new_l                  # [..., 16]
+        sl2_new = sl2 + w_ * new_l * new_l           # [..., 16]
+        better = (slx > 0) & (sl2_new > 0) & (slx2 * slx2 * suml2 > sumlx * sumlx * sl2_new) & (new_l != l)
+        l = torch.where(better, new_l, l)
+        # Global sums update incrementally over accepted elements.
+        sumlx = sumlx + (better * (slx2 - slx)).sum(dim=-1, keepdim=True)
+        suml2 = suml2 + (better * (sl2_new - sl2)).sum(dim=-1, keepdim=True)
+    scale = torch.where(suml2 > 0, sumlx / suml2.clamp_min(EPS), torch.zeros_like(sumlx))
+    return scale, l
+
+
+def pack_q3_k(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, QK_K)
+    x = wp.view(out_f, nblk, 16, 16)
+    scale, _l = _make_q3_quants(x)                              # [.., 16, 1] per sub-block
+
+    sc4 = scale.squeeze(-1)                                     # [out, nb, 16] signed
+    max_sc = sc4.gather(-1, sc4.abs().argmax(dim=-1, keepdim=True)).squeeze(-1)  # [out, nb]
+    max_sc = torch.where(sc4.abs().amax(dim=-1) < EPS, torch.zeros_like(max_sc), max_sc)
+
+    isc = -32.0 / torch.where(max_sc.abs() < EPS, torch.ones_like(max_sc), max_sc)
+    l8 = (_nearest_int(isc.unsqueeze(-1) * sc4).clamp(-32, 31) + 32).to(torch.int64)  # [.., 16] 0..63
+    l4 = l8 & 0xF
+    hi = (l8 >> 4) & 0x3
+    scales = torch.zeros(*l8.shape[:-1], 12, dtype=torch.int64, device=w.device)
+    scales[..., 0:8] = l4[..., 0:8]
+    scales[..., 0:8] |= l4[..., 8:16] << 4
+    hib = hi.view(*hi.shape[:-1], 4, 4).transpose(-1, -2)  # [.., j%4, j//4]
+    hi_sh = (hib << (2 * torch.arange(4, device=w.device)).view(1, 1, 1, 4)).sum(-1)
+    scales[..., 8:12] = hi_sh
+    d = -max_sc / 32.0
+
+    # Re-quantize levels with the actual stored scales (aux unpacking, same as
+    # dequantize_row_q3_K / dequant_q3_k_packed_rows).
+    s0 = (scales[..., 0] | (scales[..., 1] << 8) | (scales[..., 2] << 16) | (scales[..., 3] << 24)).to(torch.int64)
+    s1 = (scales[..., 4] | (scales[..., 5] << 8) | (scales[..., 6] << 16) | (scales[..., 7] << 24)).to(torch.int64)
+    s2 = (scales[..., 8] | (scales[..., 9] << 8) | (scales[..., 10] << 16) | (scales[..., 11] << 24)).to(torch.int64)
+    kmask1 = 0x03030303
+    kmask2 = 0x0F0F0F0F
+    tmp = s2
+    aux0 = (s0 & kmask2) | (((tmp >> 0) & kmask1) << 4)
+    aux1 = (s1 & kmask2) | (((tmp >> 2) & kmask1) << 4)
+    aux2 = ((s0 >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4)
+    aux3 = ((s1 >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4)
+    aux = torch.stack([aux0, aux1, aux2, aux3], dim=-1)        # [..., 4]
+    byte = (aux.unsqueeze(-1) >> (8 * torch.arange(4, device=w.device))).reshape(
+        *scales.shape[:-1], 16) & 0xFF
+    sc8 = torch.where(byte >= 128, byte - 256, byte).to(torch.float32)
+    sc = (sc8 - 32)
+    dl = d.unsqueeze(-1).unsqueeze(-1) * sc.unsqueeze(-1)
+    dl_safe = torch.where(dl == 0, torch.ones_like(dl), dl)   # C: if (!d) continue
+    Lq = _nearest_int(x / dl_safe).clamp(-4, 3).to(torch.int64)  # [-4, 3]
+
+    # hmask: bit (j%32) set when level in 4..7.
+    lq_shift = Lq + 4
+    is_hi = (lq_shift > 3).to(torch.int64)
+    lq_final = (lq_shift - 4 * is_hi).reshape(out_f, nblk, QK_K)
+    is_hi_flat = is_hi.reshape(out_f, nblk, QK_K)
+    idx = torch.arange(QK_K, device=w.device)
+    hmask = torch.zeros(out_f, nblk, 32, dtype=torch.int64, device=w.device)
+    hmask.scatter_add_(-1, (idx % 32).view(1, 1, -1).expand(out_f, nblk, QK_K),
+                       is_hi_flat << (idx // 32).view(1, 1, -1))
+    qs = _pack_q2_q3(lq_final)
+
+    return _cat_blocks([hmask.to(torch.uint8), qs, scales.to(torch.uint8),
+                        _f16_bytes(d)], out_f, nblk, 110)
+
+
+# ---------------------------------------------------------------------------
+# Q4_K (llama.cpp:1457) - 8 sub-blocks of 32.
+# ---------------------------------------------------------------------------
+
+def pack_q4_k(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, QK_K)
+    x = wp.view(out_f, nblk, 8, 32)
+    av_x = (x * x).mean(dim=-1, keepdim=True).sqrt()            # [.., 8, 1]
+    weights = av_x + x.abs()
+    scale, m = _make_qkx2(15, x, weights, -1.0, 0.1, 20, False)
+    scale = scale.squeeze(-1)
+    m = m.squeeze(-1)
+    max_scale = scale.amax(dim=-1).clamp_min(EPS)
+    max_min = m.amax(dim=-1).clamp_min(EPS)
+    ls = _nearest_int((63.0 / max_scale).unsqueeze(-1) * scale).clamp(0, 63)
+    lm = _nearest_int((63.0 / max_min).unsqueeze(-1) * m).clamp(0, 63)
+    q = _pack_scales_min_k4(ls, lm)                             # [out, nb, 12]
+    d = max_scale / 63.0
+    dmin = max_min / 63.0
+
+    d_f = d.unsqueeze(-1).unsqueeze(-1)
+    dm_f = dmin.unsqueeze(-1).unsqueeze(-1)
+    sc, mm = _unpack_scales_min_k4(q)
+    dl = d_f * sc.to(torch.float32).unsqueeze(-1)
+    ml = dm_f * mm.to(torch.float32).unsqueeze(-1)
+    l = _nearest_int((x + ml) / dl.clamp_min(EPS)).clamp(0, 15).to(torch.int64)
+    l2 = l.reshape(out_f, nblk, 4, 2, 32)
+    qs = (l2[..., 0, :] | (l2[..., 1, :] << 4)).reshape(out_f, nblk, 128)
+
+    return _cat_blocks([_f16_bytes(d), _f16_bytes(dmin), q, qs],
+                       out_f, nblk, 144)
+
+
+# ---------------------------------------------------------------------------
+# Q5_K (llama.cpp:1644) - 8 sub-blocks of 32.
+# ---------------------------------------------------------------------------
+
+def pack_q5_k(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, QK_K)
+    x = wp.view(out_f, nblk, 8, 32)
+    av_x = (x * x).mean(dim=-1, keepdim=True).sqrt()
+    weights = av_x + x.abs()
+    scale, m = _make_qkx2(31, x, weights, -0.5, 0.1, 15, False)
+    scale = scale.squeeze(-1)
+    m = m.squeeze(-1)
+    max_scale = scale.amax(dim=-1).clamp_min(EPS)
+    max_min = m.amax(dim=-1).clamp_min(EPS)
+    ls = _nearest_int((63.0 / max_scale).unsqueeze(-1) * scale).clamp(0, 63)
+    lm = _nearest_int((63.0 / max_min).unsqueeze(-1) * m).clamp(0, 63)
+    q = _pack_scales_min_k4(ls, lm)
+    d = max_scale / 63.0
+    dmin = max_min / 63.0
+
+    d_f = d.unsqueeze(-1).unsqueeze(-1)
+    dm_f = dmin.unsqueeze(-1).unsqueeze(-1)
+    sc, mm = _unpack_scales_min_k4(q)
+    dl = d_f * sc.to(torch.float32).unsqueeze(-1)
+    ml = dm_f * mm.to(torch.float32).unsqueeze(-1)
+    l = _nearest_int((x + ml) / dl.clamp_min(EPS)).clamp(0, 31).to(torch.int64)
+
+    # ql + qh: for g in 0..3 (groups of 64): pair of 32-elem sub-blocks.
+    # Each sub-block s (0..7) contributes bit s when its level > 15.
+    hi_s = (l > 15).to(torch.int64)                     # [out, nblk, 8, 32]
+    qh = (hi_s * (1 << torch.arange(8, device=w.device)).view(1, 1, 8, 1)).sum(-2)  # [out, nblk, 32]
+    lo1 = torch.where(hi_s.bool(), l - 16, l)           # [out, nblk, 8, 32]
+    ql = (lo1[..., 0::2] | (lo1[..., 1::2] << 4)).reshape(out_f, nblk, 128)
+
+    return _cat_blocks([_f16_bytes(d), _f16_bytes(dmin), q,
+                        qh.to(torch.uint8), ql], out_f, nblk, 176)
+
+
+# ---------------------------------------------------------------------------
+# Q6_K (llama.cpp:1869) - 16 sub-blocks of 16.
+# ---------------------------------------------------------------------------
+
+def _make_qx_quants_rmse(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    """Vectorized make_qx_quants(16, 32, x, L, rmse_type=1, qw=NULL) (ggml-quants.c:628).
+
+    RMSE-weighted scale: scale = sumlx/suml2 with w = x^2.
+    x: [..., 16]. Returns (scale [..., 1], L in [-32, 31] as float).
+    """
+    amax = x.abs().amax(dim=-1)
+    max_v = x.gather(-1, x.abs().argmax(dim=-1, keepdim=True)).squeeze(-1)
+    max_s = torch.where(amax < EPS, torch.zeros_like(max_v), max_v)
+    iscale = -32.0 / max_s.clamp_min(EPS)
+    l = _nearest_int(iscale.unsqueeze(-1) * x).clamp(-32, 31).to(torch.float32)
+    w_ = x * x
+    sumlx = (w_ * x * l).sum(dim=-1, keepdim=True)
+    suml2 = (w_ * l * l).sum(dim=-1, keepdim=True)
+    scale = torch.where(suml2 > 0, sumlx / suml2.clamp_min(EPS), torch.zeros_like(sumlx))
+    return scale, l
+
+
+def pack_q6_k(w: torch.Tensor) -> torch.Tensor:
+    out_f, in_f = w.shape
+    wp, nblk = _pad(w, QK_K)
+    x = wp.view(out_f, nblk, 16, 16)
+    scale, _l = _make_qx_quants_rmse(x)                         # [.., 16, 1]
+
+    sc4 = scale.squeeze(-1)                                     # [out, nb, 16] signed
+    max_sc = sc4.gather(-1, sc4.abs().argmax(dim=-1, keepdim=True)).squeeze(-1)
+    max_sc = torch.where(sc4.abs().amax(dim=-1) < EPS, torch.zeros_like(max_sc), max_sc)
+
+    isc = -128.0 / torch.where(max_sc.abs() < EPS, torch.ones_like(max_sc), max_sc)
+    sc16 = _nearest_int(isc.unsqueeze(-1) * sc4).clamp(-127, 127).to(torch.int64)
+    d = -max_sc / 128.0
+
+    dl = d.unsqueeze(-1).unsqueeze(-1) * sc16.to(torch.float32).unsqueeze(-1)
+    dl_safe = torch.where(dl == 0, torch.ones_like(dl), dl)   # C: if (!d) continue
+    l = (_nearest_int(x / dl_safe).clamp(-32, 31).to(torch.int64) + 32).to(torch.int64)  # 0..63
+
+    lo = l & 0xF
+    hi = (l >> 4) & 0x3
+    seg = l.view(out_f, nblk, 2, 8, 16)                 # [.., h, 8 subs, 16]
+    seg_lo = seg & 0xF
+    seg_hi = (seg >> 4) & 0x3
+    ql_h = torch.stack([
+        seg_lo[..., 0, :] | (seg_lo[..., 4, :] << 4),
+        seg_lo[..., 1, :] | (seg_lo[..., 5, :] << 4),
+        seg_lo[..., 2, :] | (seg_lo[..., 6, :] << 4),
+        seg_lo[..., 3, :] | (seg_lo[..., 7, :] << 4),
+    ], dim=-2)                                   # [out, nblk, 2, 4, 16]
+    ql = ql_h.reshape(out_f, nblk, 128)
+    qh_h = torch.stack([
+        seg_hi[..., 0, :] | (seg_hi[..., 2, :] << 2)
+        | (seg_hi[..., 4, :] << 4) | (seg_hi[..., 6, :] << 6),
+        seg_hi[..., 1, :] | (seg_hi[..., 3, :] << 2)
+        | (seg_hi[..., 5, :] << 4) | (seg_hi[..., 7, :] << 6),
+    ], dim=-2)                                   # [out, nblk, 2, 2, 16]
+    qh = qh_h.reshape(out_f, nblk, 64)
+
+    return _cat_blocks([ql, qh, sc16.to(torch.uint8), _f16_bytes(d)],
+                       out_f, nblk, 210)
+
+
+_GGUF_PACK = {
+    8: pack_q8_0,
+    10: pack_q2_k,
+    11: pack_q3_k,
+    12: pack_q4_k,
+    13: pack_q5_k,
+    14: pack_q6_k,
+}
+
+
+def pack_gguf_bytes(w: torch.Tensor, gguf_dtype: int) -> torch.Tensor:
+    """Pack fp32 [out, in] weight into GGUF bytes for a Q*_K / Q8_0 dtype."""
+    fn = _GGUF_PACK.get(gguf_dtype)
+    if fn is None:
+        raise ValueError(f"pack_gguf_bytes: no packer for GGML dtype {gguf_dtype}")
+    return fn(w)
+
+
+def packer_for(gguf_dtype: int):
+    """Return the packer callable for a GGML dtype, or None for i-quants."""
+    return _GGUF_PACK.get(gguf_dtype)
+
+
+def supported_pack_dtypes() -> list[int]:
+    return sorted(_GGUF_PACK.keys())

src/agiws_neural_quant/kquant/gguf_packed.py

@@ -0,0 +1,283 @@
+"""GGUF k-quant packed slice-dequantization — torch, GPU-friendly, output-row slice.
+
+Reads packed GGUF k-quant raw bytes (uint8 [out, bytes_per_row]) and dequantizes
+ONLY output rows [start:end] on-the-fly — for chunked matmul via QuantizedModule.
+Weights stay packed in VRAM; only the requested row slice is unpacked to fp16/fp32.
+
+This is the memory-efficient path: a Q3_K_S 12 GB model stays ~12 GB in VRAM,
+with only a small per-chunk dequant overhead (chunk_size rows at a time).
+
+Block layouts (from ggml-common.h, same as gguf_kquant_unpack.py):
+  Q8_0 (34 B):  d(f16) + qs[32] (int8).           y = d * q8
+  Q4_K (144 B): d(f16) + dmin(f16) + scales[12] + qs[128] (4-bit).
+                y = d * sc * q4 - dmin * m         (8 sub-blocks × 32, asymmetric)
+  Q5_K (176 B): d(f16) + dmin(f16) + scales[12] + qh[32] + qs[128] (4-bit).
+                y = d * sc * (q4 + 16*bit5) - dmin * m
+  Q6_K (210 B): ql[128] + qh[64] + scales[16] (int8) + d(f16).
+                y = d * sc * (q6 - 32)             (16 sub-blocks × 16, symmetric)
+
+All operations are on torch tensors (view uint8/int8, bit ops via int16/int32
+intermediate) so they run on GPU when the buffer is on cuda.
+"""
+
+from __future__ import annotations
+
+import torch
+
+from agiws_neural_quant.kquant._gguf_bits import f16_view_to_f32, unpack_scales_k4
+from agiws_neural_quant.kquant.gguf_packed_q2q3 import (
+    dequant_q2_k_packed_rows, dequant_q3_k_packed_rows,
+)
+from agiws_neural_quant.kquant.iq import (
+    dequant_iq2_xxs_packed_rows,
+    dequant_iq2_xs_packed,
+    dequant_iq2_s_packed,
+    dequant_iq3_xxs_packed,
+    dequant_iq3_s_packed,
+    dequant_iq1_s_packed,
+    dequant_iq1_m_packed,
+    dequant_iq4_nl_packed,
+    dequant_iq4_xs_packed,
+)
+
+
+# GGML dtype ids (kept here to avoid a top-level import of converters.gguf_reader
+# which would create a circular import via converters/__init__ -> universal -> quantizer).
+GGML_TYPE_Q8_0 = 8
+GGML_TYPE_Q4_K = 12
+GGML_TYPE_Q5_K = 13
+GGML_TYPE_Q6_K = 14
+GGML_TYPE_Q2_K = 10
+GGML_TYPE_Q3_K = 11
+GGML_TYPE_IQ2_XXS = 16
+GGML_TYPE_IQ2_XS = 17
+GGML_TYPE_IQ3_XXS = 18
+GGML_TYPE_IQ1_S = 19
+GGML_TYPE_IQ4_NL = 20
+GGML_TYPE_IQ3_S = 21
+GGML_TYPE_IQ2_S = 22
+GGML_TYPE_IQ4_XS = 23
+GGML_TYPE_IQ1_M = 29
+
+
+# Aliases kept for any external callers / tests.
+def _f16_view_to_f32(u8: torch.Tensor) -> torch.Tensor:
+    return f16_view_to_f32(u8)
+
+
+def _unpack_scales_k4(scales: torch.Tensor):
+    return unpack_scales_k4(scales)
+
+
+def dequant_q8_0_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q8_0 rows [start:end]. raw: [out, bytes_per_row] uint8."""
+    block = 32
+    elem = 34
+    chunk = raw[start:end].to(torch.int32)  # [cr, bytes_per_row]
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    d = _f16_view_to_f32(chunk[..., 0:2])  # [cr, n_blocks]
+    qs = chunk[..., 2:34].to(torch.int8).to(torch.float32)  # [cr, n_blocks, 32]
+    y = qs * d.unsqueeze(2)
+    y = y.reshape(cr, n_blocks * block)[:, :cols]
+    return y
+
+
+def dequant_q4_k_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q4_K rows [start:end]. raw: [out, bytes_per_row] uint8."""
+    block = 256
+    elem = 144
+    chunk = raw[start:end].to(torch.int32)
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    d = _f16_view_to_f32(chunk[..., 0:2])          # [cr, n_blocks]
+    dmin = _f16_view_to_f32(chunk[..., 2:4])       # [cr, n_blocks]
+    scales = chunk[..., 4:16].to(torch.uint8)      # [cr, n_blocks, 12]
+    qs = chunk[..., 16:144]                         # [cr, n_blocks, 128]
+    sc, m = _unpack_scales_k4(scales)              # each [cr, n_blocks, 8]
+    sc = sc.to(torch.float32)
+    m = m.to(torch.float32)
+    qs_pairs = qs.reshape(cr, n_blocks, 4, 32)
+    low = (qs_pairs & 0x0F).to(torch.float32)
+    high = (qs_pairs >> 4).to(torch.float32)
+    values = torch.empty(cr, n_blocks, 8, 32, dtype=torch.float32, device=raw.device)
+    values[..., 0::2, :] = low
+    values[..., 1::2, :] = high
+    y = d.unsqueeze(2).unsqueeze(3) * sc.unsqueeze(3) * values \
+        - dmin.unsqueeze(2).unsqueeze(3) * m.unsqueeze(3)
+    y = y.reshape(cr, n_blocks * block)[:, :cols]
+    return y
+
+
+def dequant_q5_k_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q5_K rows [start:end]. raw: [out, bytes_per_row] uint8."""
+    block = 256
+    elem = 176
+    chunk = raw[start:end].to(torch.int32)
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    d = _f16_view_to_f32(chunk[..., 0:2])
+    dmin = _f16_view_to_f32(chunk[..., 2:4])
+    scales = chunk[..., 4:16].to(torch.uint8)
+    qh = chunk[..., 16:48]                          # [cr, n_blocks, 32]
+    qs = chunk[..., 48:176]
+    sc, m = _unpack_scales_k4(scales)
+    sc = sc.to(torch.float32)
+    m = m.to(torch.float32)
+    qs_pairs = qs.reshape(cr, n_blocks, 4, 32)
+    low = (qs_pairs & 0x0F).to(torch.float32)
+    high = (qs_pairs >> 4).to(torch.float32)
+    values = torch.empty(cr, n_blocks, 8, 32, dtype=torch.float32, device=raw.device)
+    values[..., 0::2, :] = low
+    values[..., 1::2, :] = high
+    bit5 = torch.zeros(cr, n_blocks, 8, 32, dtype=torch.float32, device=raw.device)
+    for k in range(4):
+        bit5[..., 2 * k, :] = ((qh >> (2 * k)) & 1).to(torch.float32) * 16.0
+        bit5[..., 2 * k + 1, :] = ((qh >> (2 * k + 1)) & 1).to(torch.float32) * 16.0
+    values5 = values + bit5
+    y = d.unsqueeze(2).unsqueeze(3) * sc.unsqueeze(3) * values5 \
+        - dmin.unsqueeze(2).unsqueeze(3) * m.unsqueeze(3)
+    y = y.reshape(cr, n_blocks * block)[:, :cols]
+    return y
+
+
+def dequant_q6_k_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q6_K rows [start:end]. raw: [out, bytes_per_row] uint8.
+
+    Follows dequantize_row_q6_K (llama.cpp ggml-quants.c): per half-block n,
+    q1/q3 come from ql low/high nibble of first 32 bytes, q2/q4 of bytes 32-64,
+    with scale mapping sc[is+0/1/2/3] (is = l//16) -> sub-blocks 0/4/1/2
+    interleaved as in the reference.
+    """
+    block = 256
+    elem = 210
+    chunk = raw[start:end].to(torch.int32)
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    ql = chunk[..., 0:128]
+    qh = chunk[..., 128:192]
+    sc = chunk[..., 192:208].to(torch.uint8).view(torch.int8).to(torch.float32)  # [cr, n_blocks, 16]
+    d = _f16_view_to_f32(chunk[..., 208:210])                  # [cr, n_blocks]
+    y = torch.zeros(cr, n_blocks, 256, dtype=torch.float32, device=raw.device)
+    for n in range(2):
+        ql_c = ql[:, :, n * 64:(n + 1) * 64]
+        qh_c = qh[:, :, n * 32:(n + 1) * 32]
+        q1 = ((ql_c[:, :, 0:32] & 0x0F) | (((qh_c >> 0) & 3) << 4)).to(torch.float32) - 32.0
+        q2 = ((ql_c[:, :, 32:64] & 0x0F) | (((qh_c >> 2) & 3) << 4)).to(torch.float32) - 32.0
+        q3 = ((ql_c[:, :, 0:32] >> 4) | (((qh_c >> 4) & 3) << 4)).to(torch.float32) - 32.0
+        q4 = ((ql_c[:, :, 32:64] >> 4) | (((qh_c >> 6) & 3) << 4)).to(torch.float32) - 32.0
+        # scale mapping per C dequantize_row_q6_K: y[l]=d*sc[is+0/2/4/6] (is=l//16),
+        # i.e. q1->sc[0..1], q2->sc[2..3], q3->sc[4..5], q4->sc[6..7]; sc+=8 per n.
+        base = n * 8
+        d4 = d.unsqueeze(-1)
+        off = n * 128
+        y[:, :, off + 0:off + 16] = d4 * sc[:, :, base + 0].unsqueeze(-1) * q1[:, :, 0:16]
+        y[:, :, off + 16:off + 32] = d4 * sc[:, :, base + 1].unsqueeze(-1) * q1[:, :, 16:32]
+        y[:, :, off + 32:off + 48] = d4 * sc[:, :, base + 2].unsqueeze(-1) * q2[:, :, 0:16]
+        y[:, :, off + 48:off + 64] = d4 * sc[:, :, base + 3].unsqueeze(-1) * q2[:, :, 16:32]
+        y[:, :, off + 64:off + 80] = d4 * sc[:, :, base + 4].unsqueeze(-1) * q3[:, :, 0:16]
+        y[:, :, off + 80:off + 96] = d4 * sc[:, :, base + 5].unsqueeze(-1) * q3[:, :, 16:32]
+        y[:, :, off + 96:off + 112] = d4 * sc[:, :, base + 6].unsqueeze(-1) * q4[:, :, 0:16]
+        y[:, :, off + 112:off + 128] = d4 * sc[:, :, base + 7].unsqueeze(-1) * q4[:, :, 16:32]
+    return y.reshape(cr, n_blocks * block)[:, :cols]
+
+
+# Registry: GGML dtype id -> (slice dequant fn, block_size, bytes_per_block).
+_GGUF_SLICE_DEQUANT = {
+    GGML_TYPE_Q8_0: (dequant_q8_0_packed_rows, 32, 34),
+    GGML_TYPE_Q4_K: (dequant_q4_k_packed_rows, 256, 144),
+    GGML_TYPE_Q5_K: (dequant_q5_k_packed_rows, 256, 176),
+    GGML_TYPE_Q6_K: (dequant_q6_k_packed_rows, 256, 210),
+    GGML_TYPE_Q2_K: (dequant_q2_k_packed_rows, 256, 84),
+    GGML_TYPE_Q3_K: (dequant_q3_k_packed_rows, 256, 110),
+    # i-quants (QK_K = 256; IQ4_NL uses QK4_NL = 32).
+    GGML_TYPE_IQ2_XXS: (dequant_iq2_xxs_packed_rows, 256, 66),
+    GGML_TYPE_IQ2_XS: (dequant_iq2_xs_packed, 256, 74),
+    GGML_TYPE_IQ2_S: (dequant_iq2_s_packed, 256, 82),
+    GGML_TYPE_IQ3_XXS: (dequant_iq3_xxs_packed, 256, 98),
+    GGML_TYPE_IQ3_S: (dequant_iq3_s_packed, 256, 110),
+    GGML_TYPE_IQ1_S: (dequant_iq1_s_packed, 256, 66),
+    GGML_TYPE_IQ1_M: (dequant_iq1_m_packed, 256, 56),
+    GGML_TYPE_IQ4_NL: (dequant_iq4_nl_packed, 32, 18),
+    GGML_TYPE_IQ4_XS: (dequant_iq4_xs_packed, 256, 136),
+}
+
+
+def supported_gguf_dtypes() -> set[int]:
+    """GGML dtype ids that support packed slice-dequant."""
+    return set(_GGUF_SLICE_DEQUANT.keys())
+
+
+def dequant_gguf_slice(
+    weight_raw: torch.Tensor,
+    gguf_dtype: int,
+    start: int,
+    end: int,
+    cols: int,
+) -> torch.Tensor:
+    """Dequantize output rows [start:end] from packed GGUF k-quant bytes.
+
+    Args:
+        weight_raw: uint8 tensor [out_total, bytes_per_row] of packed GGUF data.
+        gguf_dtype: GGML_TYPE_* id (Q8_0, Q4_K, Q5_K, Q6_K, Q2_K, Q3_K).
+        start, end: output row range to dequant (0-based, end exclusive).
+        cols: in_features — trim the dequantized output to this many columns
+              (the last block may be padded beyond cols).
+
+    Returns:
+        fp32 tensor [end-start, cols].
+    """
+    entry = _GGUF_SLICE_DEQUANT.get(gguf_dtype)
+    if entry is None:
+        from agiws_neural_quant.converters.gguf_reader import GGML_TYPE_NAMES
+        raise ValueError(
+            f"dequant_gguf_slice: dtype id={gguf_dtype} "
+            f"({GGML_TYPE_NAMES.get(gguf_dtype, '?')}) not supported. "
+            f"Supported: {sorted(GGML_TYPE_NAMES[d] for d in _GGUF_SLICE_DEQUANT)}"
+        )
+    fn, _block, _bpb = entry
+    return fn(weight_raw, start, end, cols)
+
+
+def bytes_per_row(gguf_dtype: int, cols: int) -> int:
+    """Bytes per output row for a given GGUF dtype and column count."""
+    entry = _GGUF_SLICE_DEQUANT.get(gguf_dtype)
+    if entry is None:
+        raise ValueError(f"bytes_per_row: unsupported dtype {gguf_dtype}")
+    _fn, block, bpb = entry
+    n_blocks = (cols + block - 1) // block
+    return n_blocks * bpb
+
+
+__all__ = [
+    "dequant_gguf_slice",
+    "dequant_q8_0_packed_rows",
+    "dequant_q4_k_packed_rows",
+    "dequant_q5_k_packed_rows",
+    "dequant_q6_k_packed_rows",
+    "dequant_q2_k_packed_rows",
+    "dequant_q3_k_packed_rows",
+    "dequant_iq2_xxs_packed_rows",
+    "dequant_iq2_xs_packed",
+    "dequant_iq2_s_packed",
+    "dequant_iq3_xxs_packed",
+    "dequant_iq3_s_packed",
+    "dequant_iq1_s_packed",
+    "dequant_iq1_m_packed",
+    "dequant_iq4_nl_packed",
+    "dequant_iq4_xs_packed",
+    "supported_gguf_dtypes",
+    "bytes_per_row",
+]

src/agiws_neural_quant/kquant/gguf_packed_q2q3.py

@@ -0,0 +1,126 @@
+"""GGUF Q2_K / Q3_K packed slice-dequantization (torch, output-row slice).
+
+Exact llama.cpp layouts (ggml-common.h / ggml-quants.c):
+
+  block_q2_K (84 B): scales[16] (each byte: low nibble = 4-bit scale,
+                                   high nibble = 4-bit min)
+                     + qs[64] (256 x 2-bit) + d (f16) + dmin (f16)
+    dequant (dequantize_row_q2_K): 16 sub-blocks of 16.
+      sub-block is: n = is//8, j = is%8; qs byte = 32*n + 16*(is%2) + l;
+      shift = 2*(j%4); value = (qs[byte] >> shift) & 3;
+      y = d*sc*q - dmin*m, sc = scales[is]&0xF, m = scales[is]>>4
+
+  block_q3_K (110 B): hmask[32] + qs[64] + scales[12] + d(f16)
+    dequant (dequantize_row_q3_K): 16 sub-blocks of 16.
+      aux unpacking of scales[12] -> 16 int8 (signed), scale = int8 - 32
+      sub-block is: n = is//8, j_local = (is%8)//2, half = is%2;
+      qs byte = 32*n + 16*half + l; shift = 2*j_local;
+      m_bit = 1 << (j_local + 4*n); value = q - (hm[16*half+l]&m_bit ? 0 : 4);
+      y = d * scale * value
+"""
+
+from __future__ import annotations
+
+import torch
+
+from agiws_neural_quant.kquant._gguf_bits import f16_view_to_f32
+
+# GGML dtype ids (local - avoids circular import via converters/__init__).
+GGML_TYPE_Q2_K = 10
+GGML_TYPE_Q3_K = 11
+
+QK_K = 256
+
+
+def dequant_q2_k_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q2_K rows [start:end]. raw: [out, 84*n_blocks] uint8."""
+    block = QK_K
+    elem = 84
+    chunk = raw[start:end].to(torch.int32)
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    scales = chunk[..., 0:16]                                   # [cr, nb, 16]
+    qs = chunk[..., 16:80]                                      # [cr, nb, 64]
+    d = f16_view_to_f32(chunk[..., 80:82])                      # [cr, nb]
+    dmin = f16_view_to_f32(chunk[..., 82:84])                   # [cr, nb]
+
+    sc4 = (scales & 0x0F).to(torch.float32)                     # [cr, nb, 16]
+    m4 = (scales >> 4).to(torch.float32)
+
+    y = torch.zeros(cr, n_blocks, QK_K, dtype=torch.float32, device=raw.device)
+    for is_ in range(16):
+        n = is_ // 8
+        j_local = (is_ // 2) % 4
+        half = is_ % 2
+        byte_off = 32 * n + 16 * half
+        shift = 2 * j_local
+        qb = (qs[..., byte_off:byte_off + 16] >> shift) & 0x03   # [cr, nb, 16]
+        sc = sc4[..., is_:is_ + 1].unsqueeze(-1)                 # [cr, nb, 1, 1]
+        mn = m4[..., is_:is_ + 1].unsqueeze(-1)
+        d4 = d.unsqueeze(-1).unsqueeze(-1)
+        dm4 = dmin.unsqueeze(-1).unsqueeze(-1)
+        y[..., is_ * 16:is_ * 16 + 16] = (d4 * sc * qb.unsqueeze(2) - dm4 * mn).squeeze(2)
+    return y.reshape(cr, n_blocks * block)[:, :cols]
+
+
+def dequant_q3_k_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    """Dequant Q3_K rows [start:end]. raw: [out, 110*n_blocks] uint8."""
+    block = QK_K
+    elem = 110
+    chunk = raw[start:end].to(torch.int32)
+    cr = chunk.shape[0]
+    n_blocks = (cols + block - 1) // block
+    chunk = chunk.reshape(cr, n_blocks, elem)
+    hmask = chunk[..., 0:32]                                    # [cr, nb, 32]
+    qs = chunk[..., 32:96]                                      # [cr, nb, 64]
+    scales = chunk[..., 96:108]                                 # [cr, nb, 12]
+    d = f16_view_to_f32(chunk[..., 108:110])                    # [cr, nb]
+
+    # Unpack 12 bytes into 4 little-endian u32, then reorder (aux logic).
+    # aux[0..2] = scales[0:12] as 3 LE u32; aux[3] is garbage-filled in C.
+    s0 = (scales[..., 0] | (scales[..., 1] << 8) | (scales[..., 2] << 16) | (scales[..., 3] << 24)).to(torch.int64)
+    s1 = (scales[..., 4] | (scales[..., 5] << 8) | (scales[..., 6] << 16) | (scales[..., 7] << 24)).to(torch.int64)
+    s2 = (scales[..., 8] | (scales[..., 9] << 8) | (scales[..., 10] << 16) | (scales[..., 11] << 24)).to(torch.int64)
+    kmask1 = 0x03030303
+    kmask2 = 0x0F0F0F0F
+    tmp = s2
+    aux2 = ((s0 >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4)
+    aux3 = ((s1 >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4)
+    aux0 = (s0 & kmask2) | (((tmp >> 0) & kmask1) << 4)
+    aux1 = (s1 & kmask2) | (((tmp >> 2) & kmask1) << 4)
+    # 16 int8 scales from the 16 LE bytes of aux0..aux3.
+    sc8 = torch.zeros(cr, n_blocks, 16, dtype=torch.int64, device=raw.device)
+    for i, aux in enumerate([aux0, aux1, aux2, aux3]):
+        for b in range(4):
+            byte = (aux >> (8 * b)) & 0xFF
+            sc8[..., 4 * i + b] = torch.where(byte >= 128, byte - 256, byte)
+    sc_f = (sc8 - 32).to(torch.float32)                         # [cr, nb, 16]
+
+    y = torch.zeros(cr, n_blocks, QK_K, dtype=torch.float32, device=raw.device)
+    for is_ in range(16):
+        n = is_ // 8
+        j_local = (is_ // 2) % 4
+        half = is_ % 2
+        byte_off = 32 * n + 16 * half
+        shift = 2 * j_local
+        m_shift = j_local + 4 * n                     # m = 1 << m_shift
+        qb = ((qs[..., byte_off:byte_off + 16] >> shift) & 0x03).to(torch.float32)
+        hb = ((hmask[..., 16 * half:16 * half + 16] >> m_shift) & 1).to(torch.float32)
+        val = qb - 4.0 * (1.0 - hb)
+        sc = sc_f[..., is_:is_ + 1].unsqueeze(-1)
+        d4 = d.unsqueeze(-1).unsqueeze(-1)
+        y[..., is_ * 16:is_ * 16 + 16] = (d4 * sc * val.unsqueeze(2)).squeeze(2)
+    return y.reshape(cr, n_blocks * block)[:, :cols]
+
+
+__all__ = [
+    "dequant_q2_k_packed_rows",
+    "dequant_q3_k_packed_rows",
+    "GGML_TYPE_Q2_K",
+    "GGML_TYPE_Q3_K",
+]

src/agiws_neural_quant/kquant/gguf_plan.py

@@ -0,0 +1,67 @@
+"""UD (IQ2_M) layer plan - map GGUF tensor name -> quant dtype.
+
+Direct translation of llama.cpp llama-quant.cpp `set_type_for_tensor` for
+`LLAMA_FTYPE_MOSTLY_IQ2_M` (lines 450-529), for the dense `qwen35` arch:
+
+  token_embd           -> IQ3_S
+  output (lm_head)     -> Q5_K
+  attn_v               -> Q4_K   (n_gqa = 24/4 = 6 >= 4)
+  attn_output / ssm_out-> IQ3_S
+  ffn_down (first n/8) -> IQ3_S, rest -> IQ2_S
+  everything else      -> IQ2_S
+
+Small / norm / 1D tensors are handled by the caller as F32/F16 and must NOT
+be listed here.
+"""
+
+from __future__ import annotations
+
+# GGML dtype ids.
+F32 = 0
+F16 = 1
+Q8_0 = 8
+Q2_K = 10
+Q5_K = 13
+Q4_K = 12
+IQ2_S = 22
+IQ3_S = 21
+
+
+def _default_dtype(name: str, i_ffn_down: int, n_ffn_down: int,
+                   n_head: int, n_head_kv: int, ftype_iq2_m: bool) -> int:
+    """llama-quant.cpp set_type_for_tensor for MOSTLY_IQ2_M."""
+    if name == "token_embd.weight":
+        return IQ3_S if ftype_iq2_m else Q2_K
+    if name == "output.weight":
+        return Q5_K if ftype_iq2_m else Q8_0
+    if name.endswith(".attn_v.weight"):
+        # n_gqa >= 4 -> Q4_K; else IQ3_S (or Q2_K for non-IQ2_M).
+        if n_head // n_head_kv >= 4:
+            return Q4_K
+        return IQ3_S if ftype_iq2_m else Q2_K
+    if name.endswith(".attn_output.weight") or name.endswith(".ssm_out.weight"):
+        return IQ3_S if ftype_iq2_m else Q2_K
+    if name.endswith(".ffn_down.weight"):
+        if ftype_iq2_m and i_ffn_down < n_ffn_down // 8:
+            return IQ3_S
+        return IQ2_S
+    return IQ2_S
+
+
+def make_iq2_m_plan(tensor_names: list[str], *, n_head: int, n_head_kv: int,
+                    ftype_iq2_m: bool = True) -> dict[str, int]:
+    """Return {gguf_name: gguf_dtype} for the quantizable tensors.
+
+    tensor_names must be sorted in GGUF info-table order (llama.cpp walks
+    them in file order) so the `ffn_down` first-n/8 rule applies correctly.
+    """
+    plan: dict[str, int] = {}
+    n_ffn_down = sum(1 for n in tensor_names if n.endswith(".ffn_down.weight"))
+    i_ffn_down = 0
+    for name in tensor_names:
+        if name.endswith(".ffn_down.weight"):
+            plan[name] = _default_dtype(name, i_ffn_down, n_ffn_down, n_head, n_head_kv, ftype_iq2_m)
+            i_ffn_down += 1
+        else:
+            plan[name] = _default_dtype(name, 0, n_ffn_down, n_head, n_head_kv, ftype_iq2_m)
+    return plan

src/agiws_neural_quant/kquant/iq.py

@@ -0,0 +1,878 @@
+"""IQ-quant formats (llama.cpp i-quants): packed slice-dequant + quantizers.
+
+All i-quants are codebook formats: weight values are indices into a lattice
+grid; the grid entry (8 or 4 values) is scaled by a block delta and optionally
+sign-flipped. Grids come from llama.cpp ggml-common.h (see _iq_tables.py).
+
+Formats (block QK_K = 256, IQ4_NL uses QK4_NL = 32):
+  IQ2_XXS  2.0625 bpw  d(f16) + qs[64B]                       = 66 B
+  IQ2_XS   2.3125 bpw  d(f16) + qs[64B] + scales[8]           = 74 B
+  IQ2_S    2.5625 bpw  d(f16) + qs[64] + qh[8] + scales[8]    = 82 B
+  IQ3_XXS  3.0625 bpw  d(f16) + qs[96]                        = 98 B
+  IQ3_S    3.4375 bpw  d(f16) + qs[64] + qh[8] + signs[32] + scales[4] = 110 B
+  IQ1_S    1.5625 bpw  d(f16) + qs[32] + qh[16]u16            = 66 B
+  IQ1_M    1.75   bpw  qs[32] + qh[16] + scales[8]u16         = 56 B
+  IQ4_NL   4.5    bpw  d(f16) + qs[16] (block 32)             = 18 B
+  IQ4_XS   4.25   bpw  d(f16) + scales_h[2] + scales_l[4] + qs[128] = 136 B
+
+Dequant follows the CPU reference (ggml-quants.c, dequantize_row_iq*_*) so
+results match llama.cpp bit-for-bit on the same packed bytes. All functions
+slice output rows [start:end] and are GPU-friendly (torch ops only).
+
+Quantization (quantize_row_iq*_ref): for each 32-value group the sign pattern
+is chosen so the flipped magnitude vector lies on the grid; the best scale is
+found by weighted least squares over candidate scales; grid rows are looked
+up via the precomputed map + neighbour lists (same as iq2xs_init_impl).
+
+Author: AGIWS NeuralQuant team
+License: Apache 2.0
+"""
+
+from __future__ import annotations
+
+import torch
+
+from agiws_neural_quant.kquant._iq_tables import iq_table
+from agiws_neural_quant.kquant.gguf_pack import _cat_blocks
+
+QK_K = 256
+QK4_NL = 32
+IQ1S_DELTA = 0.125
+IQ1M_DELTA = 0.125
+
+# GGML dtype ids (see converters/gguf_reader.py).
+GGML_TYPE_IQ2_XXS = 16
+GGML_TYPE_IQ2_XS = 17
+GGML_TYPE_IQ3_XXS = 18
+GGML_TYPE_IQ1_S = 19
+GGML_TYPE_IQ4_NL = 20
+GGML_TYPE_IQ3_S = 21
+GGML_TYPE_IQ2_S = 22
+GGML_TYPE_IQ4_XS = 23
+GGML_TYPE_IQ1_M = 29
+
+_IQ_BYTES_PER_BLOCK = {
+    GGML_TYPE_IQ2_XXS: 66,
+    GGML_TYPE_IQ2_XS: 74,
+    GGML_TYPE_IQ2_S: 82,
+    GGML_TYPE_IQ3_XXS: 98,
+    GGML_TYPE_IQ3_S: 110,
+    GGML_TYPE_IQ1_S: 66,
+    GGML_TYPE_IQ1_M: 56,
+    GGML_TYPE_IQ4_NL: 18,
+    GGML_TYPE_IQ4_XS: 136,
+}
+
+
+def bytes_per_row_iq(ggml_dtype: int, cols: int) -> int:
+    block = QK4_NL if ggml_dtype == GGML_TYPE_IQ4_NL else QK_K
+    nbl = (cols + block - 1) // block
+    return nbl * _IQ_BYTES_PER_BLOCK[ggml_dtype]
+
+
+def _f16(u8: torch.Tensor) -> torch.Tensor:
+    """Interpret raw [..., 2] bytes as fp16 -> fp32 (LE). Returns [...]. """
+    return u8.view(torch.float16)[..., 0].to(torch.float32)
+
+
+def _le_u16(u8: torch.Tensor) -> torch.Tensor:
+    """[..., 2] bytes -> int64 (LE)."""
+    return (u8[..., 0].to(torch.int64) | (u8[..., 1].to(torch.int64) << 8))
+
+
+def _signs_apply(vals: torch.Tensor, signs: torch.Tensor) -> torch.Tensor:
+    """Flip sign of last-dim 8-tuples where the sign byte bit is set.
+
+    vals [..., 8], signs uint8 [..., 1] (last dim = 8-tuple index).
+    """
+    mask = iq_table("kmask_iq2xs").to(signs.device)  # [8]
+    bit = (signs.unsqueeze(-1) & mask).to(vals.dtype)  # [..., 1, 8] -> [..., 8]
+    return torch.where(bit != 0, -vals, vals)
+
+
+def _ksigns(idx: torch.Tensor) -> torch.Tensor:
+    """128-entry ksigns table lookup."""
+    t = iq_table("ksigns_iq2xs").to(idx.device)
+    return t[idx]
+
+
+# ---------------------------------------------------------------------------
+# IQ2_XXS -- block 66 B: d(2) + qs(32 x u16 = 64 B)
+#   per 32-group ib (8 groups):
+#     q2 = qs[4*ib..4*ib+3] (4 u16)
+#     aux32 = q2[2] | q2[3]<<16          (upper half carries scale+signs)
+#     db    = d * (0.5 + (aux32 >> 28)) * 0.25
+#     signs = ksigns[(aux32 >> 7*l) & 127]
+#     grid  = iq2xxs_grid[ (u8)q2[l] ]  (l=0..3)  -> 8 values
+# ---------------------------------------------------------------------------
+def dequant_iq2_xxs_packed_rows(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)          # [cr, nbl*66]
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 66
+    chunk = chunk.view(cr, nblk, 66)
+    d = _f16(chunk[..., 0:2])                       # [cr, nblk]
+    qs = chunk[..., 2:66].view(cr, nblk, 32, 2)     # 32 u16 LE
+    q2 = _le_u16(qs).view(cr, nblk, 8, 4)           # [cr, nblk, 8 grp, 4 u16]
+    aux32 = q2[..., 2] | (q2[..., 3] << 16)         # [cr, nblk, 8]
+    db = d.unsqueeze(-1) * (0.5 + ((aux32 >> 28) & 0xF).to(torch.float32)) * 0.25  # [cr,nblk,8]
+    l4 = torch.arange(4, device=raw.device)
+    signs7 = (aux32.unsqueeze(-1) >> (7 * l4).view(1, 1, 1, 4)) & 127  # [cr,nblk,8,4]
+    signs = _ksigns(signs7)                         # [cr,nblk,8,4]
+    g = iq_table("iq2xxs_grid").to(raw.device)      # [256, 8] int8
+    q0 = q2[..., 0]
+    q1 = q2[..., 1]
+    idx = torch.stack([q0 & 0xFF, q0 >> 8, q1 & 0xFF, q1 >> 8], dim=-1)  # [cr,nblk,8,4]
+    vals = g[idx.to(torch.int64)]                   # [cr,nblk,8,4,8]
+    vals = _signs_apply(vals, signs)
+    out = db.unsqueeze(-1).unsqueeze(-1) * vals     # [cr,nblk,8,4,8]
+    return out.reshape(cr, nblk * QK_K)[:, :cols]
+
+
+# ---------------------------------------------------------------------------
+# IQ2_XS -- block 74 B: d(2) + qs(64 B) + scales(8)
+#   qs: 32 u16; per group ib (8): q2 = qs16[4*ib+il] (il 0..3)
+#     grid = iq2xs_grid[q2 & 511], signs = ksigns[q2 >> 9]
+#     db = d * (0.5 + ((scales[ib] >> 4*(il/2)) & 0xF)) * 0.25
+# ---------------------------------------------------------------------------
+def dequant_iq2_xs_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 74
+    chunk = chunk.view(cr, nblk, 74)
+    d = _f16(chunk[..., 0:2])
+    qs = chunk[..., 2:66].view(cr, nblk, 32, 2)
+    q2 = _le_u16(qs)                                # [cr, nblk, 32]
+    q2 = q2.view(cr, nblk, 8, 4)                    # [cr, nblk, 8, 4]
+    scales = chunk[..., 66:74]                      # [cr, nblk, 8]
+    gr = iq_table("iq2xs_grid").to(raw.device)      # [512, 8]
+    vals = gr[(q2 & 0x1FF).to(torch.int64)]         # [cr, nblk, 8, 4, 8]
+    signs = _ksigns((q2 >> 9).to(torch.int64))
+    vals = _signs_apply(vals, signs)
+    il = torch.arange(4, device=raw.device)
+    code = (scales.to(torch.int64).unsqueeze(-1) >> (4 * (il // 2)).view(1, 1, 1, 4)) & 0xF  # [cr, nblk, 8, 4]
+    db = d.unsqueeze(-1).unsqueeze(-1) * (0.5 + code.to(torch.float32)) * 0.25  # [cr,nblk,8,4]
+    out = (db.unsqueeze(-1) * vals).reshape(cr, nblk * QK_K)[:, :cols]
+    return out
+
+
+# ---------------------------------------------------------------------------
+# IQ2_S -- block 82 B: d(2) + qs(64) + qh(8) + scales(8)
+#   per group ib (8), il 0..3:
+#     grid = iq2s_grid[ qs[4*ib+il] | ((qh[ib] << (8-2*il)) & 0x300) ]
+#     signs = qs[32 + 4*ib + il]
+#     db = d * (0.5 + ((scales[8] >> 4*(il/2)) & 0xF)) * 0.25
+# ---------------------------------------------------------------------------
+def dequant_iq2_s_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 82
+    chunk = chunk.view(cr, nblk, 82)
+    d = _f16(chunk[..., 0:2])
+    qs = chunk[..., 2:66].to(torch.int64)           # [cr, nblk, 64]
+    qh = chunk[..., 66:74].to(torch.int64)          # [cr, nblk, 8]
+    scales = chunk[..., 74:82].to(torch.int64)      # [cr, nblk, 8]
+    il = torch.arange(4, device=raw.device)
+    # Layout: indexes = qs[0..31] (4 per super-group), signs = qs[32..63].
+    idx_b = qs[..., :32].view(cr, nblk, 8, 4)
+    sign_b = qs[..., 32:64].view(cr, nblk, 8, 4)
+    gi = idx_b + ((qh.view(cr, nblk, 8, 1) << (8 - 2 * il.view(1, 1, 1, 4))) & 0x300)
+    gr = iq_table("iq2s_grid").to(raw.device)       # [1024, 8]
+    vals = gr[gi]                                   # [cr, nblk, 8, 4, 8]
+    vals = _signs_apply(vals, sign_b)
+    code = (scales.view(cr, nblk, 8, 1) >> (4 * (il.view(1, 1, 1, 4) // 2))) & 0xF
+    db = d.unsqueeze(-1).unsqueeze(-1) * (0.5 + code.to(torch.float32)) * 0.25
+    out = (db.unsqueeze(-1) * vals).reshape(cr, nblk * QK_K)[:, :cols]
+    return out
+
+
+# ---------------------------------------------------------------------------
+# IQ3_XXS -- block 98 B: d(2) + qs(96)
+#   qs[0:64] = 64 grid index bytes (8 groups x 8)
+#   qs[64:96] = scales_and_signs (8 groups x 4 = 32 B)
+#   per group ib: aux32 = sas[4*ib..4*ib+3] (LE u32)
+#     db = d * (0.5 + (aux32 >> 28)) * 0.5
+#     signs = ksigns[(aux32 >> 7*il) & 127]  (il 0..3)
+#     grid1 = iq3xxs_grid[qs[8*ib + 2*il]], grid2 = [qs[8*ib+2*il+1]]
+#     y[j] = db*g1[j]*sgn, y[j+4] = db*g2[j]*sgn  (j 0..3)
+# ---------------------------------------------------------------------------
+def dequant_iq3_xxs_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 98
+    chunk = chunk.view(cr, nblk, 98)
+    d = _f16(chunk[..., 0:2])
+    qs = chunk[..., 2:66].to(torch.int64)           # [cr, nblk, 64]
+    gas = chunk[..., 66:98].view(cr, nblk, 16, 2)   # scales+signs 16 u16
+    aux = _le_u16(gas).view(cr, nblk, 8, 2)         # [cr, nblk, 8 grp, 2 u16]
+    aux32 = (aux[..., 0] | (aux[..., 1] << 16)).to(torch.int64)  # [cr, nblk, 8]
+    db = d.unsqueeze(-1) * (0.5 + ((aux32 >> 28) & 0xF).to(torch.float32)) * 0.5
+    l4 = torch.arange(4, device=raw.device)
+    signs7 = (aux32.unsqueeze(-1) >> (7 * l4).view(1, 1, 1, 4)) & 127  # [cr, nblk, 8, 4]
+    signs = _ksigns(signs7)
+    gr = iq_table("iq3xxs_grid").to(raw.device)     # [256, 4]
+    idx = qs.view(cr, nblk, 8, 8)                   # [cr, nblk, 8 grp, 8 idx]
+    g1 = gr[idx[..., 0::2].to(torch.int64)]         # [cr, nblk, 8, 4, 4]
+    g2 = gr[idx[..., 1::2].to(torch.int64)]
+    vals = torch.cat([g1, g2], dim=-1)              # [cr, nblk, 8, 4, 8]
+    vals = _signs_apply(vals, signs)
+    out = (db.unsqueeze(-1).unsqueeze(-1) * vals).reshape(cr, nblk * QK_K)[:, :cols]
+    return out
+
+
+# ---------------------------------------------------------------------------
+# IQ3_S -- block 110 B: d(2) + qs(64) + qh(8) + signs(32) + scales(4)
+#   per group ib (0..7), il (0..3):
+#     grid1 = iq3s_grid[ qs[8*ib + 2*il]   | ((qh[ib] << (8-2*il)) & 256) ]
+#     grid2 = iq3s_grid[ qs[8*ib + 2*il+1] | ((qh[ib] << (7-2*il)) & 256) ]
+#     db    = d * (1 + 2*((scales[ib/2] >> 4*(ib%2)) & 0xF))
+#     signs = signs[4*ib + il]
+#     y[j] = db*grid1[j]*sgn, y[j+4] = db*grid2[j]*sgn  (j 0..3)
+# ---------------------------------------------------------------------------
+def dequant_iq3_s_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 110
+    chunk = chunk.view(cr, nblk, 110)
+    d = _f16(chunk[..., 0:2])                       # [cr, nblk]
+    qs = chunk[..., 2:66].to(torch.int64)           # [cr, nblk, 64]
+    qh = chunk[..., 66:74].to(torch.int64)          # [cr, nblk, 8]
+    signs = chunk[..., 74:106]                      # [cr, nblk, 32]
+    scales = chunk[..., 106:110].to(torch.int64)    # [cr, nblk, 4]
+    ib = torch.arange(8, device=raw.device)
+    il = torch.arange(4, device=raw.device)
+    qs8 = qs.view(cr, nblk, 8, 8)                   # [cr, nblk, grp, 8 idx bytes]
+    qh_sel = qh.view(cr, nblk, 8, 1)
+    gi1 = qs8[..., 0::2] + ((qh_sel << (8 - 2 * il.view(1, 1, 1, 4))) & 256)  # [cr,nblk,8,4]
+    gi2 = qs8[..., 1::2] + ((qh_sel << (7 - 2 * il.view(1, 1, 1, 4))) & 256)
+    gr = iq_table("iq3s_grid").to(raw.device)       # [512, 4]
+    v1 = gr[gi1]                                    # [cr, nblk, 8, 4, 4]
+    v2 = gr[gi2]
+    vals = torch.cat([v1, v2], dim=-1)              # [cr, nblk, 8, 4, 8]
+    # db per group (8 groups: scales[ib/2], nibble by ib%2)
+    code = (scales.view(cr, nblk, 4).repeat_interleave(2, dim=2)
+            >> (4 * (ib % 2).view(1, 1, 8))) & 0xF  # [cr, nblk, 8]
+    db = d.unsqueeze(-1) * (1 + 2 * code.to(torch.float32))  # [cr, nblk, 8]
+    # signs per group/il
+    sgn = signs.view(cr, nblk, 8, 4)                # [cr, nblk, 8, 4]
+    vals = _signs_apply(vals, sgn)
+    out = db.unsqueeze(-1).unsqueeze(-1) * vals     # [cr, nblk, 8, 4, 8]
+    return out.reshape(cr, nblk * QK_K)[:, :cols]
+
+
+# ---------------------------------------------------------------------------
+# IQ1_S -- block 66 B: d(2) + qs(32) + qh(16 u16)
+#   per 32-group ib (8):
+#     dl = d * (2*((qh[ib] >> 12) & 7) + 1)
+#     delta = qh[ib] & 0x8000 ? -IQ1S_DELTA : +IQ1S_DELTA
+#     grid idx = qs[4*ib+il] | (((qh[ib] >> 3*il) & 7) << 8)   (il 0..3)
+#     grid = iq1s_grid[int8]  (signed! values in {-1, 0, 1}-ish)
+#     y[j] = dl * (grid[j] + delta)
+# ---------------------------------------------------------------------------
+def dequant_iq1_s_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 66
+    chunk = chunk.view(cr, nblk, 66)
+    d = _f16(chunk[..., 0:2])                       # [cr, nblk]
+    qs = chunk[..., 2:34].to(torch.int64)           # [cr, nblk, 32]
+    qh = chunk[..., 34:66].view(cr, nblk, 16, 2)
+    qh = _le_u16(qh)                                # [cr, nblk, 16]
+    ib = torch.arange(8, device=raw.device)
+    il = torch.arange(4, device=raw.device)
+    qs8 = qs.view(cr, nblk, 8, 4)
+    qh8 = qh.view(cr, nblk, 8, 2)                   # 2 u16 per group
+    # per group: uses qh8[..., 0] for scale/delta, and qh8[..., 0] for index too
+    dl = d.unsqueeze(-1) * (2 * ((qh8[..., 0] >> 12) & 7) + 1).to(torch.float32)  # [cr,nblk,8]
+    delta = torch.where((qh8[..., 0] & 0x8000) != 0,
+                        -IQ1S_DELTA, IQ1S_DELTA)    # [cr,nblk,8]
+    idx = qs8 + (((qh8[..., 0].unsqueeze(-1) >> (3 * il.view(1, 1, 1, 4))) & 7) << 8)
+    gr = iq_table("iq1s_grid").to(raw.device)       # [2048, 8] int8 signed
+    vals = gr[idx].to(torch.float32)                # [cr,nblk,8,4,8]
+    out = dl.unsqueeze(-1).unsqueeze(-1) * (vals + delta.unsqueeze(-1).unsqueeze(-1))
+    return out.reshape(cr, nblk * QK_K)[:, :cols]
+
+
+# ---------------------------------------------------------------------------
+# IQ1_M -- block 56 B: qs(32) + qh(16) + scales(8 u16)
+#   No fp16 d! scale is f16 packed into 4 bytes of scales.
+#   scale.f16 = (sc[0]>>12) | ((sc[1]>>8)&0xF0) | ((sc[2]>>4)&0xF00) | (sc[3]&0xF000)
+#   per group ib: dl1 = d*(2*((sc[ib/2] >> 6*(ib%2)+0) & 0x7) + 1)
+#                 dl2 = d*(2*((sc[ib/2] >> 6*(ib%2)+3) & 0x7) + 1)
+#   idx[0] = qs[0] | ((qh[0]<<8)&0x700), idx[1] = qs[1] | ((qh[0]<<4)&0x700)
+#   idx[2] = qs[2] | ((qh[1]<<8)&0x700), idx[3] = qs[3] | ((qh[1]<<4)&0x700)
+#   delta[l] = qh[l/2] & (0x08 << 4*(l%2)) ? -IQ1M_DELTA : +IQ1M_DELTA
+#   y = dl1 * (grid[idx[l]] + delta[l])  (l 0,1)  dl2 for l 2,3
+# ---------------------------------------------------------------------------
+def dequant_iq1_m_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 56
+    chunk = chunk.view(cr, nblk, 56)
+    qs = chunk[..., 0:32].to(torch.int64)           # [cr,nblk,32]
+    qh = chunk[..., 32:48].to(torch.int64)          # [cr,nblk,16]
+    sc = chunk[..., 48:56].view(cr, nblk, 4, 2)
+    sc = _le_u16(sc)                                # [cr,nblk,4]
+    # reassemble f16: nibbles from sc[0..3]
+    u = (sc[..., 0] >> 12) | ((sc[..., 1] >> 8) & 0x00F0) \
+        | ((sc[..., 2] >> 4) & 0x0F00) | (sc[..., 3] & 0xF000)   # [cr,nblk]
+    d = u.to(torch.int16).view(torch.float16).to(torch.float32)
+    ib = torch.arange(8, device=raw.device)
+    il = torch.arange(4, device=raw.device)
+    sc8 = sc.repeat_interleave(2, dim=2)            # [cr,nblk,8]
+    # dl1 = d*(2*((sc8[ib] >> 6*(ib%2)) & 7)+1)
+    dl1 = d.unsqueeze(-1) * (2 * ((sc8 >> (6 * (ib % 2).view(1, 1, 8))) & 0x7) + 1).to(torch.float32)
+    dl2 = d.unsqueeze(-1) * (2 * ((sc8 >> (6 * (ib % 2).view(1, 1, 8) + 3)) & 0x7) + 1).to(torch.float32)
+    qh2 = qh.view(cr, nblk, 8, 2)                   # [cr,nblk,8,2]
+    qs8 = qs.view(cr, nblk, 8, 4)
+    idx0 = qs8[..., 0] | ((qh2[..., 0] << 8) & 0x700)
+    idx1 = qs8[..., 1] | ((qh2[..., 0] << 4) & 0x700)
+    idx2 = qs8[..., 2] | ((qh2[..., 1] << 8) & 0x700)
+    idx3 = qs8[..., 3] | ((qh2[..., 1] << 4) & 0x700)
+    gr = iq_table("iq1s_grid").to(torch.int64)      # [2048, 8]
+    g0 = gr[idx0].to(torch.float32)
+    g1 = gr[idx1].to(torch.float32)
+    g2 = gr[idx2].to(torch.float32)
+    g3 = gr[idx3].to(torch.float32)
+    delta0 = torch.where((qh2[..., 0] & 0x08) != 0, -IQ1M_DELTA, IQ1M_DELTA)
+    delta1 = torch.where((qh2[..., 0] & 0x80) != 0, -IQ1M_DELTA, IQ1M_DELTA)
+    delta2 = torch.where((qh2[..., 1] & 0x08) != 0, -IQ1M_DELTA, IQ1M_DELTA)
+    delta3 = torch.where((qh2[..., 1] & 0x80) != 0, -IQ1M_DELTA, IQ1M_DELTA)
+    out = torch.cat([
+        (dl1.unsqueeze(-1) * (g0 + delta0.unsqueeze(-1))).unsqueeze(-2),
+        (dl1.unsqueeze(-1) * (g1 + delta1.unsqueeze(-1))).unsqueeze(-2),
+        (dl2.unsqueeze(-1) * (g2 + delta2.unsqueeze(-1))).unsqueeze(-2),
+        (dl2.unsqueeze(-1) * (g3 + delta3.unsqueeze(-1))).unsqueeze(-2),
+    ], dim=-2)                                       # [cr,nblk,8,4,8]
+    return out.reshape(cr, nblk * QK_K)[:, :cols]
+
+
+# ---------------------------------------------------------------------------
+# IQ4_NL -- block 18 B: d(2) + qs(16)
+#   y[j] = d * kvalues_iq4nl[qs[j] & 0xF], y[j+16] = d * kvalues_iq4nl[qs[j] >> 4]
+# ---------------------------------------------------------------------------
+def dequant_iq4_nl_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 18
+    chunk = chunk.view(cr, nblk, 18)
+    d = _f16(chunk[..., 0:2])                       # [cr,nblk]
+    qs = chunk[..., 2:18]                           # [cr,nblk,16]
+    kv = iq_table("kvalues_iq4nl").to(raw.device)   # [16]
+    lo = kv[qs.to(torch.int64) & 0xF]               # [cr,nblk,16]
+    hi = kv[(qs >> 4).to(torch.int64)]
+    vals = torch.stack([lo, hi], dim=-1).reshape(cr, nblk, 32)
+    return (d.unsqueeze(-1) * vals).reshape(cr, nblk * QK4_NL)[:, :cols]
+
+
+# ---------------------------------------------------------------------------
+# IQ4_XS -- block 136 B: d(2) + scales_h(2) + scales_l(4) + qs(128)
+#   per 32-group ib (8): ls = ((scales_l[ib/2] >> 4*(ib%2)) & 0xF)
+#                             | (((scales_h >> 2*ib) & 3) << 4)
+#     dl = d * (ls - 32)
+#   y[j] = dl * kvalues_iq4nl[qs[j] & 0xF], y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4]
+# ---------------------------------------------------------------------------
+def dequant_iq4_xs_packed(
+    raw: torch.Tensor, start: int, end: int, cols: int
+) -> torch.Tensor:
+    chunk = raw[start:end].to(torch.uint8)
+    cr = chunk.shape[0]
+    nblk = chunk.shape[1] // 136
+    chunk = chunk.view(cr, nblk, 136)
+    d = _f16(chunk[..., 0:2])
+    scales_h = _le_u16(chunk[..., 2:4])             # [cr, nblk] uint16
+    scales_l = chunk[..., 4:8].to(torch.int64)      # [cr, nblk, 4]
+    qs = chunk[..., 8:136]                          # [cr, nblk, 128]
+    ib = torch.arange(8, device=raw.device)
+    ls = (scales_l.gather(2, (ib // 2).view(1, 1, 8).expand(cr, nblk, 8))
+          >> (4 * (ib % 2)).view(1, 1, 8)) & 0xF
+    ls = ls | (((scales_h.unsqueeze(-1) >> (2 * ib).view(1, 1, 8)) & 3) << 4)  # [cr,nblk,8]
+    dl = d.unsqueeze(-1) * (ls.to(torch.float32) - 32)
+    kv = iq_table("kvalues_iq4nl").to(raw.device)
+    qs32 = qs.view(cr, nblk, 8, 16)
+    lo = kv[qs32.to(torch.int64) & 0xF]             # [cr,nblk,8,16]
+    hi = kv[(qs32 >> 4).to(torch.int64)]
+    vals = torch.stack([lo, hi], dim=-1).reshape(cr, nblk, 8, 32)
+    return (dl.unsqueeze(-1) * vals).reshape(cr, nblk * QK_K)[:, :cols]
+
+
+
+
+
+# ===========================================================================
+# Quantizers (self-consistent with the dequantizers above)
+# ===========================================================================
+
+_SCALE_CANDIDATES = 17
+
+
+def _find_nearest(
+    x: torch.Tensor,            # [M, W] fp32 (magnitudes if not signed)
+    grid: torch.Tensor,         # [G, W] int8 grid values (W = 4 or 8)
+    signed: bool = False,       # grid rows carry signs (IQ1-style: {-1,0,1})
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Best (grid row, scale) for each W-vector by weighted nearest search.
+
+    Works for any grid row width W (4 for IQ3-style grids, 8 for IQ1/IQ2).
+    Weighted (by |x|) squared distance to every grid row over a log-spaced
+    candidate-scale grid; chunked over M. Returns (idx [M] int64,
+    scale [M] fp32). scale is the per-element multiplier: dequant gives
+    y = scale * grid values (scaled by the block d later).
+    """
+    M = x.shape[0]
+    G = grid.shape[0]
+    gf = grid.to(torch.float32)                         # [G, W]
+    xabs = x.abs()
+    gmax = gf.abs().amax(dim=-1).max().clamp(min=1e-8)
+    amax = xabs.amax(dim=-1).clamp(min=1e-8)            # [M]
+    cand = (amax / gmax).unsqueeze(-1) * torch.logspace(
+        -2.0, 1.0, _SCALE_CANDIDATES, device=x.device)  # [M, C] ~0.01..10 x amax
+    w = (xabs + 1e-6)                                   # importance weight
+    wnorm = w / w.sum(-1, keepdim=True).clamp(min=1e-8) # [M, W]
+    idx = torch.empty(M, dtype=torch.int64, device=x.device)
+    scale = torch.empty(M, dtype=torch.float32, device=x.device)
+    chunk = max(1, min(M, (1 << 21) // max(1, G)))
+    for s0 in range(0, M, chunk):
+        e = min(s0 + chunk, M)
+        xs = x[s0:e]                                    # [K, W]
+        wn = wnorm[s0:e]                                # [K, W]
+        cs = cand[s0:e]                                 # [K, C]
+        # d2[K, C, G] = sum_i wn_i * (x_i - s*g_i)^2
+        xg = (wn * xs) @ gf.T                           # [K, G]
+        g2w = (wn.unsqueeze(1) * gf.unsqueeze(0) ** 2).sum(-1)  # [K, G]
+        x2w = (wn * xs * xs).sum(-1)                    # [K]
+        d2 = x2w.unsqueeze(-1).unsqueeze(-1) \
+            - 2 * cs.unsqueeze(-1) * xg.unsqueeze(1) \
+            + (cs.unsqueeze(-1) ** 2) * g2w.unsqueeze(1)  # [K, C, G]
+        d2 = d2.clamp(min=0)
+        flat = d2.view(e - s0, -1)
+        best = flat.argmin(dim=-1)                      # [K]
+        scale_idx = best // G
+        grid_idx = best % G
+        scale[s0:e] = cs.view(e - s0, -1).gather(1, scale_idx.unsqueeze(-1)).squeeze(-1)
+        idx[s0:e] = grid_idx
+    # Refine scale by weighted least squares on the chosen row.
+    gv = grid[idx].to(torch.float32)
+    num = (w * x * gv).sum(-1)
+    den = (w * gv * gv).sum(-1).clamp(min=1e-8)
+    scale = torch.where(den > 0, num / den, scale)
+    return idx, scale
+
+
+def _pick_signs8(x8: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    """Sign bits with even parity (mirror llama.cpp ksigns trick).
+
+    Returns (sign_bits [.., 8] int64 0/1, xabs [.., 8] fp32). For odd-popcount
+    vectors the smallest-magnitude element is flipped.
+    """
+    sgn = (x8 < 0).to(torch.int64)
+    odd = (sgn.sum(-1) % 2) == 1
+    if odd.any():
+        mn = x8.abs().argmin(-1)                        # [..]
+        o = odd.unsqueeze(-1)
+        m = mn.unsqueeze(-1)
+        pos = torch.arange(x8.shape[-1], device=x8.device).view(1, -1) == m
+        sgn = torch.where(o & pos, 1 - sgn, sgn)
+    return sgn, x8.abs()
+
+
+def _ksigns_encode(sgn: torch.Tensor) -> torch.Tensor:
+    """sgn [.., 8] int64 bits -> ksigns table index [..] int64."""
+    sb = (sgn * (1 << torch.arange(8, device=sgn.device))).sum(-1)  # [..] 0..255
+    ks = iq_table("ksigns_iq2xs").to(sgn.device)        # [128] uint8
+    lut = torch.full((256,), -1, dtype=torch.int64, device=sgn.device)
+    lut[ks.to(torch.int64)] = torch.arange(128, device=sgn.device)
+    return lut[sb.to(torch.int64)]
+
+
+def _pack_u16(v: torch.Tensor) -> torch.Tensor:
+    """int64 [.., N] -> [.., N, 2] LE bytes."""
+    return torch.stack([v & 0xFF, (v >> 8) & 0xFF], dim=-1).to(torch.uint8)
+
+
+def _pack_f16(d: torch.Tensor) -> torch.Tensor:
+    """fp32 [N] -> [N, 2] LE bytes."""
+    return _pack_u16(d.to(torch.float16).view(torch.int16).to(torch.int64) & 0xFFFF)
+
+
+def quantize_iq2_xxs(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ2_XXS bytes [out, 66*nblk] (d + 32 u16).
+
+    Block 256 = 8 super-groups x 32; each super-group = 4 8-groups.
+    Per super-group: 4 u16 = [idx0|idx1, idx2|idx3, aux_lo, aux_hi];
+    aux32 = (ls << 28) | (sgn_idx[0] << 0) | (sgn_idx[1] << 7)
+          | (sgn_idx[2] << 14) | (sgn_idx[3] << 21).
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    sgn, xa = _pick_signs8(x32.view(-1, 8))
+    sgn = sgn.view(out_f, nblk, 8, 4, 8)                # [out, nblk, sg, 4x8, 8]
+    xa = xa.view(out_f, nblk, 8, 4, 8)
+    grid = iq_table("iq2xxs_grid").to(W.device)         # [256, 8]
+    idx, scale = _find_nearest(xa.reshape(-1, 8), grid)    # [out*nblk*32]
+    idx = idx.view(out_f, nblk, 8, 4)
+    scale = scale.view(out_f, nblk, 8, 4)
+    sc_g = scale.amax(dim=-1).clamp(min=1e-8)           # [out, nblk, 8] per super-group
+    # llama.cpp: scale is "unit" scale (for q=1); dequant gives y = (scale/8)*grid.
+    # Our per-8-group scale is x/grid -> unit scale = 8*scale.
+    d = 8.0 * sc_g.amax(dim=-1) / 31.0                  # [out, nblk] per block
+    ls = torch.clamp(torch.round(0.5 * (8.0 * scale / d.unsqueeze(-1).unsqueeze(-1) - 1)), 0, 15).to(torch.int64)
+    sgn_idx = _ksigns_encode(sgn)                       # [out, nblk, 8, 4] 0..127
+    aux32 = (ls << 28) | (sgn_idx << (7 * torch.arange(4, device=W.device)).view(1, 1, 1, 4))
+    aux32 = aux32[..., 0] | aux32[..., 1] | aux32[..., 2] | aux32[..., 3]  # [out, nblk, 8]
+    qb = idx.to(torch.int64) & 0xFF
+    q2 = (qb[..., 0] | (qb[..., 1] << 8)) & 0xFFFF      # [out, nblk, 8]
+    q2h = (qb[..., 2] | (qb[..., 3] << 8)) & 0xFFFF
+    aux_lo = aux32 & 0xFFFF
+    aux_hi = (aux32 >> 16) & 0xFFFF
+    u16 = torch.stack([q2, q2h, aux_lo, aux_hi], dim=-1)  # [out, nblk, 8, 4]
+    return _cat_blocks([_pack_f16(d),
+                        _pack_u16(u16).reshape(out_f, nblk, 64)],
+                       out_f, nblk, 66)
+
+
+def quantize_iq2_xs(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ2_XS bytes [out, 74*nblk] (d + 32 u16 + scales[8]).
+
+    Each 32-group: 4 u16, each = 9-bit grid index | (7-bit sign index << 9).
+    scales[ib] = 2 4-bit codes (lo = sub-group<2, hi = sub-group>=2) of the
+    unit scale / 8.
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    sgn, xa = _pick_signs8(x32.view(-1, 8))
+    sgn = sgn.view(out_f, nblk, 8, 4, 8)
+    xa = xa.view(out_f, nblk, 8, 4, 8)
+    grid = iq_table("iq2xs_grid").to(W.device)          # [512, 8]
+    idx, scale = _find_nearest(xa.reshape(-1, 8), grid)
+    idx = idx.view(out_f, nblk, 8, 4)
+    scale = scale.view(out_f, nblk, 8, 4)
+    unit = 8.0 * scale                                  # [out, nblk, 8, 4]
+    pair = unit.view(out_f, nblk, 8, 2, 2).amax(-1)     # [out, nblk, 8, 2]
+    d = pair.amax(dim=(-1, -2)) / 31.0                  # [out, nblk] per block
+    d4 = d.unsqueeze(-1).unsqueeze(-1)                  # [out, nblk, 1, 1]
+    code = torch.clamp(torch.round(0.5 * (pair / d4 - 1)), 0, 15).to(torch.int64)
+    scales = code[..., 0] | (code[..., 1] << 4)         # [out, nblk, 8]
+    sgn_idx = _ksigns_encode(sgn)                       # [out, nblk, 8, 4] 0..127
+    q2 = (idx.to(torch.int64) & 0x1FF) | (sgn_idx << 9)  # [out, nblk, 8, 4]
+    return _cat_blocks([_pack_f16(d),
+                        _pack_u16(q2.reshape(out_f, nblk, 32)).reshape(out_f, nblk, 64),
+                        scales.to(torch.uint8)],
+                       out_f, nblk, 74)
+
+
+def quantize_iq2_s(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ2_S bytes [out, 82*nblk].
+
+    qs[4*ib+il] = low byte of grid index, qh[ib] = 2 high bits per sub-group,
+    signs byte at qs[32+4*ib+il], scales[8] = 4-bit pair.
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    sgn, xa = _pick_signs8(x32.view(-1, 8))
+    sgn = sgn.view(out_f, nblk, 8, 4, 8)
+    xa = xa.view(out_f, nblk, 8, 4, 8)
+    grid = iq_table("iq2s_grid").to(W.device)           # [1024, 8]
+    idx, scale = _find_nearest(xa.reshape(-1, 8), grid)
+    idx = idx.view(out_f, nblk, 8, 4)
+    scale = scale.view(out_f, nblk, 8, 4)
+    unit = 8.0 * scale
+    unit_p = unit.view(out_f, nblk, 8, 2, 2).amax(-1)   # [out, nblk, 8, 2]
+    d = unit_p.amax(dim=(-1, -2)) / 31.0
+    d4 = d.unsqueeze(-1).unsqueeze(-1)
+    code = torch.clamp(torch.round(0.5 * (unit_p / d4 - 1)), 0, 15).to(torch.int64)
+    scales = code[..., 0] | (code[..., 1] << 4)
+    # signs: plain 8-bit byte (NOT ksigns!) at qs[32+4*ib+il]
+    signs_b = (sgn * (1 << torch.arange(8, device=W.device))).sum(-1).to(torch.uint8)
+    qs_low = (idx.to(torch.int64) & 0xFF).to(torch.uint8)
+    qs_hi = signs_b
+    qh = ((idx.to(torch.int64) >> 8) << (2 * torch.arange(4, device=W.device)).view(1, 1, 1, 4)).sum(-1) & 0xFF
+    return _cat_blocks([_pack_f16(d), qs_low.reshape(out_f, nblk, 32),
+                        qs_hi.reshape(out_f, nblk, 32), qh,
+                        scales.to(torch.uint8)],
+                       out_f, nblk, 82)
+
+
+def quantize_iq3_xxs(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ3_XXS bytes [out, 98*nblk].
+
+    qs[0:64] = 64 grid-index bytes (8 groups x 8), qs[64:96] = scales+signs
+    (8 groups x 4 u16 = aux32 per group). grid = iq3xxs_grid [256, 4].
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    sgn, xa = _pick_signs8(x32.view(-1, 8))
+    sgn = sgn.view(out_f, nblk, 8, 4, 8)
+    xa = xa.view(out_f, nblk, 8, 4, 8)
+    grid = iq_table("iq3xxs_grid").to(W.device)         # [256, 4]
+    # Each 8-group = 2 interleaved 4-halves (lo at idx[2*il], hi at idx[2*il+1]).
+    xa_4 = xa.view(out_f, nblk, 8, 4, 2, 4)             # [.,.,8,4,2 halves,4]
+    xa_lo = xa_4[..., 0, :].reshape(-1, 4)
+    xa_hi = xa_4[..., 1, :].reshape(-1, 4)
+    idx_lo, s_lo = _find_nearest(xa_lo, grid)
+    idx_hi, s_hi = _find_nearest(xa_hi, grid)
+    scale = torch.maximum(s_lo, s_hi).view(out_f, nblk, 8, 4)
+    idx = (idx_lo.view(out_f, nblk, 8, 4).to(torch.int64)
+           | (idx_hi.view(out_f, nblk, 8, 4).to(torch.int64) << 8))  # 2 x 8-bit per sub-group
+    # per-32-group scale (4 sub-groups): max over the 4
+    sg = scale.amax(dim=-1).clamp(min=1e-8)             # [out, nblk, 8]
+    d = sg.amax(dim=-1) / 15.0                          # [out, nblk]
+    d4 = d.unsqueeze(-1).unsqueeze(-1)
+    # db = d*(0.5+(aux>>28))*0.5 -> aux = 4*scale/d - 1... solve: aux = round(scale/d/0.5 - 0.5)
+    ls = torch.clamp(torch.round(2.0 * scale / d4 - 0.5), 0, 15).to(torch.int64)
+    sgn_idx = _ksigns_encode(sgn)                       # [out, nblk, 8, 4] 0..127
+    aux32 = (ls << 28) | (sgn_idx << (7 * torch.arange(4, device=W.device)).view(1, 1, 1, 4))
+    aux32 = aux32[..., 0] | aux32[..., 1] | aux32[..., 2] | aux32[..., 3]  # [out, nblk, 8]
+    # Per sub-group: 2 grid codes of 4 -> 2 bytes, interleaved (lo, hi).
+    # Layout of qs[0:64] per 8-group: [lo_il0, hi_il0, lo_il1, hi_il1, ...].
+    lo_b = (idx & 0xFF).view(out_f, nblk, 8, 4)         # [.,.,8,4]
+    hi_b = ((idx >> 8) & 0xFF).view(out_f, nblk, 8, 4)
+    qs_idx = torch.stack([lo_b, hi_b], dim=-1).reshape(out_f, nblk, 64).to(torch.uint8)
+    aux_pairs = torch.stack([aux32 & 0xFFFF, aux32 >> 16], dim=-1)  # [out, nblk, 8, 2]
+    return _cat_blocks([_pack_f16(d), qs_idx,
+                        _pack_u16(aux_pairs.reshape(out_f, nblk, 16)).reshape(out_f, nblk, 32)],
+                       out_f, nblk, 98)
+
+
+def quantize_iq3_s(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ3_S bytes [out, 110*nblk].
+
+    qs[0:64] grid-index bytes (lo/hi interleaved), qh[8] high bits,
+    signs[32], scales[4] (nibble per group of a pair).
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    sgn, xa = _pick_signs8(x32.view(-1, 8))
+    sgn = sgn.view(out_f, nblk, 8, 4, 8)
+    xa = xa.view(out_f, nblk, 8, 4, 8)
+    grid = iq_table("iq3s_grid").to(W.device)           # [512, 4]
+    xa_4 = xa.view(out_f, nblk, 8, 4, 2, 4)
+    idx_lo, s_lo = _find_nearest(xa_4[..., 0, :].reshape(-1, 4), grid)
+    idx_hi, s_hi = _find_nearest(xa_4[..., 1, :].reshape(-1, 4), grid)
+    idx_lo = idx_lo.view(out_f, nblk, 8, 4).to(torch.int64)
+    idx_hi = idx_hi.view(out_f, nblk, 8, 4).to(torch.int64)
+    scale = torch.stack([s_lo, s_hi], -1).amax(-1).view(out_f, nblk, 8, 4)
+    # db per group (8 groups; scales[ib/2] nibble by ib%2, code = (unit/d-1)/2)
+    scale_g = scale.amax(dim=-1)                        # [out, nblk, 8]
+    sg_pairs = scale_g.view(out_f, nblk, 4, 2)          # [out, nblk, 4, 2] (even, odd)
+    d = scale_g.amax(dim=-1) / 15.0                     # [out, nblk]
+    d4 = d.unsqueeze(-1).unsqueeze(-1)                  # [out, nblk, 1, 1]
+    code_pair = torch.clamp(torch.round((sg_pairs / d4 - 1.0) * 0.5), 0, 15).to(torch.int64)
+    scales = (code_pair[..., 0] | (code_pair[..., 1] << 4)).to(torch.uint8)  # [out, nblk, 4]
+    # qs bytes: interleave lo/hi codes per sub-group; qh carries high bits.
+    lo_code = (idx_lo & 0xFF).to(torch.uint8)
+    hi_code = (idx_hi & 0xFF).to(torch.uint8)
+    qs_idx = torch.stack([lo_code, hi_code], dim=-1).reshape(out_f, nblk, 64)
+    il2 = torch.arange(4, device=W.device)
+    qh_lo = (idx_lo >> 8) & 1
+    qh_hi = (idx_hi >> 8) & 1
+    qh = ((qh_lo << (2 * il2).view(1, 1, 1, 4))
+          | (qh_hi << (2 * il2 + 1).view(1, 1, 1, 4))).sum(-1) & 0xFF  # [out, nblk, 8]
+    qh = qh.to(torch.uint8)
+    signs_b = (sgn * (1 << torch.arange(8, device=W.device))).sum(-1).to(torch.uint8)  # [out,nblk,8,4]
+    return _cat_blocks([_pack_f16(d), qs_idx, qh,
+                        signs_b.reshape(out_f, nblk, 32), scales],
+                       out_f, nblk, 110)
+
+
+def quantize_iq1_s(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ1_S bytes [out, 66*nblk].
+
+    d(f16) + qs[32] + qh[16]u16. grid = iq1s_grid [2048, 8] (signed, {-1,0,1}).
+    Per 32-group ib: dl = d*(2*((qh[ib]>>12)&7)+1); delta = +-IQ1S_DELTA
+    (sign from qh[ib]&0x8000); idx = qs[4*ib+il] | (((qh[ib]>>3*il)&7)<<8);
+    y[j] = dl * (grid[idx][j] + delta).
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    grid = iq_table("iq1s_grid").to(W.device)           # [2048, 8] int8 signed
+    x8 = x32.view(out_f * nblk * 32, 8)
+    # Search signed grid: dequant y = dl*(g+delta); for a candidate delta
+    # the effective target is x/dl - delta. Do a 2-pass: first find the best
+    # (idx, scale=dl) on the signed grid, then decide delta by the residual
+    # sign and re-pick if needed. Approx is fine at 1.5 bpw.
+    idx, scale = _find_nearest(x8, grid, signed=True)
+    idx = idx.view(out_f, nblk, 8, 4)
+    scale = scale.view(out_f, nblk, 8, 4)               # dl per 8-group
+    # group multiplier: dl = d*(2*m+1), m 0..7
+    g_max = scale.amax(dim=-1).clamp(min=1e-8)          # [out, nblk, 8]
+    m = torch.clamp(torch.round((g_max / g_max.amax(dim=-1).unsqueeze(-1) * 7 - 1) * 0.5), 0, 7).to(torch.int64)
+    d = (g_max.amax(dim=-1) / (2 * 7 + 1)).clamp(min=1e-8)  # [out, nblk]
+    dl = d.unsqueeze(-1) * (2 * m + 1)                  # [out, nblk, 8] per super-group
+    id_dl = 1.0 / dl.unsqueeze(-1).unsqueeze(-1)
+    gv = grid[idx].to(torch.float32)                    # [out, nblk, 8, 4, 8]
+    rec = (dl.unsqueeze(-1).unsqueeze(-1) * gv)         # [out, nblk, 8, 4, 8]
+    res = x32.view(out_f, nblk, 8, 4, 8) - rec
+    # delta: one sign per 32-group (aggregate residual over all 32 values)
+    delta = torch.where(res.mean(dim=(-1, -2)) < 0, -IQ1S_DELTA, IQ1S_DELTA)  # [out, nblk, 8]
+    # adjust idx for the delta (approximate: re-search on x/dl - delta)
+    tgt = x32.view(out_f, nblk, 8, 4, 8) * id_dl - delta.unsqueeze(-1).unsqueeze(-1)
+    idx2, _ = _find_nearest(tgt.reshape(-1, 8), grid, signed=True)
+    idx = idx2.view(out_f, nblk, 8, 4)
+    # encode: qh[ib] = (sign delta << 15) | (m << 12) | (3 high bits of idx per il)
+    qh = (delta < 0).to(torch.int64) << 15              # [out, nblk, 8]
+    qh = qh | (m << 12)                                 # [out, nblk, 8]
+    qs = (idx.to(torch.int64) & 0xFF)                   # low 8 bits per (ib, il)
+    hi3 = (idx.to(torch.int64) >> 8) & 7
+    qh = qh | (hi3 << (3 * torch.arange(4, device=W.device)).view(1, 1, 1, 4)).sum(-1)
+    # qh storage: 16 u16 per block (2 per group); decoder reads first of each pair.
+    qh16 = torch.zeros(out_f, nblk, 16, dtype=torch.int64, device=W.device)
+    qh16[..., 0::2] = qh
+    return _cat_blocks([_pack_f16(d), qs.to(torch.uint8).reshape(out_f, nblk, 32),
+                        _pack_u16(qh16).reshape(out_f, nblk, 32)],
+                       out_f, nblk, 66)
+
+
+def quantize_iq4_xs(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ4_XS bytes [out, 136*nblk].
+
+    d(f16) + scales_h[2] + scales_l[4] + qs[128].
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    bl = Wp.view(out_f, nblk, 8, 32)
+    amax = bl.abs().amax(dim=-1).clamp(min=1e-8)        # [out, nblk, 8]
+    d = amax.amax(dim=-1) / 31.0                        # [out, nblk]
+    kv = iq_table("kvalues_iq4nl").to(W.device)         # [16]
+    id_d = 1.0 / d.unsqueeze(-1).unsqueeze(-1)
+    al = bl * id_d                                      # [out, nblk, 8, 32]
+    codes = (al.unsqueeze(-1) - kv.view(1, 1, 1, 1, 16)).abs().argmin(-1)  # [out,nblk,8,32]
+    # scales: 6-bit per group = (scales_l nibble) | (scales_h 2-bit << 4), -32 offset
+    ls = torch.clamp(torch.round(amax / d.unsqueeze(-1)), 0, 63).to(torch.int64)  # [out,nblk,8]
+    ls = ls - 32
+    scales_l = (ls & 0xF).view(out_f, nblk, 4, 2)
+    scales_l = (scales_l[..., 0] | (scales_l[..., 1] << 4)).view(out_f, nblk, 4)
+    hi2 = (ls >> 4) & 3                                # [out,nblk,8] 2-bit each
+    h0 = hi2[..., 0] | (hi2[..., 1] << 2) | (hi2[..., 2] << 4) | (hi2[..., 3] << 6)
+    h1 = hi2[..., 4] | (hi2[..., 5] << 2) | (hi2[..., 6] << 4) | (hi2[..., 7] << 6)
+    scales_h = torch.stack([h0, h1], dim=-1).view(out_f, nblk, 2).to(torch.int64)
+    packed = (codes[..., 0::2].to(torch.int64) | (codes[..., 1::2].to(torch.int64) << 4)).to(torch.uint8)  # [out,nblk,8,16]
+    return _cat_blocks([_pack_f16(d), scales_h.to(torch.uint8),
+                        scales_l.to(torch.uint8),
+                        packed.reshape(out_f, nblk, 128)],
+                       out_f, nblk, 136)
+
+
+def quantize_iq4_nl(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ4_NL bytes [out, 18*nblk32].
+
+    Block = 32 values: d(f16) + qs[16]. y[j] = d*kvalues[qs&0xF],
+    y[j+16] = d*kvalues[qs>>4]. kvalues = [-127,-104,-83,-65,-49,-35,-22,
+    -10,1,13,25,38,53,69,89,113].
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK4_NL - 1) // QK4_NL
+    Wp = torch.zeros(out_f, nblk * QK4_NL, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    bl = Wp.view(out_f, nblk, QK4_NL)                   # [out, nblk, 32]
+    kv = iq_table("kvalues_iq4nl").to(W.device)         # [16] fp32
+    amax = bl.abs().amax(dim=-1).clamp(min=1e-8)        # [out, nblk]
+    d = amax / kv.abs().max().clamp(min=1e-8)           # [out, nblk]
+    id_d = 1.0 / d.unsqueeze(-1)
+    al = bl * id_d                                      # [out, nblk, 32]
+    codes = (al.unsqueeze(-1) - kv.view(1, 1, 1, 16)).abs().argmin(-1)  # [out,nblk,32]
+    packed = (codes[..., 0::2].to(torch.int64) | (codes[..., 1::2].to(torch.int64) << 4)).to(torch.uint8)  # [out,nblk,16]
+    return _cat_blocks([_pack_f16(d), packed], out_f, nblk, 18)
+
+
+def quantize_iq1_m(W: torch.Tensor) -> torch.Tensor:
+    """[out, in] fp32 -> IQ1_M bytes [out, 56*nblk].
+
+    qs[32] + qh[16] + scales[8 u16]. No fp16 d: scale is reassembled from the
+    top nibbles of the 8 scale u16 (see dequant_iq1_m_packed).
+    """
+    out_f, in_f = W.shape
+    nblk = (in_f + QK_K - 1) // QK_K
+    Wp = torch.zeros(out_f, nblk * QK_K, dtype=torch.float32, device=W.device)
+    Wp[:, :in_f] = W
+    x32 = Wp.view(out_f, nblk, QK_K)
+    grid = iq_table("iq1s_grid").to(W.device)           # [2048, 8] signed {-1,0,1}
+    x8 = x32.view(out_f, nblk, 8, 4, 8)
+    # per 8-group: find best (idx, scale) on the signed grid
+    idx, scale = _find_nearest(x8.reshape(-1, 8), grid, signed=True)
+    idx = idx.view(out_f, nblk, 8, 4)
+    scale = scale.view(out_f, nblk, 8, 4)
+    # group dl (per 4 sub-groups of a 32-group... actually per 32-group pair):
+    # dl1 = d*(2*sc3+1), dl2 = d*(2*sc3+3)  -> sc3 = 3-bit scale per 16-group.
+    # Approx: per 32-group, dl = max scale over its 4 sub-groups.
+    g_max = scale.amax(dim=-1).clamp(min=1e-8)          # [out, nblk, 8] per 32-group
+    # per 32-group 2 codes: dl1 = max(scale of sub-groups 0,1), dl2 = max(2,3)
+    pair_scale = scale.view(out_f, nblk, 8, 2, 2).amax(dim=-1)  # [out, nblk, 8, 2]
+    dmax = pair_scale.amax(dim=-1).amax(dim=-1).clamp(min=1e-8)  # [out, nblk]
+    code2 = torch.clamp(torch.round((pair_scale / dmax.unsqueeze(-1).unsqueeze(-1) * 7 - 1) * 0.5), 0, 7).to(torch.int64)  # [out,nblk,8,2]
+    dl = dmax.unsqueeze(-1).unsqueeze(-1) * (2 * code2 + 1)  # [out, nblk, 8, 2]
+    # sc[p] u16 (p=0..3): 2 codes per 32-group, groups 2p (even, bits 0-5) and
+    # 2p+1 (odd, bits 6-11): [dl1, dl2] of each group.
+    c = code2.view(out_f, nblk, 4, 2, 2)                # [out,nblk,4 u16,2 groups,2 dl]
+    gA1, gA2 = c[..., 0, 0], c[..., 0, 1]
+    gB1, gB2 = c[..., 1, 0], c[..., 1, 1]
+    sc16 = gA1 | (gA2 << 3) | (gB1 << 6) | (gB2 << 9)   # [out, nblk, 4]
+    # fp16 d in the top nibbles across the 4 u16:
+    # d16 = (sc0>>12) | ((sc1>>8)&0xF0) | ((sc2>>4)&0xF00) | (sc3&0xF000)
+    d16 = dmax.to(torch.float16).view(torch.int16).to(torch.int64) & 0xFFFF  # [out,nblk]
+    d16e = d16.unsqueeze(-1)                            # [out, nblk, 1]
+    sc16 = sc16 | (((d16e >> 12) & 0xF) << 12)
+    sc16 = sc16 | ((((d16e >> 8) & 0xF) << 8) & 0x0F00)
+    sc16 = sc16 | ((((d16e >> 4) & 0xF) << 4) & 0x00F0)
+    sc16 = sc16 | (d16e & 0x000F)
+    # delta per group (from qh 0x08 / 0x80 bits) + idx high bits in qh
+    delta = torch.where(g_max / dmax.unsqueeze(-1) < 0.5, -IQ1M_DELTA, IQ1M_DELTA)  # placeholder
+    delta = torch.full_like(g_max, IQ1M_DELTA)          # sign from residual below
+    # delta: reconstruct with dl per (group, pair): dl1 for il 0..1, dl2 for 2..3
+    dl_pair = dl.view(out_f, nblk, 8, 2, 1)             # [out,nblk,8,2 dl,1]
+    dl_full = dl_pair.expand(-1, -1, -1, -1, 2).reshape(out_f, nblk, 8, 4)  # per sub-group
+    gv = grid[idx].to(torch.float32)                    # [out,nblk,8,4,8]
+    res = x8 - dl_full.unsqueeze(-1) * gv
+    delta = torch.where(res.mean(dim=(-1, -2)) < 0, -IQ1M_DELTA, IQ1M_DELTA)  # [out,nblk,8]
+    # qh: 16 bytes per block (2 per 32-group). Per 32-group:
+    #   qh[0] bits: idx0 hi3<<8, idx1 hi3<<4, delta0<<3, delta1<<7  (via 0x08/0x80)
+    #   qh[1] bits: idx2 hi3<<8, idx3 hi3<<4, delta2<<3, delta3<<7
+    hi3 = (idx.to(torch.int64) >> 8) & 7                # [out,nblk,8,4]
+    qh2 = torch.zeros(out_f, nblk, 8, 2, dtype=torch.int64, device=W.device)
+    qh2[..., 0] |= (hi3[..., 0] << 8) | (hi3[..., 1] << 4)
+    qh2[..., 1] |= (hi3[..., 2] << 8) | (hi3[..., 3] << 4)
+    dlt = (delta < 0).to(torch.int64)                   # [out,nblk,8]
+    qh2[..., 0] |= (dlt << 3) & 0x08
+    qh2[..., 0] |= (dlt << 7) & 0x80
+    qh2[..., 1] |= (dlt << 3) & 0x08
+    qh2[..., 1] |= (dlt << 7) & 0x80
+    qs = (idx.to(torch.int64) & 0xFF)
+    return _cat_blocks([qs.to(torch.uint8).reshape(out_f, nblk, 32),
+                        qh2.to(torch.uint8).reshape(out_f, nblk, 16),
+                        _pack_u16(sc16).reshape(out_f, nblk, 8)],
+                       out_f, nblk, 56)