ArGrigorov commited on
Commit
3731ea4
·
verified ·
1 Parent(s): eba1afe

fix iq2_s qh int64->uint8 (packer byte width); v0.5.7

Browse files
src/agiws_neural_quant/kquant/__pycache__/iq.cpython-313.pyc CHANGED
Binary files a/src/agiws_neural_quant/kquant/__pycache__/iq.cpython-313.pyc and b/src/agiws_neural_quant/kquant/__pycache__/iq.cpython-313.pyc differ
 
src/agiws_neural_quant/kquant/iq.py CHANGED
@@ -610,7 +610,7 @@ def quantize_iq2_s(W: torch.Tensor) -> torch.Tensor:
610
  signs_b = (sgn * (1 << torch.arange(8, device=W.device))).sum(-1).to(torch.uint8)
611
  qs_low = (idx.to(torch.int64) & 0xFF).to(torch.uint8)
612
  qs_hi = signs_b
613
- qh = ((idx.to(torch.int64) >> 8) << (2 * torch.arange(4, device=W.device)).view(1, 1, 1, 4)).sum(-1) & 0xFF
614
  return _cat_blocks([_pack_f16(d), qs_low.reshape(out_f, nblk, 32),
615
  qs_hi.reshape(out_f, nblk, 32), qh,
616
  scales.to(torch.uint8)],
 
610
  signs_b = (sgn * (1 << torch.arange(8, device=W.device))).sum(-1).to(torch.uint8)
611
  qs_low = (idx.to(torch.int64) & 0xFF).to(torch.uint8)
612
  qs_hi = signs_b
613
+ qh = (((idx.to(torch.int64) >> 8) << (2 * torch.arange(4, device=W.device)).view(1, 1, 1, 4)).sum(-1) & 0xFF).to(torch.uint8)
614
  return _cat_blocks([_pack_f16(d), qs_low.reshape(out_f, nblk, 32),
615
  qs_hi.reshape(out_f, nblk, 32), qh,
616
  scales.to(torch.uint8)],