Qwen
/

Qwen-72B-Chat-Int4

Text Generation

4-bit precision

Model card Files Files and versions

yangapku commited on Dec 7, 2023

Commit

f81518f

·

1 Parent(s): ccda762

update modeling_qwen.py

Files changed (1) hide show

modeling_qwen.py +1 -1

modeling_qwen.py CHANGED Viewed

@@ -520,7 +520,7 @@ class QWenAttention(nn.Module):
             if not self.use_cache_quantization and SUPPORT_TORCH2:
                 if attention_mask is not None:
-                    attention_mask = attention_mask.expand(-1, -1, key_size, -1)
                     if causal_mask is not None:
                         attention_mask = attention_mask.masked_fill(~causal_mask, torch.finfo(query.dtype).min)
                 else:

             if not self.use_cache_quantization and SUPPORT_TORCH2:
                 if attention_mask is not None:
+                    attention_mask = attention_mask.expand(-1, -1, query.size(2), -1)
                     if causal_mask is not None:
                         attention_mask = attention_mask.masked_fill(~causal_mask, torch.finfo(query.dtype).min)
                 else: