Image-Text-to-Text
Transformers
Safetensors
qwen3_5
text-generation
dashq
quantized
post-training-quantization
int3
conversational
custom_code
Instructions to use jkim96/Qwen3.6-27B-DASHQ-INT3-g64 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use jkim96/Qwen3.6-27B-DASHQ-INT3-g64 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="jkim96/Qwen3.6-27B-DASHQ-INT3-g64", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# Load model directly from transformers import AutoProcessor, AutoModelForCausalLM processor = AutoProcessor.from_pretrained("jkim96/Qwen3.6-27B-DASHQ-INT3-g64", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("jkim96/Qwen3.6-27B-DASHQ-INT3-g64", trust_remote_code=True, device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=40) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use jkim96/Qwen3.6-27B-DASHQ-INT3-g64 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "jkim96/Qwen3.6-27B-DASHQ-INT3-g64" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "jkim96/Qwen3.6-27B-DASHQ-INT3-g64", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/jkim96/Qwen3.6-27B-DASHQ-INT3-g64
- SGLang
How to use jkim96/Qwen3.6-27B-DASHQ-INT3-g64 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "jkim96/Qwen3.6-27B-DASHQ-INT3-g64" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "jkim96/Qwen3.6-27B-DASHQ-INT3-g64", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "jkim96/Qwen3.6-27B-DASHQ-INT3-g64" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "jkim96/Qwen3.6-27B-DASHQ-INT3-g64", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use jkim96/Qwen3.6-27B-DASHQ-INT3-g64 with Docker Model Runner:
docker model run hf.co/jkim96/Qwen3.6-27B-DASHQ-INT3-g64
Add DASH-Q remote-code inference (Triton decode kernel)
Browse files- dashq_kernel.py +0 -23
dashq_kernel.py
CHANGED
|
@@ -165,29 +165,6 @@ class TritonQuantLinear(nn.Module):
|
|
| 165 |
in_features // self.group_size,
|
| 166 |
)
|
| 167 |
|
| 168 |
-
@classmethod
|
| 169 |
-
def from_packed(cls, module: nn.Module, **kwargs) -> "TritonQuantLinear":
|
| 170 |
-
"""Build from a dashq.quantization.PackedQuantizedLinear instance."""
|
| 171 |
-
from dashq.quantization import _unpack_int_values
|
| 172 |
-
|
| 173 |
-
K = int(getattr(module, "quant_in_features", module.in_features))
|
| 174 |
-
N = int(module.out_features)
|
| 175 |
-
W_int = _unpack_int_values(module.W_q_packed, module.nbits, module.numel).view(N, K)
|
| 176 |
-
num_groups = K // int(module.group_size)
|
| 177 |
-
scale = module.scale.view(N, num_groups)
|
| 178 |
-
zero = module.zero.view(N, num_groups)
|
| 179 |
-
bias = module.bias if getattr(module, "bias", None) is not None else None
|
| 180 |
-
return cls(
|
| 181 |
-
W_int,
|
| 182 |
-
scale,
|
| 183 |
-
zero,
|
| 184 |
-
int(module.nbits),
|
| 185 |
-
int(module.group_size),
|
| 186 |
-
bias=bias,
|
| 187 |
-
out_dtype=getattr(module, "linear_dtype", torch.float16),
|
| 188 |
-
**kwargs,
|
| 189 |
-
)
|
| 190 |
-
|
| 191 |
def dequantize_weight(self, dtype: torch.dtype) -> torch.Tensor:
|
| 192 |
"""Returns W^T as (in_features, out_features), matching the K-major layout."""
|
| 193 |
if self.nbits == 3:
|
|
|
|
| 165 |
in_features // self.group_size,
|
| 166 |
)
|
| 167 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
def dequantize_weight(self, dtype: torch.dtype) -> torch.Tensor:
|
| 169 |
"""Returns W^T as (in_features, out_features), matching the K-major layout."""
|
| 170 |
if self.nbits == 3:
|