| { |
| "schema_version": "5.1", |
| "name": "qwen3.8-27b-p150x2", |
| "tt_metal_version": "0.65.2.dev9856", |
| "arch": "blackhole", |
| "device_count": 2, |
| "producer": { |
| "tt_kernel_version": "0.1.0", |
| "created_at": "2026-10-02T20:12:52.368580+00:00", |
| "hostname": "tt-quietbox" |
| }, |
| "weights": { |
| "repo_id": "Qwen/Qwen3.8-27B", |
| "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", |
| "allow_patterns": null, |
| "ignore_patterns": null, |
| "repo_type": "model" |
| }, |
| "mesh": null, |
| "entrypoint": null, |
| "resources": null, |
| "capabilities": null, |
| "env": {}, |
| "bundled": null, |
| "deps": null, |
| "container": { |
| "image": { |
| "registry": "hf", |
| "repository": "qwen3.8-27b-p150x2", |
| "tag": "tt-model/qwen3.8-27b-p150x2:58b3a5c0c045", |
| "digest": "sha256:58b3a5c0c045a4853ae1174db6e0e0e77393131f099295b059238f273d38b5c2" |
| }, |
| "kind": "vllm-plugin", |
| "runtime": { |
| "vllm": { |
| "version": "0.26.0" |
| }, |
| "plugin": { |
| "repo": "https://github.com/changh95/vllm-tt-plugin", |
| "ref": "ab5f7f3d795ee519b5d96f41f31cc1e0f635d6a7", |
| "sha": "ab5f7f3d795ee519b5d96f41f31cc1e0f635d6a7" |
| }, |
| "extra_models_dir": "models/demos/blackhole/qwen36/vllm_bundle", |
| "lock": "requirements.lock" |
| }, |
| "serve": { |
| "hardware": "p150x2", |
| "mesh_device": "P150x2", |
| "port": 8000, |
| "max_model_len": null, |
| "max_num_seqs": null, |
| "block_size": 64, |
| "server_timeout": null, |
| "capabilities": { |
| "tool_parser": "qwen3_coder", |
| "reasoning_parser": "qwen3" |
| }, |
| "additional_config": { |
| "tt": { |
| "l1_small_size": 24576 |
| } |
| }, |
| "args": [], |
| "env": { |
| "ARCH_NAME": "blackhole", |
| "TT_QWEN35_TEXT_VER": "qwen36_blackhole", |
| "QWEN36_SKIP_VISION": "1", |
| "QWEN_SDPA_BF8": "1", |
| "VLLM_RPC_TIMEOUT": "900000", |
| "VLLM_CONFIGURE_LOGGING": "1", |
| "TORCHDYNAMO_DISABLE": "1", |
| "HF_HUB_OFFLINE": "1" |
| } |
| }, |
| "serve_profiles": [ |
| { |
| "hardware": null, |
| "mesh_device": null, |
| "port": null, |
| "max_model_len": 65536, |
| "max_num_seqs": 32, |
| "block_size": null, |
| "server_timeout": null, |
| "capabilities": null, |
| "additional_config": { |
| "tt": { |
| "fabric_config": "FABRIC_1D", |
| "trace_region_size": 1073741824 |
| } |
| }, |
| "args": [ |
| [ |
| "--max-num-batched-tokens", |
| "65536" |
| ] |
| ], |
| "env": { |
| "MESH_DEVICE": "P150x2", |
| "TT_MESH_GRAPH_DESC_PATH": "/opt/tt-metal/models/demos/blackhole/qwen36/mesh/p150_x2_2link_mesh_graph_descriptor.textproto", |
| "TT_MODEL_CLASS_OVERRIDES": "TTQwen3_5ForConditionalGeneration=models.demos.blackhole.qwen36.tt.qwen36_vllm:Qwen36ForCausalLM,Qwen3_5ForConditionalGeneration=models.demos.blackhole.qwen36.tt.qwen36_vllm:Qwen36ForCausalLM", |
| "QWEN36_MTP": "0", |
| "QWEN36_MAX_TOKENS_ALL_USERS": "622592", |
| "TT_CACHE_PATH": "/cache/tensors-tp2" |
| }, |
| "name": "tp2", |
| "description": "Plain TP=2 decode over both chips, up to 32 concurrent users, full sampling support, 64K context." |
| }, |
| { |
| "hardware": null, |
| "mesh_device": null, |
| "port": null, |
| "max_model_len": 65536, |
| "max_num_seqs": 8, |
| "block_size": null, |
| "server_timeout": null, |
| "capabilities": null, |
| "additional_config": { |
| "tt": { |
| "trace_region_size": 536870912 |
| } |
| }, |
| "args": [ |
| [ |
| "--max-num-batched-tokens", |
| "65536" |
| ], |
| "--no-enable-prefix-caching", |
| "--no-async-scheduling" |
| ], |
| "env": { |
| "TT_SERVE_LAUNCHER": "pd_container", |
| "MESH_DEVICE": "P150", |
| "TT_MODEL_CLASS_OVERRIDES": "TTQwen3_5ForConditionalGeneration=models.demos.blackhole.qwen36.tt.qwen36_vllm:Qwen36ForCausalLM,Qwen3_5ForConditionalGeneration=models.demos.blackhole.qwen36.tt.qwen36_vllm:Qwen36ForCausalLM", |
| "QWEN36_FORCE_TP_PATH": "1", |
| "QWEN36_PREFILL_BUCKET_TRACE": "1", |
| "QWEN36_MTP": "0", |
| "QWEN36_MAX_TOKENS_ALL_USERS": "163840", |
| "TT_PD_CHIPS": "0,1", |
| "TT_PD_CONNS": "2", |
| "TT_PD_P_POOL": "65536", |
| "TT_PD_D_POOL": "163840", |
| "TT_PD_EXPORT_SLOTS": "1", |
| "TT_PD_LEASE": "30", |
| "TT_PD_FABRIC_CLAIM_LEASE_S": "0", |
| "TT_PD_TAG": "ttm", |
| "TT_PD_GDN_FUSED": "2", |
| "TT_PD_ALLOW_FUSED_CONV": "1", |
| "TT_PD_SSE_KEEPALIVE": "0", |
| "TT_CACHE_PATH": "/cache/tensors-tp1", |
| "TT_METAL_OPERATION_TIMEOUT_SECONDS": "600" |
| }, |
| "name": "p1d1", |
| "description": "Prefill on chip 0, decode on chip 1 (TP=1 each), KV handed over the ethernet fabric; up to 8 concurrent decodes, 64K context." |
| }, |
| { |
| "hardware": null, |
| "mesh_device": null, |
| "port": null, |
| "max_model_len": 65536, |
| "max_num_seqs": 4, |
| "block_size": null, |
| "server_timeout": null, |
| "capabilities": null, |
| "additional_config": { |
| "tt": { |
| "fabric_config": "FABRIC_1D", |
| "trace_region_size": 1073741824, |
| "sample_on_device_mode": "decode_only" |
| } |
| }, |
| "args": [ |
| [ |
| "--max-num-batched-tokens", |
| "65536" |
| ], |
| "--no-async-scheduling" |
| ], |
| "env": { |
| "MESH_DEVICE": "P150x2", |
| "TT_MESH_GRAPH_DESC_PATH": "/opt/tt-metal/models/demos/blackhole/qwen36/mesh/p150_x2_2link_mesh_graph_descriptor.textproto", |
| "QWEN36_DRAFTER": "dflash2", |
| "QWEN36_DFLASH_TP": "1", |
| "QWEN36_GDN_SPEC_FUSED": "1", |
| "QWEN36_DFLASH_BLOCK": "8", |
| "QWEN36_DFLASH_FOLD_SEED": "1", |
| "QWEN36_DFLASH_SERVE_BLOCK": "32", |
| "QWEN36_PREFILL_BUCKET_TRACE": "0", |
| "DFLASH_WEIGHTS": "incoai/Qwen3.8-27B-DFlash2@dedf8df68adfb1afeaf7b7480c0a0243108177b4", |
| "QWEN36_MAX_TOKENS_ALL_USERS": "262144", |
| "TT_CACHE_PATH": "/cache/tensors-tp2" |
| }, |
| "name": "tp2-dflash2", |
| "description": "TP=2 with lossless DFlash2 speculative decoding (single 4x8 bucket, composite GDN verify), 1-4 concurrent users, greedy output, 64K context." |
| } |
| ], |
| "default_profile": "tp2", |
| "code_dir": "code", |
| "verify": [ |
| "from models.demos.blackhole.qwen36.tt.qwen36_vllm import Qwen36ForCausalLM; assert Qwen36ForCausalLM", |
| "from models.demos.blackhole.qwen36.tt.kv_transfer import Qwen36KVTransfer; assert Qwen36KVTransfer", |
| "import vllm_tt_plugin.kv_transfer.tt_connector, vllm_tt_plugin.kv_transfer.pd_proxy, vllm_tt_plugin.kv_transfer.launch.pd_fabric_rank, vllm_tt_plugin.kv_transfer.launch.pd_container, vllm_tt_plugin.kv_transfer.launch.dispatch", |
| "import importlib.metadata as m; eps = {e.name for e in m.entry_points(group='vllm.general_plugins')}; assert 'tt_serve_launcher' in eps, eps", |
| "import importlib.resources as r; assert r.files('vllm_tt_plugin.kv_transfer.launch').joinpath('data/pd_two_p150_mesh_graph_descriptor.textproto').is_file()", |
| "import importlib; importlib.import_module('ttnn.distributed.ttrun')", |
| "from pathlib import Path; assert Path('/opt/openmpi-v5.0.7-ulfm/bin/mpirun').is_file()", |
| "from pathlib import Path; p = Path('/opt/tt-metal/models/demos/blackhole/qwen36/mesh/p150_x2_2link_mesh_graph_descriptor.textproto'); assert p.is_file(); assert 'count: 2' in p.read_text(), p", |
| "import os, subprocess, sys; os.environ.update(TT_PD_CHIPS='0,1', TT_METAL_CACHE='/tmp/c', TT_CACHE_PATH='/tmp/t'); subprocess.run([sys.executable, '-m', 'vllm_tt_plugin.kv_transfer.launch.pd_container', 'Qwen/Qwen3.8-27B', '--max-model-len', '65536', '--max-num-seqs', '8', '--block-size', '64', '--port', '20000', '--dry-run'], check=True)", |
| "import os; os.environ['QWEN36_DRAFTER']='dflash2'; from models.demos.blackhole.qwen36.tt.qwen36_vllm_dflash import Qwen36DFlashForCausalLM as C; assert C.model_capabilities['tt_adaptive_block_batched']", |
| "import json; m=json.load(open('/opt/tt-metal/models/demos/blackhole/qwen36/vllm_bundle/qwen36/vllm_metadata.json')); assert m['main_class'].endswith(':Qwen36DFlashForCausalLM'), m", |
| "import vllm.model_executor.models.qwen3_5; import transformers.models.qwen3_5", |
| "from pathlib import Path; assert Path('/opt/tt-metal/models/model_trace_region_sizes.yaml').is_file()" |
| ], |
| "built": { |
| "image": "tt-model/qwen3.8-27b-p150x2:58b3a5c0c045", |
| "repo": "changh95/qwen3.8-27b-p150x2", |
| "tt_model_version": "0.1.0", |
| "created_at": "2026-10-02T20:06:49+00:00", |
| "tt_metal": { |
| "sha": "fee9e0d35948111be29083c4eb144023d86890d8", |
| "describe": "v0.79.0-dev20260903-138-gfee9e0d359", |
| "dirty": false, |
| "scm_version": "0.65.2.dev9856", |
| "mode": "local", |
| "remote": "https://github.com/tenstorrent/tt-metal.git", |
| "branch": "qwen36-p150x2", |
| "pushed": true |
| }, |
| "code_sha256": "b0c865d431db3b437ab3e681b01835c22f2f116164a2fe2e30c635fd43e0d6b9", |
| "plugin": { |
| "sha": "ab5f7f3d795ee519b5d96f41f31cc1e0f635d6a7", |
| "repo": "https://github.com/changh95/vllm-tt-plugin" |
| }, |
| "image_digest": "sha256:58b3a5c0c045a4853ae1174db6e0e0e77393131f099295b059238f273d38b5c2" |
| } |
| } |
| } |
