docker-compose: re-add use_index_cache:true to HF_OVERRIDES (winning DCP4+MTP5 recipe)
Browse files- docker-compose.yml +2 -2
docker-compose.yml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
# GLM-5.2-NVFP4-REAP (168 experts / ~504B) — vLLM serving on 4× 96 GB Blackwell GPUs
|
| 2 |
# (e.g. RTX PRO 6000, SM120). Works for both the -504B (code-calib) and -504B-term variants.
|
| 3 |
#
|
| 4 |
-
# Best config (defaults below): DCP4 + MTP5 + global-topk + DCP-sharded draft
|
| 5 |
# → 474k-token KV pool; 70-94 tok/s codegen, ~50 tok/s @256k (flat at depth),
|
| 6 |
# 256k prefill no-wedge, ~200 tok/s aggregate @ concurrency 4.
|
| 7 |
#
|
|
@@ -74,7 +74,7 @@ services:
|
|
| 74 |
NUM_SPECULATIVE_TOKENS: ${NUM_SPECULATIVE_TOKENS:-5}
|
| 75 |
MAX_CUDAGRAPH_CAPTURE_SIZE: ${MAX_CUDAGRAPH_CAPTURE_SIZE:-}
|
| 76 |
SPEC_CONFIG: ${SPEC_CONFIG:-}
|
| 77 |
-
HF_OVERRIDES: '{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'
|
| 78 |
entrypoint:
|
| 79 |
- bash
|
| 80 |
- -lc
|
|
|
|
| 1 |
# GLM-5.2-NVFP4-REAP (168 experts / ~504B) — vLLM serving on 4× 96 GB Blackwell GPUs
|
| 2 |
# (e.g. RTX PRO 6000, SM120). Works for both the -504B (code-calib) and -504B-term variants.
|
| 3 |
#
|
| 4 |
+
# Best config (defaults below): DCP4 + MTP5 + global-topk + DCP-sharded draft + use_index_cache
|
| 5 |
# → 474k-token KV pool; 70-94 tok/s codegen, ~50 tok/s @256k (flat at depth),
|
| 6 |
# 256k prefill no-wedge, ~200 tok/s aggregate @ concurrency 4.
|
| 7 |
#
|
|
|
|
| 74 |
NUM_SPECULATIVE_TOKENS: ${NUM_SPECULATIVE_TOKENS:-5}
|
| 75 |
MAX_CUDAGRAPH_CAPTURE_SIZE: ${MAX_CUDAGRAPH_CAPTURE_SIZE:-}
|
| 76 |
SPEC_CONFIG: ${SPEC_CONFIG:-}
|
| 77 |
+
HF_OVERRIDES: '{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS","use_index_cache":true}'
|
| 78 |
entrypoint:
|
| 79 |
- bash
|
| 80 |
- -lc
|