Add FP8 (dynamic) quantization for Nanbeige4.2-3B-DSpark

README.md

@@ -0,0 +1,68 @@
+---
+license: other
+base_model: Nanbeige/Nanbeige4.2-3B-DSpark
+base_model_relation: quantized
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- fp8
+- compressed-tensors
+- vllm
+- quantized
+quantized_by: liodon-ai
+---
+
+# Nanbeige4.2-3B-DSpark — FP8 (dynamic)
+
+FP8 quantization of [Nanbeige/Nanbeige4.2-3B-DSpark](https://huggingface.co/Nanbeige/Nanbeige4.2-3B-DSpark), published by [Liodon AI](https://huggingface.co/liodon-ai).
+
+Quantized with [llm-compressor](https://github.com/vllm-project/llm-compressor) using the
+`FP8_DYNAMIC` scheme: weights are cast to FP8 (E4M3) per-channel ahead of time, activations are
+quantized to FP8 dynamically per-token at inference time. No calibration dataset is needed for this
+scheme, so the quantized weights are numerically just a direct cast of the original — no calibration-set
+bias to worry about. `lm_head` is left unquantized (standard practice — negligible size, disproportionate
+quality impact if quantized).
+
+Original size: 1.7 GB → Quantized: 2.8 GB.
+
+## Quick Start
+
+**vLLM**
+```bash
+vllm serve liodon-ai/Nanbeige4.2-3B-DSpark-FP8
+```
+
+**Text Generation Inference (TGI)**
+```bash
+docker run --gpus all -p 8080:80 ghcr.io/huggingface/text-generation-inference \
+    --model-id liodon-ai/Nanbeige4.2-3B-DSpark-FP8
+```
+
+**SGLang**
+```bash
+python -m sglang.launch_server --model-path liodon-ai/Nanbeige4.2-3B-DSpark-FP8
+```
+
+FP8 execution requires an NVIDIA GPU with compute capability ≥ 8.9 (Ada/Hopper/Blackwell — RTX 40-series,
+L4/L40S, H100/H200, B100/B200/GB10). On older GPUs, vLLM/TGI will dequantize to run, which loses the
+speed/memory benefit.
+
+## Source
+
+- **Model**: [Nanbeige/Nanbeige4.2-3B-DSpark](https://huggingface.co/Nanbeige/Nanbeige4.2-3B-DSpark)
+- **License**: other
+
+## Citation
+
+```bibtex
+@misc{liodonai_nanbeige4_2_3b_dspark_fp8,
+  title        = {Nanbeige4.2-3B-DSpark — FP8},
+  author       = {{Liodon AI}},
+  year         = {2026},
+  howpublished = {\url{https://huggingface.co/liodon-ai/Nanbeige4.2-3B-DSpark-FP8}},
+  note         = {FP8 (dynamic) quantization of Nanbeige/Nanbeige4.2-3B-DSpark}
+}
+```
+
+---
+*Quantized by [Liodon AI](https://huggingface.co/liodon-ai)*

config.json

@@ -0,0 +1,114 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModel": "dspark.DSparkDraftModel"
+  },
+  "block_size": 7,
+  "bos_token_id": 166100,
+  "dflash_config": {
+    "attention_mode": "gqa",
+    "confidence_head_alpha": 1.0,
+    "confidence_head_with_markov": true,
+    "enable_confidence_head": true,
+    "markov_head_type": "vanilla",
+    "markov_rank": 256,
+    "mask_token_id": 166143,
+    "projector_type": "dspark",
+    "target_layer_ids": [
+      1,
+      10,
+      21,
+      32,
+      40
+    ]
+  },
+  "dtype": "bfloat16",
+  "eos_token_id": 166101,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 10752,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "markov_head_type": "vanilla",
+  "markov_rank": 256,
+  "max_position_embeddings": 262144,
+  "max_window_layers": 5,
+  "model_type": "qwen3",
+  "num_attention_heads": 48,
+  "num_hidden_layers": 5,
+  "num_key_value_heads": 8,
+  "num_target_layers": 44,
+  "pad_token_id": 0,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "float-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "token",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "channel",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "float-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 70000000,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.14.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 166144
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 166100,
+  "eos_token_id": 166101,
+  "pad_token_id": 0,
+  "transformers_version": "5.14.1",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f5d5c115114a5f1ede1f97f19ba001756cc4e709a9d731c60caa4f0c9a5a60
+size 2757670040

recipe.yaml

@@ -0,0 +1,8 @@
+default_stage:
+  default_modifiers:
+    QuantizationModifier:
+      targets: [Linear]
+      ignore: [lm_head]
+      scheme: FP8_DYNAMIC
+      bypass_divisibility_checks: false
+      requires_calibration_data: false

tokenizer.json

@@ -0,0 +1,84 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": true,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|endoftext|>": 0
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}