Publish Swift HyperQwen collection with performance and quality comparisons

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,233 @@
+Swift Open License v1.0
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by this document.
+
+"Licensor" shall mean UkisAI.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation source,
+configuration files, and model weights in an unquantized, trainable format.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object
+code, generated documentation, quantized or otherwise converted model weights,
+and conversions to other media types or file formats.
+
+"Base Model" shall mean the Qwen3.8-27B model, Copyright 2026 Alibaba Cloud,
+made available at https://huggingface.co/Qwen/Qwen3.8-27B, including its
+weights, configuration, tokenizer, and chat template, in any form.
+
+"Base Model License" shall mean the Apache License, Version 2.0, under which
+the Base Model is made available. A copy is distributed with the Work in the
+file LICENSE-APACHE-2.0.
+
+"Swift Contribution" shall mean the modifications to the Base Model authored
+by Licensor, in any form, including without limitation trained weight adapters,
+weight deltas, model weights to the extent they differ from the Base Model, and
+any configuration, documentation, and evaluation materials created by Licensor
+and distributed with the Work.
+
+"Work" shall mean the Swift Contribution together with, to the extent of
+Licensor's rights therein, the Derivative Work of the Base Model made available
+by Licensor under this License (as indicated by a copyright notice that is
+included in or attached to the work), in any format made available by Licensor.
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner. For the purposes
+of this definition, "submitted" means any form of electronic, verbal, or
+written communication sent to the Licensor or its representatives, including
+but not limited to communication on electronic mailing lists, source code
+control systems, and issue tracking systems that are managed by, or on behalf
+of, the Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise designated in
+writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+"Commercial Use" shall mean any use of the Work or a Derivative Work for direct
+or indirect commercial advantage or monetary compensation.
+
+"Qualified Non-Profit Organization" shall mean a Legal Entity that is organized
+and operated exclusively for religious, charitable, scientific, testing for
+public safety, literary, or educational purposes, and which is exempt from
+federal income tax under Section 501(c)(3) of the United States Internal
+Revenue Code of 1986, as amended, or any equivalent non-profit or charitable
+organization in a foreign jurisdiction.
+
+"Non-Commercial or Research Purposes" shall mean purposes that do not involve
+any use of the Work or a Derivative Work for Commercial Use.
+
+"Threshold" shall mean gross revenue of one million United States dollars
+(US$1,000,000) or more, measured over the most recently completed fiscal year
+of You together with every Legal Entity that controls, is controlled by, or is
+under common control with You.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+License, including the Commercial Use limitation set forth in Section 5, each
+Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable copyright license to reproduce, prepare
+Derivative Works of, publicly display, publicly perform, sublicense, and
+distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this
+License, including the Commercial Use limitation set forth in Section 5, each
+Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section) patent
+license to make, have made, use, offer to sell, sell, import, and otherwise
+transfer the Work, where such license applies only to those patent claims
+licensable by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s) with the Work
+to which such Contribution(s) was submitted. If You institute patent litigation
+against any entity (including a cross-claim or counterclaim in a lawsuit)
+alleging that the Work or a Contribution incorporated within the Work
+constitutes direct or contributory patent infringement, then any patent
+licenses granted to You under this License for that Work shall terminate as of
+the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+Derivative Works thereof in any medium, with or without modifications, and in
+Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a copy
+of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating that
+You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You
+distribute, all copyright, patent, trademark, and attribution notices from the
+Source form of the Work, excluding those notices that do not pertain to any
+part of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution, then
+any Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License; and
+
+(e) If the copy You distribute contains any portion of the Base Model
+(including merged, quantized, or otherwise converted weights that incorporate
+the Base Model), You must also give recipients a copy of the Base Model License
+and must comply with the Base Model License with respect to the Base Model.
+
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a
+whole, provided Your use, reproduction, and distribution of the Work otherwise
+complies with the conditions stated in this License, and provided that Section
+5 continues to apply to the Swift Contribution contained in any such Derivative
+Works.
+
+5. Commercial Use Limitation.
+
+(a) The rights granted under this License for Commercial Use are conditioned
+upon You or Your Legal Entity not exceeding the Threshold.
+
+(b) Any Commercial Use of the Work or a Derivative Work by a Legal Entity that
+exceeds the Threshold is not licensed under this License.
+
+(c) The Threshold shall not apply to a Qualified Non-Profit Organization's use
+of the Work or a Derivative Work for Non-Commercial or Research Purposes.
+
+(d) A Legal Entity that exceeds the Threshold may obtain a separate written
+license for Commercial Use from Licensor (the "Swift Enterprise License").
+Contact: https://ukisai.com/contact.
+
+6. Base Model Rights. The Work incorporates the Base Model. Nothing in this
+License limits, restricts, conditions, or modifies any rights You have in the
+Base Model under the Base Model License, and the Base Model remains available
+to You from its licensor under the Base Model License. Sections 5 and 12 of
+this License apply solely to the Swift Contribution and to the Work or any
+Derivative Work to the extent it contains, incorporates, or is derived from the
+Swift Contribution.
+
+7. Submission of Contributions. Unless You explicitly state otherwise, any
+Contribution intentionally submitted for inclusion in the Work by You to the
+Licensor shall be under the terms and conditions of this License, without any
+additional terms or conditions. Notwithstanding the above, nothing herein shall
+supersede or modify the terms of any separate license agreement you may have
+executed with Licensor regarding such Contributions.
+
+8. Trademarks. This License does not grant permission to use the trade names,
+trademarks, service marks, or product names of the Licensor (including "UkisAI"
+and "Swift"), except for the reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+9. Disclaimer of Warranty. Unless required by applicable law or agreed to in
+writing, Licensor provides the Work (and each Contributor provides its
+Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied, including, without limitation, any warranties
+or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any risks
+associated with Your exercise of permissions under this License.
+
+10. Limitation of Liability. In no event and under no legal theory, whether in
+tort (including negligence), contract, or otherwise, unless required by
+applicable law (such as deliberate and grossly negligent acts) or agreed to in
+writing, shall any Contributor be liable to You for damages, including any
+direct, indirect, special, incidental, or consequential damages of any
+character arising as a result of this License or out of the use or inability
+to use the Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all other commercial
+damages or losses), even if such Contributor has been advised of the
+possibility of such damages.
+
+11. Accepting Warranty or Additional Liability. While redistributing the Work
+or Derivative Works thereof, You may choose to offer, and charge a fee for,
+acceptance of support, warranty, indemnity, or other liability obligations
+and/or rights consistent with this License. However, in accepting such
+obligations, You may act only on Your own behalf and on Your sole
+responsibility, not on behalf of any other Contributor, and only if You agree
+to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+12. Termination. This License will terminate automatically and immediately if
+You fail to comply with any of its terms and conditions. Upon termination, You
+must cease all use of the Swift Contribution and of any Work or Derivative
+Works containing it, and delete all copies in Your possession. Termination does
+not affect Your rights in the Base Model under the Base Model License.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: Notice for redistributors (quantizations, conversions, merges).
+
+   Copyright 2026 UkisAI. Swift Contribution licensed under the Swift Open
+   License v1.0 (https://huggingface.co/ukisai/Swift-Qwen3.8-27b/blob/main/LICENSE).
+   Derivative of Qwen3.8-27B, Copyright 2026 Alibaba Cloud, Apache License 2.0.

LICENSE-APACHE-2.0

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,23 @@
+Swift-Qwen3.8-27B
+Copyright 2026 UkisAI
+
+UkisAI's contribution (the "Swift Contribution") is licensed under the
+Swift Open License v1.0. See LICENSE.
+
+This model is a Derivative Work of Qwen3.8-27B
+  https://huggingface.co/Qwen/Qwen3.8-27B
+  Copyright 2026 Alibaba Cloud
+  Licensed under the Apache License, Version 2.0. See LICENSE-APACHE-2.0.
+
+Changes made by UkisAI (Apache License 2.0, Section 4(b) change notice):
+  - model-*.safetensors, model.safetensors.index.json: model weights were
+    fine-tuned by UkisAI (LoRA adapter trained by UkisAI and merged into the
+    Base Model weights).
+  - generation_config.json: added "min_p": 0 and "repetition_penalty": 1.0.
+  - README.md: replaced. ukisai-banner.png and swift-speed-demo.mp4 added.
+  - All other files (config.json, chat_template.jinja, tokenizer.json,
+    tokenizer_config.json, vocab.json, merges.txt, preprocessor_config.json,
+    video_preprocessor_config.json) are unmodified from Qwen3.8-27B and
+    remain under the Apache License, Version 2.0.
+
+HyperQwen conversion by daavidhauser: INT8 embeddings, main output head and MTP linear weights; reference MTP draft shortlist. Upstream AWQ body retained.

QUANTIZATION_MANIFEST.json

@@ -0,0 +1,13 @@
+{
+  "source_repo": "TheUnderscore/Swift-Qwen3.8-27b-W4A16-AWQ",
+  "source_revision": "6ced337b9c99adddf9871ff84abc78a7f1acafad",
+  "revision_evidence": "Preserved Hugging Face local download metadata",
+  "license_source_repo": "ukisai/Swift-Qwen3.8-27b",
+  "license_source_revision": "6bc57e4eca31ee61d4e92a631978655a78bfa465",
+  "variant": "HyperQwen INT8 heads",
+  "body": "Preserved upstream asymmetric AWQ INT4 group128",
+  "embedding_bits": 8,
+  "lm_head_bits": 8,
+  "mtp_bits": 8,
+  "draft_vocabulary": "HyperQwen reference shortlist"
+}

README.md

@@ -0,0 +1,75 @@
+---
+license: other
+license_name: swift-open-license-1.0
+license_link: LICENSE
+base_model: ukisai/Swift-Qwen3.8-27b
+base_model_relation: quantized
+library_name: vllm
+pipeline_tag: text-generation
+tags:
+- compressed-tensors
+- awq
+- hyperqwen
+- efficient-thinking
+- int8-heads
+---
+
+Model card written by GPT-6 Astra: 
+
+# Swift 1.0 HyperQwen
+
+Swift 1.0 adapted for **HyperQwen serving on a single RTX 3090 24 GB**, with FP8 KV cache, MTP speculative decoding, and a **150k-token configured context**.
+
+In our 630-task comparison, this model used **42% fewer output tokens** and achieved **39% lower average request completion time** than the tested Qwen fast checkpoint. Swift's efficient-reasoning training reduces how much text the model generates; HyperQwen provides the serving runtime. Both matter for getting tasks finished quickly.
+
+[Browse all three Swift HyperQwen variants](https://huggingface.co/collections/daavidhauser/swift-for-hyperqwen-rtx-3090-benchmarks-6abac232566f58d5e7c5a046).
+
+## Performance
+
+| Measurement | Qwen fast | Swift 1.0 | Swift 1.5 INT8 heads | Swift 1.5 fast |
+|---|---:|---:|---:|---:|
+| **Average request time ↓** | **108.1 s** | **66.2 s** | **72.2 s** | **68.2 s** |
+| **Average output tokens/task ↓** | **8,985** | **5,245** | **5,751** | **5,669** |
+| Median decode tokens/s ↑ | 112.1 | 105.9 | 104.0 | 107.2 |
+| Total output tokens, 630 tasks | 5.66M | 3.30M | 3.62M | 3.57M |
+
+Request times come from the full quality suite at **two concurrent requests**, including failures. They are not single-user latency measurements. Output counts include reasoning. TPS is a separate **single-request** test: four short prompts repeated twice, with 512 greedy output tokens each. The configured context is 150k; these TPS figures are not measured at 150k occupied context.
+
+## Quality
+
+| Test | Qwen fast | Swift 1.0 | Swift 1.5 INT8 heads | Swift 1.5 fast |
+|---|---:|---:|---:|---:|
+| **GSM8K — 200-question subset** | 97.5% | 98.0% | 98.0% | 97.5% |
+| **IFBench — 300 prompts, strict** | 74.0% | 73.3% | 73.7% | 72.3% |
+| **LiveCodeBench — 100-problem subset** | 90% | 89% | 89% | 91% |
+| **Custom tool-call/JSON checks — 30 tasks** | 29/30 | 28/30 | 30/30 | 30/30 |
+| Perplexity, English/Danish/Python ↓ | 8.143 | 8.215 | 8.252 | 8.318 |
+| Truncated answers, counted wrong | 2 | 1 | 1 | 0 |
+
+- **GSM8K:** the first 200 questions from the [test split](https://huggingface.co/datasets/openai/gsm8k), with thinking disabled.
+- **IFBench:** all 300 prompts in the pinned [IFBench test dataset](https://huggingface.co/datasets/allenai/IFBench_test), scored with the official strict prompt-level verifier.
+- **LiveCodeBench:** a frozen [v6-era dataset](https://huggingface.co/datasets/livecodebench/code_generation_lite) subset of Python stdin/stdout problems: 34 easy, 33 medium, 33 hard. Scored against supplied public/private tests with a custom judge; not a full official LiveCodeBench result.
+- **Tool-call/JSON checks:** 20 custom weather-tool tasks checking the function name, arguments, Celsius-to-Fahrenheit conversion and final JSON; plus 10 JSON inventory-filtering tasks. These are integration checks, not an external agent benchmark.
+- **Perplexity:** 32,646 scored tokens from English Wikipedia, Danish web text and Python source; lower is better.
+
+All models used the same serving settings and task budgets. Thinking tests used xhigh effort, temperature 1.0, top_p 0.95, top_k 20 and seed 15027; GSM8K/tool checks were greedy. Outputs were capped at 128,000 tokens per call. These are single-seed local results; small accuracy differences do not establish a universal ranking. Qwen fast is a quantized AutoRound reference, not BF16 Qwen. Swift 1.0 was the most token-efficient model on this task mix.
+
+## Changes from upstream
+
+The **upstream AWQ INT4 body is preserved**, and embeddings are converted to **INT8**. The main output head (`lm_head`) and MTP linear weights are converted to **INT8**. MTP uses HyperQwen's reference draft shortlist. This is the **INT8-head HyperQwen variant**, rather than the untouched upstream checkpoint. The target model retains its full vocabulary; the shortlist only limits speculative proposals. No additional fine-tuning is applied by this release.
+
+The upstream AWQ quantization is [TheUnderscore/Swift-Qwen3.8-27b-W4A16-AWQ](https://huggingface.co/TheUnderscore/Swift-Qwen3.8-27b-W4A16-AWQ). Swift's training is by [UkisAI](https://huggingface.co/ukisai); the serving runtime is [HyperQwen](https://github.com/syv-ai/HyperQwen). This is an independent conversion with local evaluation by daavidhauser.
+
+## Setup
+
+Requires the **patched HyperQwen runtime**, not stock vLLM or GGUF tools. From an installed HyperQwen checkout:
+
+```bash
+hf download daavidhauser/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen --local-dir models/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen
+MODEL="$PWD/models/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen" CTX=long MAX_LEN=150000 SPEC=mtp \
+  bash single-user/start_qwen.sh
+```
+
+See [RUNTIME.md](RUNTIME.md) for the pinned runtime, launcher snapshot, complete evaluated settings and installation notes. The checkpoint is already converted: do not requantize its heads. Vision weights are retained, but the evaluation is text-only. Multi-user batch settings were not benchmarked in this campaign.
+
+Detailed results and evaluation code are in [evaluation/](evaluation/). Quantization/source provenance is included with the model. The upstream [Swift Open License v1.0](LICENSE), [Apache 2.0 base-model license](LICENSE-APACHE-2.0), and [NOTICE](NOTICE) are retained.

RELEASE_MANIFEST.json

@@ -0,0 +1,433 @@
+{
+  "repo": "daavidhauser/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen",
+  "files": {
+    "LICENSE": {
+      "size": 13304,
+      "sha256": "915ffe920f90088d15986bb6f7fab02e08b9a167cfbde65b012741ad88010e4f"
+    },
+    "LICENSE-APACHE-2.0": {
+      "size": 11544,
+      "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
+    },
+    "NOTICE": {
+      "size": 1172,
+      "sha256": "2fcf73df7395f673cad9b4ed6ae0de2d6b012d8e7d62cd64a361d45ac3cb4101"
+    },
+    "QUANTIZATION_MANIFEST.json": {
+      "size": 543,
+      "sha256": "9646e189bb9cfb1f35467eece46b59faf73c5eb0cc9800993343947cd875594d"
+    },
+    "README.md": {
+      "size": 5569,
+      "sha256": "d150cd7359b9b1923664c39438ad1a662ce5312d994961e5c2cca8e4ec86d6e0"
+    },
+    "RUNTIME.md": {
+      "size": 2588,
+      "sha256": "2f34c2a1daa4d900f39b961ad6179de530df2c5f44d853e43400514dfe805200"
+    },
+    "chat_template.jinja": {
+      "size": 8952,
+      "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041"
+    },
+    "config.json": {
+      "size": 22357,
+      "sha256": "13bcc57aaa12c046cf379d0680f6a8dd144db48696c294798daae0d485a1e255"
+    },
+    "evaluation/code/swift15/checkpoint.py": {
+      "size": 3656,
+      "sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d"
+    },
+    "evaluation/code/swift15/code_runner.py": {
+      "size": 2144,
+      "sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9"
+    },
+    "evaluation/code/swift15/common.py": {
+      "size": 1439,
+      "sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c"
+    },
+    "evaluation/code/swift15/corpus.py": {
+      "size": 7800,
+      "sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050"
+    },
+    "evaluation/code/swift15/eval_data.py": {
+      "size": 8097,
+      "sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e"
+    },
+    "evaluation/code/swift15/evaluate.py": {
+      "size": 14113,
+      "sha256": "4d553cf10b42cbff0eeacfe72b3b6df87405aa2604fcbe19a5a0a78133cd18af"
+    },
+    "evaluation/code/swift15/generate.py": {
+      "size": 4412,
+      "sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089"
+    },
+    "evaluation/code/swift15/ifbench_score.py": {
+      "size": 876,
+      "sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d"
+    },
+    "evaluation/code/swift15/measure.py": {
+      "size": 4726,
+      "sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37"
+    },
+    "evaluation/code/swift15/quantize.py": {
+      "size": 8098,
+      "sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7"
+    },
+    "evaluation/code/swift15/reference.py": {
+      "size": 1786,
+      "sha256": "0433adf523a676c0cb30c5ad25b90bdc5a4565c2b69180e171aa4a4d59c15bf8"
+    },
+    "evaluation/code/swift15/release.py": {
+      "size": 7409,
+      "sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e"
+    },
+    "evaluation/code/swift15/report.py": {
+      "size": 4938,
+      "sha256": "873db1c8e86fb281b15f0b3c2e66adfb3cf5da652cf0139788acd00f8bd2ac02"
+    },
+    "evaluation/code/swift15/run.py": {
+      "size": 9281,
+      "sha256": "e5d42332192e5bafdde88ae4acc20802f5c5477e6447ff5165fb6b836aa78ea2"
+    },
+    "evaluation/code/swift15/runtime_compat.py": {
+      "size": 1885,
+      "sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6"
+    },
+    "evaluation/code/swift15/serve.py": {
+      "size": 6020,
+      "sha256": "c705174eda86e57ae1a51c0b2e7a93f9b0e1f37fa32217a9739b569f52d2d56d"
+    },
+    "evaluation/code/swift15/setup.py": {
+      "size": 2219,
+      "sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495"
+    },
+    "evaluation/code/swift15/smoke.py": {
+      "size": 3162,
+      "sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe"
+    },
+    "evaluation/code/swift15/test_workflow.py": {
+      "size": 3468,
+      "sha256": "8d2dcb057318d482530ef9bf7b4bcf0d59347b594531e0df18b81dbea0beaddd"
+    },
+    "evaluation/environment.json": {
+      "size": 3282,
+      "sha256": "2e9ac10516648b0c634eaca007d58456d241e0c1ef1b6734fa51671bf315a841"
+    },
+    "evaluation/manifest.json": {
+      "size": 2685,
+      "sha256": "82c1c6eaad6c06ad91e20cac5f9dae6b74ceaba0a3063c868a6a4313366e7948"
+    },
+    "evaluation/pilot-ids.json": {
+      "size": 354,
+      "sha256": "44cffc01ab88b366f20da82d3150c7de8eb1d7801cd1381570e32313f120da41"
+    },
+    "evaluation/qwen-fast/perplexity.json": {
+      "size": 12570,
+      "sha256": "9db1599421dc916b97602a8921dbd219b815a5a76fc7def41f508500eb10e7e7"
+    },
+    "evaluation/qwen-fast/server.json": {
+      "size": 1942,
+      "sha256": "b2988235fd03e4cd30366b5ca418ba5bb687c1ab58c1eb29e3e53c57612ac0ae"
+    },
+    "evaluation/qwen-fast/speed-c1.json": {
+      "size": 630,
+      "sha256": "3d70c2868a2f2832e6bf7562abd3fa4589389baf87e02dbd89c76b154e49238d"
+    },
+    "evaluation/qwen-fast-full/manifest.json": {
+      "size": 14395,
+      "sha256": "3de5a8af19d01a53594d7f29349e19bf90f1bbe8d935e82fe33aa0cf8325391b"
+    },
+    "evaluation/qwen-fast-full/summary.json": {
+      "size": 3134,
+      "sha256": "a2ffcd9ed991279b5639274596a5139b39553d407753adc48ff465e55fd2eaf6"
+    },
+    "evaluation/qwen-fast-pilot/manifest.json": {
+      "size": 2663,
+      "sha256": "f06f4539feaa46b92d559427cd13809ecf3e8b0fc44b15768e54392a0ce9c34f"
+    },
+    "evaluation/qwen-fast-pilot/summary.json": {
+      "size": 2979,
+      "sha256": "8a0ec26f335c5a0c98bac1db6bd852a5292b0b5d1bfcc6a218a0b9aa1ce9b1c7"
+    },
+    "evaluation/swift10/perplexity.json": {
+      "size": 12560,
+      "sha256": "fe393bd4962d43f9cef2a08ea40c0dd2f15e6695b87a09a20e184d448d941775"
+    },
+    "evaluation/swift10/server.json": {
+      "size": 2041,
+      "sha256": "d6004a26531972be5407c0f542ec0b708c931e21f67801e5206906d4cb148547"
+    },
+    "evaluation/swift10/speed-c1.json": {
+      "size": 632,
+      "sha256": "a82f761f6a6ca5446808b1e8b129fee4e5b3b733787e286a7829b23f6dd1722f"
+    },
+    "evaluation/swift10-full/manifest.json": {
+      "size": 14501,
+      "sha256": "6af95e530b920a84a3c5f5dde5b76bc2c791d0a4cc5b3043686bd6946cac4acf"
+    },
+    "evaluation/swift10-full/summary.json": {
+      "size": 3125,
+      "sha256": "1c431ee8cc16e36090152526f8553c3e76cb5f66e5ddd5c6df1f90c3b3426add"
+    },
+    "evaluation/swift10-pilot/manifest.json": {
+      "size": 2769,
+      "sha256": "83df9bd1a41e0d96f62039ad1941593e4cafc57c52f63851eeec4523048f4315"
+    },
+    "evaluation/swift10-pilot/summary.json": {
+      "size": 2981,
+      "sha256": "3f8aed7d263dc65abb0095bc3af73515052645886f5f0565e6c21e82afa27bf8"
+    },
+    "evaluation/swift15-baseline/perplexity.json": {
+      "size": 12553,
+      "sha256": "6f63e857421957c76b3ca290ec8c95a62fd4374e3839adc2cdcdca54f14786fa"
+    },
+    "evaluation/swift15-baseline/server.json": {
+      "size": 1937,
+      "sha256": "9063a7b84cd89e83672d673d5fc30ff563d01adebd1a705fcae70c33b5ba47b4"
+    },
+    "evaluation/swift15-baseline/speed-c1.json": {
+      "size": 629,
+      "sha256": "079f1c870da2484a43ec214ee2ced3723b2636c83eaed4291636e121b4e81e3a"
+    },
+    "evaluation/swift15-baseline-full/manifest.json": {
+      "size": 14389,
+      "sha256": "b122ac2835f4c868522aa6982e1c44e2090ae176b27293cd8a71040108751d18"
+    },
+    "evaluation/swift15-baseline-full/summary.json": {
+      "size": 3091,
+      "sha256": "cd0ec9d98535c6c32c2ae9f32798f81c0dae628ebda0d1d56f6005df85edb91b"
+    },
+    "evaluation/swift15-baseline-pilot/manifest.json": {
+      "size": 2657,
+      "sha256": "588d9a88f35aa9207934bde6cbce4e0956868287316f964585b784b9d4f7cec3"
+    },
+    "evaluation/swift15-baseline-pilot/summary.json": {
+      "size": 2982,
+      "sha256": "511e69269b8d98d64876ab7033e124fc1d8aca196b022fbd27b34f65867ac8ef"
+    },
+    "evaluation/swift15-fast/perplexity.json": {
+      "size": 12561,
+      "sha256": "3fa2ac71c6c24072036837c3ae0afcf96ce03e85a09b276874c75432aeb091a1"
+    },
+    "evaluation/swift15-fast/server.json": {
+      "size": 1940,
+      "sha256": "53099847622dbdd7416196bfd8a7591aca1adb64db92049cf2be6ffdf489536d"
+    },
+    "evaluation/swift15-fast/speed-c1.json": {
+      "size": 629,
+      "sha256": "88d292f00ba50773e80c0dbf6ba9a1e18d3413737221d2ff8c5a969749747108"
+    },
+    "evaluation/swift15-fast-full/manifest.json": {
+      "size": 14393,
+      "sha256": "d242d0bd7f947c37a74430190ce4a1309c2ae8e2f5ba76313e1ba056044aa134"
+    },
+    "evaluation/swift15-fast-full/summary.json": {
+      "size": 2969,
+      "sha256": "3ffd7dc76fa4105089688731c6a8365519f80567acbca3b1bc15b64e93de7ab3"
+    },
+    "evaluation/swift15-fast-pilot/manifest.json": {
+      "size": 2661,
+      "sha256": "13203d3b56258d785ea74ddd3f89322edb6c58245f7d4ecf60756a7af66d4f60"
+    },
+    "evaluation/swift15-fast-pilot/summary.json": {
+      "size": 2979,
+      "sha256": "fe44f8972f40cf5236ca93d1ee7362cfea238a03e58f09c6e033312820fc8d89"
+    },
+    "generation_config.json": {
+      "size": 221,
+      "sha256": "0b65d40c797e19ad9bbd38538f1b45155b2832ffaffa7a69df96ddf57213fc9b"
+    },
+    "hyperqwen_provenance/upstream-README.md": {
+      "size": 3468,
+      "sha256": "17d82fcb87967b726b88c9e3cb47e0ef94815aae2abf26306ab9d6a053c86e37"
+    },
+    "merges.txt": {
+      "size": 3353259,
+      "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
+    },
+    "model-00001-of-00007.safetensors": {
+      "size": 2212761608,
+      "sha256": "6872c54bf590710482a55f2a4fc7648a55242e819785ea4f64a91064ee933ba4"
+    },
+    "model-00002-of-00007.safetensors": {
+      "size": 2186190664,
+      "sha256": "a44afcd2154f6bac8141a8af48ba7d4788584544a088da6a3112085d695acbfe"
+    },
+    "model-00003-of-00007.safetensors": {
+      "size": 2179699752,
+      "sha256": "07b0a5737261377859be9cb4fa19b0a0dbadb5b158bea0179367da9953de947b"
+    },
+    "model-00004-of-00007.safetensors": {
+      "size": 2179699760,
+      "sha256": "bf91ce93aa363b88392fbe32c4f785222de13c06e376768ed56ea60330a9b67e"
+    },
+    "model-00005-of-00007.safetensors": {
+      "size": 2179699760,
+      "sha256": "c3deb171898650ccc806f4a2e21c1aee5cea8efdce8fe149ab5f0bfc9049a83f"
+    },
+    "model-00006-of-00007.safetensors": {
+      "size": 2186211856,
+      "sha256": "007e4ccf91e5310b19a335be197342daffbe5bad46fc92dc4a0d2d37abb1c5b4"
+    },
+    "model-00007-of-00007.safetensors": {
+      "size": 3071134144,
+      "sha256": "a942b481417c1ecd24802576969ac66dd003d6bb1a734a1fde438e45e2e1bcb1"
+    },
+    "model-nonquant.safetensors": {
+      "size": 431364472,
+      "sha256": "9710e90c273e7cd36d22e8cdb635a81f8d78da5c79dd1bd217445137ece29b9d"
+    },
+    "model.safetensors.index.json": {
+      "size": 243539,
+      "sha256": "30bde9db796be88667575f52843af2a23538008cadfeb7d58cd1fd2201bd07b6"
+    },
+    "model_extra_tensors.safetensors": {
+      "size": 212992352,
+      "sha256": "e2c1a453b550d9e3c9d5c8df5c6fad58255128acdef570d493a5f376c2123e43"
+    },
+    "mtp_draft_vocab_ids.pt": {
+      "size": 329341,
+      "sha256": "8af9028616e41932e0c949cd5364d2da0a69d15375af9bb404cab1e4d723f581"
+    },
+    "preprocessor_config.json": {
+      "size": 390,
+      "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "runtime/BASE_REVISION.txt": {
+      "size": 41,
+      "sha256": "c1f6fbc4227edb7daa83504c936afc8e23d8e31e8b62ff375fbbb5e29e81e546"
+    },
+    "runtime/LICENSE": {
+      "size": 11358,
+      "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
+    },
+    "runtime/batch/start_qwen.sh": {
+      "size": 11677,
+      "sha256": "ac6a84b455317537a230bc190ccc06dda169dcdbdeee4bebeb51e76032972944"
+    },
+    "runtime/patches/dflash2-backport.patch": {
+      "size": 40396,
+      "sha256": "2e937ef748c1942ab04ace05de884e7c24041d34b23e5be8428cac6939ac6875"
+    },
+    "runtime/patches/dflash2-lookup-drafting.patch": {
+      "size": 46770,
+      "sha256": "5df09ef03b592d4a2c3b47dd8d2dbf8862fa7383d68aef3dd6ef1d97d29ce196"
+    },
+    "runtime/patches/dflash2-ngram-chains.patch": {
+      "size": 19037,
+      "sha256": "555b5a75d9023c99b2cd17634ac5b1dc8505f9275d857b1eba08c2d9cbf2df6e"
+    },
+    "runtime/patches/dflash2-prewarm.patch": {
+      "size": 5181,
+      "sha256": "e1e8012fa2c304c6948c19fc0071332f80eebcff802b402255445a30d553ab92"
+    },
+    "runtime/patches/hybrid-kv-groups-v2-cudagraph.patch": {
+      "size": 6306,
+      "sha256": "143825dd9744ab81dabcb438ec0bf974b6e0be8d0d7b0ffbbdfbf7c16a549664"
+    },
+    "runtime/patches/hybrid-sw-block-promote.patch": {
+      "size": 7828,
+      "sha256": "10876ac706546e74fe74b8964b90576a7b36f90a2f35a856997c558d7817f154"
+    },
+    "runtime/patches/int4-kv-per-token-head.patch": {
+      "size": 7038,
+      "sha256": "c03ff10c8c997b355f04fc521827ee2fa99c332ce4127cc2d878b749b79ca3d9"
+    },
+    "runtime/patches/mamba-align-checkpoint-order.patch": {
+      "size": 12020,
+      "sha256": "515d9bf76e860c95d832d615bdab4a97b71e2b0412b4ccc2445f11535ddddf45"
+    },
+    "runtime/patches/marlin-int8-layer-select.patch": {
+      "size": 3048,
+      "sha256": "833405e2ed2916529eb20184c38243c843378935522b74bab8c39707bf3ee800"
+    },
+    "runtime/patches/marlin-int8-negative-scales.patch": {
+      "size": 2926,
+      "sha256": "4cb8a064c706cc62dc77224479898da58864fbfff6b65012d90aa8326035300d"
+    },
+    "runtime/patches/marlin-repack-staged-sm80.patch": {
+      "size": 6841,
+      "sha256": "1588e2f10b5f82194d449483e766c0b39c84ba522d1623d39caa9502040f1fab"
+    },
+    "runtime/patches/marlin-tune-table.patch": {
+      "size": 4422,
+      "sha256": "1cac17f12e4ce389b0cb0fc733ceceb0a8cf60c02696ee378dc1d8d0c1ef8914"
+    },
+    "runtime/patches/offload-dflash-eagle-groups.patch": {
+      "size": 5406,
+      "sha256": "f2791af64d8066b31e4250d866c4322f45670d75522aff221307281c97c73156"
+    },
+    "runtime/patches/offload-wsl2-devptr.patch": {
+      "size": 4778,
+      "sha256": "8c6f9e3e5723571d1f33925794435e55062b684b2706d9284d1c5b94778d34db"
+    },
+    "runtime/patches/qwen3_5-embed-quant.patch": {
+      "size": 1395,
+      "sha256": "0a1b9ca06798c1aef582995de5a0beb3ad9a22a54cdbd2361986563a9c7a980e"
+    },
+    "runtime/patches/qwen3_5-mtp-draft-vocab.patch": {
+      "size": 3943,
+      "sha256": "292ef662d17bcc10556b787d5bb5f2cd12d3d3fc1f3bd2fe982485ce0d916298"
+    },
+    "runtime/patches/sampler-small-topk-fast-softmax.patch": {
+      "size": 13679,
+      "sha256": "8828646ce1916c065282529d9c1bb52064f668397d8e1f33652aef2f308292b6"
+    },
+    "runtime/patches/spec-decode-attn.patch": {
+      "size": 15424,
+      "sha256": "007791047a1d60143f8e3fe4e0a56dabadfc0134cb288847338b76ba7d1f9fe7"
+    },
+    "runtime/patches/spec-decode-int4-kv-mq3d.patch": {
+      "size": 3876,
+      "sha256": "b93b186deba1513ad9c4803ee2c574c923b21ba5e88e23bca8b8594b6a6b20a0"
+    },
+    "runtime/patches/spec-decode-int8-kv.patch": {
+      "size": 11915,
+      "sha256": "3cfd31304353237af5277c861dc2b43ad33d0f9bff2700594300eff150f64382"
+    },
+    "runtime/patches/spec-sampler-prewarm.patch": {
+      "size": 4516,
+      "sha256": "18ed9608baa5a09ed2ccbb214d62763da3ce7bcf09c6aad9003778ea4e56d18f"
+    },
+    "runtime/patches/speed-knobs-envs.patch": {
+      "size": 2300,
+      "sha256": "841ec93021b1b90f90313a1880a791bbfbdb79cbe34d9b99fdb4c3a7e52ed7c0"
+    },
+    "runtime/patches/triton-prefill-attn-int8.patch": {
+      "size": 15661,
+      "sha256": "6bd36db0226dce7b92ebd1231b6f9718a25da4a933ab4b2a6a8be77860364fae"
+    },
+    "runtime/patches/vision-tower-cpu-offload.patch": {
+      "size": 8345,
+      "sha256": "81dff64a1177058783dcf7d4e8552f1547f74fa8a68019e349dacdb9d66e968e"
+    },
+    "runtime/patches/vllm-pr50021-gdn-spec-bounds.patch": {
+      "size": 8886,
+      "sha256": "cd6e00270fa28e37a8c7ad11f965662f4153b2fcaf840f2c4043710e7b078655"
+    },
+    "runtime/patches/xgrammar-spec-terminated.patch": {
+      "size": 3993,
+      "sha256": "37589b9a45d5ece37cc16e82b195362719011e52ea31b2e70cdc89845c825040"
+    },
+    "runtime/single-user/start_qwen.sh": {
+      "size": 42439,
+      "sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8"
+    },
+    "tokenizer.json": {
+      "size": 19989325,
+      "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523"
+    },
+    "tokenizer_config.json": {
+      "size": 1124,
+      "sha256": "66e427c470fe580fe8c7b5725d857af23d8417e37fae62667ec698306a19987b"
+    },
+    "video_preprocessor_config.json": {
+      "size": 385,
+      "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
+    },
+    "vocab.json": {
+      "size": 6722759,
+      "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
+    }
+  }
+}

RUNTIME.md

@@ -0,0 +1,48 @@
+# HyperQwen runtime
+
+This checkpoint requires the patched HyperQwen vLLM runtime. It is not a GGUF.
+Tested HyperQwen base revision: `253c76aea0a240bf7cb2bd3ed92b672c0f258d0d`. Exact local launcher snapshots and
+patch files are in `runtime/`; calibration/evaluation scripts are included separately.
+
+Tested packages: vLLM 0.27.1, PyTorch 2.13.0, Transformers 5.15.0,
+compressed-tensors 0.17.0, safetensors 0.8.0. Follow the
+[pinned HyperQwen setup instructions](https://github.com/syv-ai/HyperQwen/blob/253c76aea0a240bf7cb2bd3ed92b672c0f258d0d/README.md#setup)
+for the compiler, attention libraries, and patch installation. Use the supplied
+`runtime/patches/` set when applying vLLM patches, and the supplied launcher for
+the final serve command. Do not requantize this already prepared model.
+An independent clean-machine installation has not been tested.
+
+## Download and launch from an installed HyperQwen checkout
+
+```bash
+hf download daavidhauser/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen \
+  --local-dir models/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen
+
+# In the HyperQwen checkout; use the launcher snapshot accompanying the model.
+cp models/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen/runtime/single-user/start_qwen.sh single-user/start_qwen.sh
+
+MODEL="$PWD/models/Swift-1.0-Qwen3.8-27B-W4A16-HyperQwen" \
+CTX=long MAX_LEN=150000 SPEC=mtp DRAFT_TOKENS=3 \
+PREFIX_CACHE=1 TOOLS=1 VISION=1 VISION_OFFLOAD=1 \
+GPU_UTIL=0.93 MAX_SEQS=8 API_SERVERS=1 HOST=127.0.0.1 PORT=18020 \
+VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS=1 FLASHINFER_DISABLE_VERSION_CHECK=1 \
+EXTRA_ARGS='--limit-mm-per-prompt {"image":{"count":10}}' \
+  bash single-user/start_qwen.sh
+```
+
+The tested run did not enable INT8 activation quantization. Remove conflicting
+INT8_ACT, PREFILL_ATTN, or other performance overrides from your local environment
+and `.env` when reproducing it. The launcher uses FP8 KV and FP16 recurrent state
+in this profile. Test memory on your own stack before assuming 150k capacity.
+Use model name `qwen3.8-27b` in API requests. Authentication follows HyperQwen's
+normal `api_key.txt` / `VLLM_API_KEY` configuration.
+
+## Evaluation requests
+
+Thinking: temperature 1.0, top_p 0.95, top_k 20, min_p 0, presence_penalty 0,
+repetition_penalty 1.0, seed 15027, enable_thinking true, reasoning_effort xhigh.
+Output limit: 128000 tokens. Nonthinking GSM8K/tool tests were greedy.
+The quality suite used two concurrent requests; the speed test used one.
+
+The final published campaign validates this single-user configuration only.
+Separate batch launchers/INT8 activation settings are not benchmarked by these results.

chat_template.jinja

@@ -0,0 +1,170 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is undefined or enable_thinking is true %}
+    {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+    {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+    {%- endif %}
+    {%- if resolved_reasoning_effort == 'xhigh' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+    {%- elif resolved_reasoning_effort == 'low' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+    {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {%- if reasoning_instructions %}
+        {{- reasoning_instructions + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '')  + content + '<|im_end|>\n' }}
+        {%- elif reasoning_instructions %}
+            {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+        {%- endif %}
+    {%- elif reasoning_instructions %}
+        {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,546 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "image_token_id": 248056,
+  "language_model_only": false,
+  "model_type": "qwen3_5",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": 248044,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "initializer_range": 0.02,
+    "intermediate_size": 17408,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 48,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 24,
+    "num_hidden_layers": 64,
+    "num_key_value_heads": 4,
+    "output_gate_type": "swish",
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 248320
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.8.0.dev0",
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 27,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_5",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 5120,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2,
+    "dtype": "bfloat16"
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 4,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": false,
+          "type": "int",
+          "zp_dtype": "torch.int8"
+        }
+      },
+      "group_1": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "re:.*lm_head$"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      },
+      "group_2": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "re:.*embed_tokens$"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      },
+      "group_3": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "re:^mtp\\..*"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.visual.blocks.0.attn.qkv",
+      "model.visual.blocks.0.attn.proj",
+      "model.visual.blocks.0.mlp.linear_fc1",
+      "model.visual.blocks.0.mlp.linear_fc2",
+      "model.visual.blocks.1.attn.qkv",
+      "model.visual.blocks.1.attn.proj",
+      "model.visual.blocks.1.mlp.linear_fc1",
+      "model.visual.blocks.1.mlp.linear_fc2",
+      "model.visual.blocks.2.attn.qkv",
+      "model.visual.blocks.2.attn.proj",
+      "model.visual.blocks.2.mlp.linear_fc1",
+      "model.visual.blocks.2.mlp.linear_fc2",
+      "model.visual.blocks.3.attn.qkv",
+      "model.visual.blocks.3.attn.proj",
+      "model.visual.blocks.3.mlp.linear_fc1",
+      "model.visual.blocks.3.mlp.linear_fc2",
+      "model.visual.blocks.4.attn.qkv",
+      "model.visual.blocks.4.attn.proj",
+      "model.visual.blocks.4.mlp.linear_fc1",
+      "model.visual.blocks.4.mlp.linear_fc2",
+      "model.visual.blocks.5.attn.qkv",
+      "model.visual.blocks.5.attn.proj",
+      "model.visual.blocks.5.mlp.linear_fc1",
+      "model.visual.blocks.5.mlp.linear_fc2",
+      "model.visual.blocks.6.attn.qkv",
+      "model.visual.blocks.6.attn.proj",
+      "model.visual.blocks.6.mlp.linear_fc1",
+      "model.visual.blocks.6.mlp.linear_fc2",
+      "model.visual.blocks.7.attn.qkv",
+      "model.visual.blocks.7.attn.proj",
+      "model.visual.blocks.7.mlp.linear_fc1",
+      "model.visual.blocks.7.mlp.linear_fc2",
+      "model.visual.blocks.8.attn.qkv",
+      "model.visual.blocks.8.attn.proj",
+      "model.visual.blocks.8.mlp.linear_fc1",
+      "model.visual.blocks.8.mlp.linear_fc2",
+      "model.visual.blocks.9.attn.qkv",
+      "model.visual.blocks.9.attn.proj",
+      "model.visual.blocks.9.mlp.linear_fc1",
+      "model.visual.blocks.9.mlp.linear_fc2",
+      "model.visual.blocks.10.attn.qkv",
+      "model.visual.blocks.10.attn.proj",
+      "model.visual.blocks.10.mlp.linear_fc1",
+      "model.visual.blocks.10.mlp.linear_fc2",
+      "model.visual.blocks.11.attn.qkv",
+      "model.visual.blocks.11.attn.proj",
+      "model.visual.blocks.11.mlp.linear_fc1",
+      "model.visual.blocks.11.mlp.linear_fc2",
+      "model.visual.blocks.12.attn.qkv",
+      "model.visual.blocks.12.attn.proj",
+      "model.visual.blocks.12.mlp.linear_fc1",
+      "model.visual.blocks.12.mlp.linear_fc2",
+      "model.visual.blocks.13.attn.qkv",
+      "model.visual.blocks.13.attn.proj",
+      "model.visual.blocks.13.mlp.linear_fc1",
+      "model.visual.blocks.13.mlp.linear_fc2",
+      "model.visual.blocks.14.attn.qkv",
+      "model.visual.blocks.14.attn.proj",
+      "model.visual.blocks.14.mlp.linear_fc1",
+      "model.visual.blocks.14.mlp.linear_fc2",
+      "model.visual.blocks.15.attn.qkv",
+      "model.visual.blocks.15.attn.proj",
+      "model.visual.blocks.15.mlp.linear_fc1",
+      "model.visual.blocks.15.mlp.linear_fc2",
+      "model.visual.blocks.16.attn.qkv",
+      "model.visual.blocks.16.attn.proj",
+      "model.visual.blocks.16.mlp.linear_fc1",
+      "model.visual.blocks.16.mlp.linear_fc2",
+      "model.visual.blocks.17.attn.qkv",
+      "model.visual.blocks.17.attn.proj",
+      "model.visual.blocks.17.mlp.linear_fc1",
+      "model.visual.blocks.17.mlp.linear_fc2",
+      "model.visual.blocks.18.attn.qkv",
+      "model.visual.blocks.18.attn.proj",
+      "model.visual.blocks.18.mlp.linear_fc1",
+      "model.visual.blocks.18.mlp.linear_fc2",
+      "model.visual.blocks.19.attn.qkv",
+      "model.visual.blocks.19.attn.proj",
+      "model.visual.blocks.19.mlp.linear_fc1",
+      "model.visual.blocks.19.mlp.linear_fc2",
+      "model.visual.blocks.20.attn.qkv",
+      "model.visual.blocks.20.attn.proj",
+      "model.visual.blocks.20.mlp.linear_fc1",
+      "model.visual.blocks.20.mlp.linear_fc2",
+      "model.visual.blocks.21.attn.qkv",
+      "model.visual.blocks.21.attn.proj",
+      "model.visual.blocks.21.mlp.linear_fc1",
+      "model.visual.blocks.21.mlp.linear_fc2",
+      "model.visual.blocks.22.attn.qkv",
+      "model.visual.blocks.22.attn.proj",
+      "model.visual.blocks.22.mlp.linear_fc1",
+      "model.visual.blocks.22.mlp.linear_fc2",
+      "model.visual.blocks.23.attn.qkv",
+      "model.visual.blocks.23.attn.proj",
+      "model.visual.blocks.23.mlp.linear_fc1",
+      "model.visual.blocks.23.mlp.linear_fc2",
+      "model.visual.blocks.24.attn.qkv",
+      "model.visual.blocks.24.attn.proj",
+      "model.visual.blocks.24.mlp.linear_fc1",
+      "model.visual.blocks.24.mlp.linear_fc2",
+      "model.visual.blocks.25.attn.qkv",
+      "model.visual.blocks.25.attn.proj",
+      "model.visual.blocks.25.mlp.linear_fc1",
+      "model.visual.blocks.25.mlp.linear_fc2",
+      "model.visual.blocks.26.attn.qkv",
+      "model.visual.blocks.26.attn.proj",
+      "model.visual.blocks.26.mlp.linear_fc1",
+      "model.visual.blocks.26.mlp.linear_fc2",
+      "model.visual.merger.linear_fc1",
+      "model.visual.merger.linear_fc2",
+      "model.language_model.layers.0.linear_attn",
+      "model.language_model.layers.0.linear_attn.norm",
+      "model.language_model.layers.0.linear_attn.in_proj_b",
+      "model.language_model.layers.0.linear_attn.in_proj_a",
+      "model.language_model.layers.1.linear_attn",
+      "model.language_model.layers.1.linear_attn.norm",
+      "model.language_model.layers.1.linear_attn.in_proj_b",
+      "model.language_model.layers.1.linear_attn.in_proj_a",
+      "model.language_model.layers.2.linear_attn",
+      "model.language_model.layers.2.linear_attn.norm",
+      "model.language_model.layers.2.linear_attn.in_proj_b",
+      "model.language_model.layers.2.linear_attn.in_proj_a",
+      "model.language_model.layers.4.linear_attn",
+      "model.language_model.layers.4.linear_attn.norm",
+      "model.language_model.layers.4.linear_attn.in_proj_b",
+      "model.language_model.layers.4.linear_attn.in_proj_a",
+      "model.language_model.layers.5.linear_attn",
+      "model.language_model.layers.5.linear_attn.norm",
+      "model.language_model.layers.5.linear_attn.in_proj_b",
+      "model.language_model.layers.5.linear_attn.in_proj_a",
+      "model.language_model.layers.6.linear_attn",
+      "model.language_model.layers.6.linear_attn.norm",
+      "model.language_model.layers.6.linear_attn.in_proj_b",
+      "model.language_model.layers.6.linear_attn.in_proj_a",
+      "model.language_model.layers.8.linear_attn",
+      "model.language_model.layers.8.linear_attn.norm",
+      "model.language_model.layers.8.linear_attn.in_proj_b",
+      "model.language_model.layers.8.linear_attn.in_proj_a",
+      "model.language_model.layers.9.linear_attn",
+      "model.language_model.layers.9.linear_attn.norm",
+      "model.language_model.layers.9.linear_attn.in_proj_b",
+      "model.language_model.layers.9.linear_attn.in_proj_a",
+      "model.language_model.layers.10.linear_attn",
+      "model.language_model.layers.10.linear_attn.norm",
+      "model.language_model.layers.10.linear_attn.in_proj_b",
+      "model.language_model.layers.10.linear_attn.in_proj_a",
+      "model.language_model.layers.12.linear_attn",
+      "model.language_model.layers.12.linear_attn.norm",
+      "model.language_model.layers.12.linear_attn.in_proj_b",
+      "model.language_model.layers.12.linear_attn.in_proj_a",
+      "model.language_model.layers.13.linear_attn",
+      "model.language_model.layers.13.linear_attn.norm",
+      "model.language_model.layers.13.linear_attn.in_proj_b",
+      "model.language_model.layers.13.linear_attn.in_proj_a",
+      "model.language_model.layers.14.linear_attn",
+      "model.language_model.layers.14.linear_attn.norm",
+      "model.language_model.layers.14.linear_attn.in_proj_b",
+      "model.language_model.layers.14.linear_attn.in_proj_a",
+      "model.language_model.layers.16.linear_attn",
+      "model.language_model.layers.16.linear_attn.norm",
+      "model.language_model.layers.16.linear_attn.in_proj_b",
+      "model.language_model.layers.16.linear_attn.in_proj_a",
+      "model.language_model.layers.17.linear_attn",
+      "model.language_model.layers.17.linear_attn.norm",
+      "model.language_model.layers.17.linear_attn.in_proj_b",
+      "model.language_model.layers.17.linear_attn.in_proj_a",
+      "model.language_model.layers.18.linear_attn",
+      "model.language_model.layers.18.linear_attn.norm",
+      "model.language_model.layers.18.linear_attn.in_proj_b",
+      "model.language_model.layers.18.linear_attn.in_proj_a",
+      "model.language_model.layers.20.linear_attn",
+      "model.language_model.layers.20.linear_attn.norm",
+      "model.language_model.layers.20.linear_attn.in_proj_b",
+      "model.language_model.layers.20.linear_attn.in_proj_a",
+      "model.language_model.layers.21.linear_attn",
+      "model.language_model.layers.21.linear_attn.norm",
+      "model.language_model.layers.21.linear_attn.in_proj_b",
+      "model.language_model.layers.21.linear_attn.in_proj_a",
+      "model.language_model.layers.22.linear_attn",
+      "model.language_model.layers.22.linear_attn.norm",
+      "model.language_model.layers.22.linear_attn.in_proj_b",
+      "model.language_model.layers.22.linear_attn.in_proj_a",
+      "model.language_model.layers.24.linear_attn",
+      "model.language_model.layers.24.linear_attn.norm",
+      "model.language_model.layers.24.linear_attn.in_proj_b",
+      "model.language_model.layers.24.linear_attn.in_proj_a",
+      "model.language_model.layers.25.linear_attn",
+      "model.language_model.layers.25.linear_attn.norm",
+      "model.language_model.layers.25.linear_attn.in_proj_b",
+      "model.language_model.layers.25.linear_attn.in_proj_a",
+      "model.language_model.layers.26.linear_attn",
+      "model.language_model.layers.26.linear_attn.norm",
+      "model.language_model.layers.26.linear_attn.in_proj_b",
+      "model.language_model.layers.26.linear_attn.in_proj_a",
+      "model.language_model.layers.28.linear_attn",
+      "model.language_model.layers.28.linear_attn.norm",
+      "model.language_model.layers.28.linear_attn.in_proj_b",
+      "model.language_model.layers.28.linear_attn.in_proj_a",
+      "model.language_model.layers.29.linear_attn",
+      "model.language_model.layers.29.linear_attn.norm",
+      "model.language_model.layers.29.linear_attn.in_proj_b",
+      "model.language_model.layers.29.linear_attn.in_proj_a",
+      "model.language_model.layers.30.linear_attn",
+      "model.language_model.layers.30.linear_attn.norm",
+      "model.language_model.layers.30.linear_attn.in_proj_b",
+      "model.language_model.layers.30.linear_attn.in_proj_a",
+      "model.language_model.layers.32.linear_attn",
+      "model.language_model.layers.32.linear_attn.norm",
+      "model.language_model.layers.32.linear_attn.in_proj_b",
+      "model.language_model.layers.32.linear_attn.in_proj_a",
+      "model.language_model.layers.33.linear_attn",
+      "model.language_model.layers.33.linear_attn.norm",
+      "model.language_model.layers.33.linear_attn.in_proj_b",
+      "model.language_model.layers.33.linear_attn.in_proj_a",
+      "model.language_model.layers.34.linear_attn",
+      "model.language_model.layers.34.linear_attn.norm",
+      "model.language_model.layers.34.linear_attn.in_proj_b",
+      "model.language_model.layers.34.linear_attn.in_proj_a",
+      "model.language_model.layers.36.linear_attn",
+      "model.language_model.layers.36.linear_attn.norm",
+      "model.language_model.layers.36.linear_attn.in_proj_b",
+      "model.language_model.layers.36.linear_attn.in_proj_a",
+      "model.language_model.layers.37.linear_attn",
+      "model.language_model.layers.37.linear_attn.norm",
+      "model.language_model.layers.37.linear_attn.in_proj_b",
+      "model.language_model.layers.37.linear_attn.in_proj_a",
+      "model.language_model.layers.38.linear_attn",
+      "model.language_model.layers.38.linear_attn.norm",
+      "model.language_model.layers.38.linear_attn.in_proj_b",
+      "model.language_model.layers.38.linear_attn.in_proj_a",
+      "model.language_model.layers.40.linear_attn",
+      "model.language_model.layers.40.linear_attn.norm",
+      "model.language_model.layers.40.linear_attn.in_proj_b",
+      "model.language_model.layers.40.linear_attn.in_proj_a",
+      "model.language_model.layers.41.linear_attn",
+      "model.language_model.layers.41.linear_attn.norm",
+      "model.language_model.layers.41.linear_attn.in_proj_b",
+      "model.language_model.layers.41.linear_attn.in_proj_a",
+      "model.language_model.layers.42.linear_attn",
+      "model.language_model.layers.42.linear_attn.norm",
+      "model.language_model.layers.42.linear_attn.in_proj_b",
+      "model.language_model.layers.42.linear_attn.in_proj_a",
+      "model.language_model.layers.44.linear_attn",
+      "model.language_model.layers.44.linear_attn.norm",
+      "model.language_model.layers.44.linear_attn.in_proj_b",
+      "model.language_model.layers.44.linear_attn.in_proj_a",
+      "model.language_model.layers.45.linear_attn",
+      "model.language_model.layers.45.linear_attn.norm",
+      "model.language_model.layers.45.linear_attn.in_proj_b",
+      "model.language_model.layers.45.linear_attn.in_proj_a",
+      "model.language_model.layers.46.linear_attn",
+      "model.language_model.layers.46.linear_attn.norm",
+      "model.language_model.layers.46.linear_attn.in_proj_b",
+      "model.language_model.layers.46.linear_attn.in_proj_a",
+      "model.language_model.layers.48.linear_attn",
+      "model.language_model.layers.48.linear_attn.norm",
+      "model.language_model.layers.48.linear_attn.in_proj_b",
+      "model.language_model.layers.48.linear_attn.in_proj_a",
+      "model.language_model.layers.49.linear_attn",
+      "model.language_model.layers.49.linear_attn.norm",
+      "model.language_model.layers.49.linear_attn.in_proj_b",
+      "model.language_model.layers.49.linear_attn.in_proj_a",
+      "model.language_model.layers.50.linear_attn",
+      "model.language_model.layers.50.linear_attn.norm",
+      "model.language_model.layers.50.linear_attn.in_proj_b",
+      "model.language_model.layers.50.linear_attn.in_proj_a",
+      "model.language_model.layers.52.linear_attn",
+      "model.language_model.layers.52.linear_attn.norm",
+      "model.language_model.layers.52.linear_attn.in_proj_b",
+      "model.language_model.layers.52.linear_attn.in_proj_a",
+      "model.language_model.layers.53.linear_attn",
+      "model.language_model.layers.53.linear_attn.norm",
+      "model.language_model.layers.53.linear_attn.in_proj_b",
+      "model.language_model.layers.53.linear_attn.in_proj_a",
+      "model.language_model.layers.54.linear_attn",
+      "model.language_model.layers.54.linear_attn.norm",
+      "model.language_model.layers.54.linear_attn.in_proj_b",
+      "model.language_model.layers.54.linear_attn.in_proj_a",
+      "model.language_model.layers.56.linear_attn",
+      "model.language_model.layers.56.linear_attn.norm",
+      "model.language_model.layers.56.linear_attn.in_proj_b",
+      "model.language_model.layers.56.linear_attn.in_proj_a",
+      "model.language_model.layers.57.linear_attn",
+      "model.language_model.layers.57.linear_attn.norm",
+      "model.language_model.layers.57.linear_attn.in_proj_b",
+      "model.language_model.layers.57.linear_attn.in_proj_a",
+      "model.language_model.layers.58.linear_attn",
+      "model.language_model.layers.58.linear_attn.norm",
+      "model.language_model.layers.58.linear_attn.in_proj_b",
+      "model.language_model.layers.58.linear_attn.in_proj_a",
+      "model.language_model.layers.60.linear_attn",
+      "model.language_model.layers.60.linear_attn.norm",
+      "model.language_model.layers.60.linear_attn.in_proj_b",
+      "model.language_model.layers.60.linear_attn.in_proj_a",
+      "model.language_model.layers.61.linear_attn",
+      "model.language_model.layers.61.linear_attn.norm",
+      "model.language_model.layers.61.linear_attn.in_proj_b",
+      "model.language_model.layers.61.linear_attn.in_proj_a",
+      "model.language_model.layers.62.linear_attn",
+      "model.language_model.layers.62.linear_attn.norm",
+      "model.language_model.layers.62.linear_attn.in_proj_b",
+      "model.language_model.layers.62.linear_attn.in_proj_a"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  },
+  "dtype": "bfloat16"
+}

evaluation/code/swift15/checkpoint.py

@@ -0,0 +1,87 @@
+"""Prepare and audit a separate Swift baseline; never mutate the source checkpoint."""
+import argparse
+import copy
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from common import ROOT, RUN, SOURCE, BASELINE, read_json, write_json, sha256, stamp
+
+
+def clone(source, destination):
+    source, destination = Path(source), Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    for p in source.iterdir():
+        if not p.is_file() or ".bak" in p.name or p.name.endswith(".tmp"):
+            continue
+        target = destination / p.name
+        if target.exists():
+            continue
+        if p.suffix == ".safetensors":
+            os.link(p, target)
+        else:
+            shutil.copy2(p, target)
+
+
+def audit(directory):
+    from safetensors import safe_open
+    directory = Path(directory)
+    index = read_json(directory / "model.safetensors.index.json")
+    mapping = index["weight_map"]
+    actual = {}
+    duplicates = []
+    total = 0
+    for name in sorted(set(mapping.values())):
+        p = directory / name
+        total += p.stat().st_size
+        with safe_open(p, framework="pt") as f:
+            for key in f.keys():
+                if key in actual:
+                    duplicates.append(key)
+                actual[key] = name
+    missing = sorted(set(mapping) - set(actual))
+    orphaned = sorted(set(actual) - set(mapping))
+    wrong = [k for k in mapping if actual.get(k) != mapping[k]]
+    assert not (duplicates or missing or orphaned or wrong), (duplicates, missing, orphaned, wrong)
+    config = read_json(directory / "config.json")
+    result = {"time": stamp(), "directory": str(directory), "tensor_count": len(actual),
+              "shard_bytes": total, "duplicate_keys": duplicates,
+              "quantization": config["quantization_config"]}
+    write_json(directory / "hyperqwen-audit.json", result)
+    print("Verified", len(actual), "unique indexed tensors;", round(total / 2**30, 2), "GiB", flush=True)
+    return result
+
+
+def baseline():
+    if (BASELINE / "hyperqwen-build.json").exists():
+        return audit(BASELINE)
+    clone(SOURCE, BASELINE)
+    index = read_json(BASELINE / "model.safetensors.index.json")
+    if "lm_head.weight" in index["weight_map"]:
+        subprocess.run([sys.executable, str(ROOT / "prepare/quant_heads_stream.py"), str(BASELINE)], check=True)
+    if not (BASELINE / "mtp_draft_vocab_ids.pt").exists():
+        subprocess.run([sys.executable, str(ROOT / "prepare/build_draft_vocab.py"), str(BASELINE),
+                        "--ids", str(ROOT / "prepare/draft_vocab_ids.json")], check=True)
+    # config.json is authoritative. Some exports carry a second quantization file.
+    config = read_json(BASELINE / "config.json")
+    if (BASELINE / "quantization_config.json").exists():
+        write_json(BASELINE / "quantization_config.json", config["quantization_config"])
+    write_json(BASELINE / "hyperqwen-build.json", {
+        "created": stamp(), "source": read_json(RUN / "source.json"),
+        "variant": "baseline", "body": "unchanged asymmetric AWQ INT4 group128",
+        "embedding_bits": 8, "lm_head_bits": 8, "mtp_bits": 8,
+        "preparation_script_sha256": sha256(ROOT / "prepare/quant_heads_stream.py"),
+        "draft_vocabulary": "HyperQwen reference ids; rebuilt for Swift in fast variant",
+    })
+    audit(BASELINE)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("action", choices=["baseline", "audit"])
+    ap.add_argument("--model", type=Path, default=BASELINE)
+    args = ap.parse_args()
+    baseline() if args.action == "baseline" else audit(args.model)

evaluation/code/swift15/code_runner.py

@@ -0,0 +1,59 @@
+"""Run only INSIDE the restricted Docker container, never on the host."""
+import decimal
+import json
+import os
+import resource
+import subprocess
+import sys
+
+
+def equal(actual, expected):
+    a, b = actual.split(), expected.split()
+    if len(a) != len(b):
+        return False
+    for x, y in zip(a, b):
+        if x == y:
+            continue
+        try:
+            xx, yy = decimal.Decimal(x), decimal.Decimal(y)
+            if not (xx.is_finite() and yy.is_finite()):
+                return False
+            # Integer output must match exactly, without float precision loss.
+            if all(t.lstrip("+-").isdigit() for t in (x,y)):
+                if xx != yy:
+                    return False
+            elif abs(xx-yy) > max(decimal.Decimal("0.000001"), abs(yy)*decimal.Decimal("0.000001")):
+                return False
+        except decimal.InvalidOperation:
+            return False
+    return True
+
+
+def limits():
+    resource.setrlimit(resource.RLIMIT_CPU, (3, 3))
+    resource.setrlimit(resource.RLIMIT_FSIZE, (1 << 20, 1 << 20))
+
+
+def main():
+    if os.environ.get("SWIFT15_CODE_SANDBOX") != "1":
+        raise SystemExit("Refusing to execute generated code outside the configured container")
+    task = json.load(sys.stdin)
+    with open("/tmp/solution.py", "w") as f:
+        f.write(task["code"])
+    passed = 0
+    for test in task["tests"]:
+        try:
+            r = subprocess.run([sys.executable, "-I", "/tmp/solution.py"], input=test["input"],
+                               text=True, capture_output=True, timeout=4, preexec_fn=limits)
+            if r.returncode or not equal(r.stdout, test["output"]):
+                print(json.dumps({"correct": False, "passed": passed, "total": len(task["tests"]), "reason": "runtime_error" if r.returncode else "wrong_answer"}))
+                return
+        except subprocess.TimeoutExpired:
+            print(json.dumps({"correct": False, "passed": passed, "total": len(task["tests"]), "reason": "test_timeout"}))
+            return
+        passed += 1
+    print(json.dumps({"correct": True, "passed": passed, "total": passed}))
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/common.py

@@ -0,0 +1,49 @@
+"""Shared reproducibility helpers for the Swift 1.5 build."""
+import hashlib
+import json
+import os
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+RUN = Path(os.environ.get("SWIFT15_RUN_DIR", ROOT / "runs/swift15")).resolve()
+SOURCE = ROOT / "models/Swift-1.5-Qwen3.8-27B-AWQ-source"
+BASELINE = ROOT / "models/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen"
+FAST = Path(os.environ.get("SWIFT15_FAST_MODEL", ROOT / "models/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-fast")).resolve()
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text())
+
+
+def write_json(path, value):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + ".tmp")
+    temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
+    os.replace(temporary, path)
+
+
+def sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def records(path):
+    with open(path) as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def write_records(path, rows):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def stamp():
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

evaluation/code/swift15/corpus.py

@@ -0,0 +1,154 @@
+"""Download pinned public training sources and select 6,000 calibration prompts."""
+import argparse
+import collections
+import hashlib
+import json
+import random
+from pathlib import Path
+
+import pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download
+from common import RUN, write_json, write_records, read_json, sha256
+
+SOURCES = {
+    "chat": ("HuggingFaceH4/ultrachat_200k", "8049631c405ae6576f93f445c6b8166f76f5505a", "data/train_sft-00000-of-00003-a3ecf92756993583.parquet"),
+    "code": ("ise-uiuc/Magicoder-OSS-Instruct-75K", "5f839b1f368a76b161028bb9edff055db34022b2", "data-oss_instruct-decontaminated.jsonl"),
+    "tools": ("NousResearch/hermes-function-calling-v1", "dae3e1d28cfbcf4b915c04ea1e072030529b4bda", "func-calling-singleturn.json"),
+    "math": ("openai/gsm8k", "740312add88f781978c0658806c59bc2815b9866", "main/train-00000-of-00001.parquet"),
+    "multilingual": ("CohereLabs/aya_dataset", "f9ea04583f02a8f86404ff6c58bf75fe637df8a2", "data/train-00000-of-00001.parquet"),
+}
+
+
+def load_source(name):
+    repo, revision, filename = SOURCES[name]
+    path = hf_hub_download(repo, filename, repo_type="dataset", revision=revision,
+                           local_dir=RUN / "datasets" / name)
+    if filename.endswith(".parquet"):
+        rows = pq.read_table(path).to_pylist()
+    elif filename.endswith(".jsonl"):
+        rows = [json.loads(s) for s in open(path) if s.strip()]
+    else:
+        rows = json.load(open(path))
+    assert isinstance(rows, list), type(rows)
+    return rows, {"repository": repo, "revision": revision, "file": filename,
+                  "sha256": sha256(path), "rows_available": len(rows)}
+
+
+def message(text):
+    return [{"role": "user", "content": text}]
+
+
+def structured_prompts(n):
+    rows = []
+    for i in range(n):
+        a, b = 11 + i * 3, 7 + i % 37
+        variants = [
+            f'Return only JSON with keys "sum", "difference", "product" for the integers {a} and {b}.',
+            f'A tool returned {{"matches":[{{"name":"item-{i}","price":{a}}},{{"name":"item-{i+1}","price":{b}}}]}}. Return the cheaper item as JSON with keys name and price.',
+            f'Convert these records into CSV with columns name,count: item-{i} has {a}; item-{i+1} has {b}. Output only CSV.',
+            f'An API request for record {i} failed with HTTP 429 and Retry-After: {b}. Explain a safe retry policy and provide a Python implementation.',
+            f'Extract an object with fields city, nights and guests from: "Book accommodation in Vienna for {i%12+1} nights for {i%5+1} guests." Output JSON only.',
+            f'A file-reading tool returned a JSON parse error on line {i%20+1}. Give a debugging plan that preserves the original file and validates the repaired JSON.',
+        ]
+        rows.append({"src": "structured", "source_row": i, "messages": message(f"Request reference: calibration-{i}.\n" + variants[i % len(variants)])})
+    return rows
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--seed", type=int, default=15027)
+    args = ap.parse_args()
+    rng = random.Random(args.seed)
+    counts = {"code": 2100, "tools": 900, "chat": 1200, "math": 900, "multilingual": 600}
+    all_rows, sources = [], {}
+    candidate_hashes = set()
+    for category, count in counts.items():
+        raw, info = load_source(category)
+        sources[category] = info
+        candidates = []
+        for index, r in enumerate(raw):
+            tools = None
+            language = None
+            if category == "chat":
+                msgs = r["messages"][:3] if len(r["messages"]) >= 3 and rng.random() < .25 else r["messages"][:1]
+                if not msgs or msgs[-1]["role"] != "user":
+                    continue
+            elif category == "code":
+                msgs = message(r["problem"])
+                language = r.get("lang", "unknown")
+            elif category == "math":
+                msgs = message(r["question"])
+            elif category == "multilingual":
+                language = r["language_code"]
+                if language not in {"deu", "fra", "spa", "zho", "dan", "jpn", "arb", "hin", "por", "ita"}:
+                    continue
+                msgs = message(r["inputs"])
+            else:
+                user_turns = [m["value"] for m in r["conversations"] if m["from"] in {"human", "user"}]
+                if not user_turns:
+                    continue
+                msgs = message(user_turns[0])
+                raw_tools = json.loads(r["tools"]) if isinstance(r["tools"], str) else r["tools"]
+                tools = [{"type": "function", "function": t.get("function", t)} for t in raw_tools]
+            if not all(isinstance(m.get("content"), str) for m in msgs):
+                continue
+            length = sum(len(m["content"]) for m in msgs)
+            if not 20 <= length <= 16000:
+                continue
+            row = {"src": category, "source_row": index, "messages": msgs}
+            if tools:
+                row["tools"] = tools
+            if language:
+                row["language"] = language
+            fingerprint = hashlib.sha256(json.dumps([msgs, tools], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+            if fingerprint in candidate_hashes:
+                continue
+            candidate_hashes.add(fingerprint)
+            candidates.append(row)
+        rng.shuffle(candidates)
+        # Balance languages instead of allowing the largest source language to dominate.
+        if category in {"multilingual", "code"}:
+            groups = collections.defaultdict(list)
+            for row in candidates:
+                groups[row["language"]].append(row)
+            chosen = []
+            while len(chosen) < count and groups:
+                for lang in list(sorted(groups)):
+                    chosen.append(groups[lang].pop())
+                    if not groups[lang]:
+                        del groups[lang]
+                    if len(chosen) == count:
+                        break
+        else:
+            chosen = candidates[:count]
+        assert len(chosen) == count, (category, len(chosen), count)
+        all_rows.extend(chosen)
+        print(category, len(chosen), flush=True)
+    all_rows.extend(structured_prompts(300))
+    rng.shuffle(all_rows)
+    seen = set()
+    for i, row in enumerate(all_rows):
+        digest = hashlib.sha256(json.dumps([row["messages"], row.get("tools")], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+        assert digest not in seen, "Duplicate calibration prompt: " + digest
+        seen.add(digest)
+        row.update(id=i, prompt_sha256=digest, think=rng.random() < (.7 if row["src"] == "math" else .5))
+    # Split whole examples before generation/capture, preserving category proportions.
+    groups = collections.defaultdict(list)
+    for row in all_rows:
+        groups[row["src"]].append(row)
+    holdout = {r["id"] for g in groups.values() for r in g[:max(1, len(g)//10)]}
+    for row in all_rows:
+        row["split"] = "holdout" if row["id"] in holdout else "calibration"
+    write_records(RUN / "calibration/prompts.jsonl", all_rows)
+    write_json(RUN / "calibration/manifest.json", {
+        "seed": args.seed, "sources": sources, "counts": dict(collections.Counter(r["src"] for r in all_rows)),
+        "holdout_examples": len(holdout), "total_examples": len(all_rows),
+        "prompts_sha256": sha256(RUN / "calibration/prompts.jsonl"),
+        "structured_source": "swift15/corpus.py deterministic templates, not benchmark test prompts",
+        "source_substitution": "Salesforce xLAM returned gated-access 403; use the independently released Apache-2.0 NousResearch source with native Swift tool formatting.",
+    })
+    print("Wrote", len(all_rows), "prompts;", len(holdout), "held out", flush=True)
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/eval_data.py

@@ -0,0 +1,135 @@
+"""Freeze test data independently of calibration, including a reproducible LCB subset."""
+import base64
+import collections
+import io
+import json
+import pickle
+import random
+import zlib
+from pathlib import Path
+
+import pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download
+from common import ROOT, RUN, write_json, write_records, sha256
+
+
+class DataOnlyUnpickler(pickle.Unpickler):
+    def find_class(self, module, name):
+        raise ValueError("Executable object in benchmark test data")
+
+
+def private_tests(value):
+    try:
+        return json.loads(value)
+    except (ValueError, TypeError):
+        decoded = DataOnlyUnpickler(io.BytesIO(zlib.decompress(base64.b64decode(value)))).load()
+        return json.loads(decoded) if isinstance(decoded, str) else decoded
+
+
+def download(repo, revision, filename):
+    return Path(hf_hub_download(repo, filename, repo_type="dataset", revision=revision,
+                               local_dir=RUN / "evaluation/downloads" / repo.replace("/", "__")))
+
+
+def tool_tasks():
+    rows = []
+    cities = ["Graz", "Linz", "Salzburg", "Innsbruck", "Klagenfurt", "Villach", "Wels", "Steyr", "Bregenz", "Eisenstadt"]
+    tool = {"type": "function", "function": {"name": "get_weather", "description": "Get current weather for a city.",
+            "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"], "additionalProperties": False}}}
+    for i in range(30):
+        if i < 20:
+            city = cities[i % len(cities)]
+            rows.append({"id": f"tool-{i}", "suite": "tools", "think": False, "max_tokens": 2048,
+                         "messages": [{"role": "user", "content": f"Use get_weather to check {city}. Then return only a JSON object with keys city and fahrenheit. Convert the tool's Celsius temperature to Fahrenheit."}],
+                         "tools": [tool], "expected_call": {"name": "get_weather", "arguments": {"city": city}},
+                         "tool_result": {"city": city, "celsius": i - 7},
+                         "expected": {"city": city, "fahrenheit": (i-7)*1.8+32}})
+        else:
+            codes = [f"SKU-{i}-{j}" for j in range(4)]
+            records = [{"sku": code, "stock": (i*j+3)%11} for j,code in enumerate(codes)]
+            expected = [r["sku"] for r in records if r["stock"] >= 5]
+            rows.append({"id": f"json-{i}", "suite": "tools", "think": False, "max_tokens": 2048,
+                         "messages": [{"role": "user", "content": f'Return only JSON with one key "available" containing the SKUs with stock >= 5, preserving input order. Records: {json.dumps(records)}'}],
+                         "expected": {"available": expected}})
+    return rows
+
+
+def main():
+    out = RUN / "evaluation"
+    out.mkdir(parents=True, exist_ok=True)
+    rng = random.Random(15027)
+    tasks, sources = [], []
+    gsm = download("openai/gsm8k", "740312add88f781978c0658806c59bc2815b9866", "main/test-00000-of-00001.parquet")
+    for i, r in enumerate(pq.read_table(gsm).to_pylist()[:200]):
+        tasks.append({"id": f"gsm8k-{i}", "suite": "gsm8k", "think": False, "max_tokens": 768,
+                      "messages": [{"role": "user", "content": r["question"] + "\n\nSolve step by step, then give the final answer as 'Final answer: <number>'."}],
+                      "expected": r["answer"].split("####")[-1].strip()})
+    sources.append({"dataset": "openai/gsm8k", "split": "main/test", "file_sha256": sha256(gsm), "selection": "first 200, same as HyperQwen"})
+    iff = download("allenai/IFBench_test", "2e8a48de45ff3bf41242f927254ca81b59ca3ae2", "data/train-00000-of-00001.parquet")
+    for r in pq.read_table(iff).to_pylist():
+        tasks.append({"id": f"ifbench-{r['key']}", "suite": "ifbench", "think": True, "max_tokens": 4096,
+                      "messages": [{"role": "user", "content": r["prompt"]}],
+                      "instruction_id_list": r["instruction_id_list"], "kwargs": r["kwargs"]})
+    sources.append({"dataset": "allenai/IFBench_test", "file_sha256": sha256(iff),
+                    "note": "Upstream names this split train, but it is the benchmark test set; never used for calibration."})
+    pool = {}
+    for filename in ["test.jsonl"] + [f"test{i}.jsonl" for i in range(2, 7)]:
+        p = download("livecodebench/code_generation_lite", "0fe84c3912ea0c4d4a78037083943e8f0c4dd505", filename)
+        sources.append({"dataset": "livecodebench/code_generation_lite", "file": filename, "sha256": sha256(p)})
+        for line in p.read_text().splitlines():
+            r = json.loads(line)
+            # v6 timeframe; stdin programs only, to keep execution protocol explicit.
+            if r["contest_date"][:10] > "2025-04-30" or r.get("starter_code"):
+                continue
+            public = json.loads(r["public_test_cases"])
+            private = private_tests(r["private_test_cases"])
+            tests = public + private
+            if not tests or any(t.get("testtype") != "stdin" for t in tests):
+                continue
+            pool[(r["platform"], r["question_id"])] = (r, tests)
+    grouped = collections.defaultdict(list)
+    for value in pool.values():
+        grouped[value[0]["difficulty"]].append(value)
+    for level, n in [("easy", 34), ("medium", 33), ("hard", 33)]:
+        group = sorted(grouped[level], key=lambda x: (x[0]["platform"], x[0]["question_id"]))
+        rng.shuffle(group)
+        assert len(group) >= n, (level, len(group))
+        for r, tests in group[:n]:
+            tasks.append({"id": f"lcb-{r['platform']}-{r['question_id']}", "suite": "livecodebench", "think": True,
+                          "difficulty": level, "max_tokens": 4096,
+                          "messages": [{"role": "user", "content": r["question_content"] + "\n\nWrite a complete Python 3 program that reads from standard input and writes to standard output. Put the final solution in a single ```python``` code block."}],
+                          "tests": tests})
+    tasks.extend(tool_tasks())
+    write_records(out / "tasks.jsonl", tasks)
+    pilot = []
+    for suite in ["gsm8k", "ifbench", "livecodebench", "tools"]:
+        candidates = [t for t in tasks if t["suite"] == suite]
+        rng.shuffle(candidates)
+        pilot.extend(t["id"] for t in candidates[:5])
+    write_json(out / "pilot-ids.json", pilot)
+    # Freeze the old battery's input bytes, especially installed vLLM source text.
+    texts = []
+    p = ROOT / "bench/quality-data/wikitext/wikitext-2-raw-v1/test-00000-of-00001.parquet"
+    text = "".join(pq.read_table(p).column("text").to_pylist())
+    for i in range(0, min(len(text), 40*1200), 1200):
+        texts.append({"language": "en", "text": text[i:i+1200]})
+    p = ROOT / "bench/quality-data/fineweb2/data/dan_Latn/test/000_00000.parquet"
+    danish = pq.read_table(p, columns=["text"]).column("text").to_pylist()
+    random.Random(0).shuffle(danish)
+    texts.extend({"language": "da", "text": t[:1200]} for t in [t for t in danish if len(t)>1500][:40])
+    for p in sorted((ROOT / "venv/lib/python3.12/site-packages/vllm/v1/core").glob("*.py")):
+        text = p.read_text()
+        texts.extend({"language": "code", "text": text[i:i+1200]} for i in range(0,min(len(text),4800),1200) if len(text[i:i+1200])>800)
+        if sum(t["language"] == "code" for t in texts) >= 40:
+            break
+    write_records(out / "perplexity.jsonl", texts)
+    write_json(out / "manifest.json", {"seed": 15027, "sources": sources,
+               "tasks": dict(collections.Counter(t["suite"] for t in tasks)),
+               "task_file_sha256": sha256(out / "tasks.jsonl"), "ppl_file_sha256": sha256(out / "perplexity.jsonl"),
+               "protocol": "Single-seed bounded-budget local comparison, not official full-release leaderboard scores.",
+               "lcb_scoring": "100 stratified stdin-only v6-era tasks; all supplied public/private tests; whitespace token comparison with numeric tolerance; not the full official LCB runner."})
+    print("Frozen", len(tasks), "tasks and", len(texts), "perplexity windows", flush=True)
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/evaluate.py

@@ -0,0 +1,265 @@
+"""Stream naturally terminated tasks, preserve raw responses and score failures too."""
+import argparse
+import collections
+import concurrent.futures
+import copy
+import json
+import math
+import os
+import re
+import statistics
+import subprocess
+import time
+import urllib.request
+import uuid
+from pathlib import Path
+
+from common import ROOT, RUN, records, read_json, write_json, sha256, stamp
+
+
+def key():
+    return os.environ.get("VLLM_API_KEY") or (ROOT / "api_key.txt").read_text().strip()
+
+
+def post(api, path, body, timeout=600):
+    req = urllib.request.Request(api + path, json.dumps(body).encode(),
+                                 headers={"Content-Type": "application/json", "Authorization": "Bearer " + key()})
+    return urllib.request.urlopen(req, timeout=timeout)
+
+
+def stream(api, body):
+    body = dict(body, stream=True, stream_options={"include_usage": True})
+    start = time.monotonic()
+    first, first_answer, last = None, None, None
+    content, reasoning, calls = [], [], {}
+    usage, finish = {}, None
+    with post(api, "/chat/completions", body) as response:
+        for line in response:
+            if not line.startswith(b"data: "):
+                continue
+            raw = line[6:].strip()
+            if raw == b"[DONE]":
+                break
+            chunk = json.loads(raw)
+            if "error" in chunk:
+                raise RuntimeError(str(chunk["error"]))
+            if chunk.get("usage"):
+                usage = chunk["usage"]
+            for choice in chunk.get("choices", []):
+                delta = choice.get("delta", {})
+                now = time.monotonic()
+                think = delta.get("reasoning_content") or delta.get("reasoning") or ""
+                text = delta.get("content") or ""
+                tool = delta.get("tool_calls") or []
+                if think or text or tool:
+                    first = first or now
+                    last = now
+                if text:
+                    first_answer = first_answer or now
+                    content.append(text)
+                reasoning.append(think)
+                for call in tool:
+                    c = calls.setdefault(call["index"], {"id": "", "type": "function", "function": {"name": "", "arguments": ""}})
+                    if call.get("id"):
+                        c["id"] = call["id"]
+                    for k in ["name", "arguments"]:
+                        c["function"][k] += call.get("function", {}).get(k) or ""
+                finish = choice.get("finish_reason") or finish
+    end = time.monotonic()
+    if not usage:
+        raise RuntimeError("Missing usage counters; refusing to report guessed token counts")
+    n = usage.get("completion_tokens", 0)
+    return {"content": "".join(content), "reasoning": "".join(reasoning),
+            "tool_calls": [calls[i] for i in sorted(calls)], "usage": usage, "finish_reason": finish,
+            "wall_seconds": end-start, "ttft_seconds": None if first is None else first-start,
+            "time_to_answer_seconds": None if first_answer is None else first_answer-start,
+            "decode_seconds": 0 if first is None or last is None else last-first,
+            "decode_tps": (n-1)/(last-first) if n>1 and last is not None and last>first else None}
+
+
+def json_equal(a, b):
+    if isinstance(b, bool):
+        return isinstance(a, bool) and a == b
+    if isinstance(b, dict):
+        return isinstance(a, dict) and set(a) == set(b) and all(json_equal(a[k], v) for k,v in b.items())
+    if isinstance(b, list):
+        return isinstance(a, list) and len(a) == len(b) and all(json_equal(x,y) for x,y in zip(a,b))
+    if isinstance(b, (int,float)) and not isinstance(b, bool):
+        return isinstance(a, (int,float)) and not isinstance(a,bool) and math.isfinite(a) and abs(a-b)<1e-6
+    return a == b
+
+
+def gsm_score(text, expected):
+    match = re.search(r"Final answer:\s*\**\s*\$?(-?[\d,]*\.?\d+)", text)
+    numbers = re.findall(r"-?\d[\d,]*\.?\d*", text.replace("$", ""))
+    pred = match.group(1) if match else (numbers[-1] if numbers else "")
+    try:
+        value, gold = float(pred.replace(",", "")), float(expected.replace(",", ""))
+        return math.isfinite(value) and abs(value-gold)<1e-6
+    except ValueError:
+        return False
+
+
+def code_score(text, task):
+    blocks = re.findall(r"```(?:python|py)?\s*\n(.*?)```", text, re.S)
+    if not blocks:
+        return {"correct": False, "reason": "no_code_block"}
+    # This image is pinned locally in the run manifest before evaluation begins.
+    image = read_json(RUN / "evaluation/runtime.json")["code_image"]
+    container = "swift15-test-" + uuid.uuid4().hex
+    cmd = ["docker", "run", "--name", container, "--rm", "-i", "--network", "none", "--read-only", "--memory", "512m", "--cpus", "1",
+           "--pids-limit", "64", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--user", "65534:65534",
+           "--tmpfs", "/tmp:rw,noexec,nosuid,size=64m", "-e", "SWIFT15_CODE_SANDBOX=1",
+           "-v", str(ROOT / "swift15/code_runner.py") + ":/runner.py:ro", image, "python", "-I", "/runner.py"]
+    try:
+        r = subprocess.run(cmd, input=json.dumps({"code": blocks[-1], "tests": task["tests"]}),
+                           text=True, capture_output=True, timeout=180)
+        if r.returncode:
+            return {"correct": False, "reason": "sandbox_error", "detail": r.stderr[-1000:]}
+        return json.loads(r.stdout)
+    except subprocess.TimeoutExpired:
+        return {"correct": False, "reason": "task_test_timeout"}
+    finally:
+        subprocess.run(["docker", "rm", "-f", container], capture_output=True, timeout=30)
+
+
+def execute(task, api):
+    start = time.monotonic()
+    result = {"id": task["id"], "suite": task["suite"], "calls": [], "correct": False, "error": None}
+    body = {"model": "qwen3.8-27b", "messages": copy.deepcopy(task["messages"]), "max_tokens": task["max_tokens"],
+            "temperature": 0, "seed": 15027, "top_p": 1.0,
+            "chat_template_kwargs": {"enable_thinking": task["think"], "reasoning_effort": "xhigh"}}
+    # HyperQwen evaluates thinking tasks at the model's recommended sampling.
+    # Greedy remains the repository's GSM8K protocol and our deterministic tool test.
+    if task["think"]:
+        body.update(temperature=1.0, top_p=.95, top_k=20, min_p=0,
+                    presence_penalty=0, repetition_penalty=1.0)
+    result["sampling"] = {k:body[k] for k in ["temperature","top_p","seed"]}
+    if "top_k" in body:
+        result["sampling"]["top_k"] = body["top_k"]
+    if task.get("tools"):
+        body.update(tools=task["tools"], tool_choice="auto")
+    try:
+        first = stream(api, body)
+        result["calls"].append(first)
+        response = first
+        if task.get("tools"):
+            calls = first["tool_calls"]
+            expected = task["expected_call"]
+            if len(calls)!=1 or calls[0]["function"]["name"]!=expected["name"] or not json_equal(json.loads(calls[0]["function"]["arguments"]),expected["arguments"]):
+                result["error"] = "incorrect_tool_call"
+            else:
+                body["messages"].append({"role":"assistant","content":first["content"] or None,"tool_calls":calls})
+                body["messages"].append({"role":"tool","tool_call_id":calls[0]["id"],"content":json.dumps(task["tool_result"])})
+                body["tool_choice"] = "none"
+                response = stream(api, body)
+                result["calls"].append(response)
+        result["model_seconds"] = sum(c["wall_seconds"] for c in result["calls"])
+        result["response"] = response["content"]
+        if result["error"] is None and all(c["finish_reason"] != "length" for c in result["calls"]):
+            if task["suite"] == "gsm8k":
+                result["correct"] = gsm_score(response["content"],task["expected"])
+            elif task["suite"] == "tools":
+                try: result["correct"] = json_equal(json.loads(response["content"]),task["expected"])
+                except ValueError: pass
+            elif task["suite"] == "livecodebench":
+                result["code_score"] = code_score(response["content"],task)
+                result["correct"] = result["code_score"]["correct"]
+            else:
+                result["correct"] = None  # official IFBench scoring, batched after generation
+    except Exception as e:
+        result["error"] = type(e).__name__ + ": " + str(e)[:500]
+        result["model_seconds"] = time.monotonic()-start
+        result["token_counts_incomplete"] = True
+    result["task_seconds"] = time.monotonic()-start
+    result.setdefault("model_seconds", sum(c["wall_seconds"] for c in result["calls"]))
+    result["input_tokens"] = sum(c["usage"].get("prompt_tokens",0) for c in result["calls"])
+    result["output_tokens"] = sum(c["usage"].get("completion_tokens",0) for c in result["calls"])
+    result["total_tokens"] = result["input_tokens"] + result["output_tokens"]
+    result["truncated"] = any(c["finish_reason"] == "length" for c in result["calls"])
+    if result["truncated"]:
+        result["correct"] = False  # token-limit truncation counts as a wrong answer
+    return result
+
+
+def aggregate(rows):
+    correct = sum(bool(r["correct"]) and not r["truncated"] for r in rows)
+    truncated = sum(r["truncated"] for r in rows)
+    return {"attempted":len(rows),"correct":correct,"accuracy":correct/len(rows),
+            "truncation_policy": "count_as_wrong",
+            "truncated_counted_as_wrong":truncated,
+            "errors":sum(r["error"] is not None for r in rows),"truncated":sum(r["truncated"] for r in rows),
+            "incomplete_token_counts":sum(r.get("token_counts_incomplete",False) for r in rows),
+            "mean_output_tokens":statistics.mean(r["output_tokens"] for r in rows),
+            "mean_input_tokens":statistics.mean(r["input_tokens"] for r in rows),
+            "mean_total_tokens":statistics.mean(r["input_tokens"] + r["output_tokens"] for r in rows),
+            "mean_model_seconds":statistics.mean(r["model_seconds"] for r in rows),
+            "median_model_seconds":statistics.median(r["model_seconds"] for r in rows),
+            "p95_model_seconds":sorted(r["model_seconds"] for r in rows)[math.ceil(.95*len(rows))-1],
+            "summed_request_seconds_per_correct":sum(r["model_seconds"] for r in rows)/correct if correct else None,
+            "note":"Summed request seconds are not GPU compute time when concurrency exceeds one."}
+
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument("tag")
+    ap.add_argument("--api",default="http://127.0.0.1:18021/v1")
+    ap.add_argument("--pilot",action="store_true")
+    ap.add_argument("--suites",default="gsm8k,ifbench,livecodebench,tools")
+    ap.add_argument("--concurrency",type=int,default=1)
+    args=ap.parse_args()
+    tasks=records(RUN/"evaluation/tasks.jsonl")
+    wanted=set(read_json(RUN/"evaluation/pilot-ids.json")) if args.pilot else None
+    tasks=[t for t in tasks if t["suite"] in args.suites.split(",") and (wanted is None or t["id"] in wanted)]
+    folder=RUN/"results"/args.tag
+    folder.mkdir(parents=True,exist_ok=True)
+    manifest={"tasks_sha256":sha256(RUN/"evaluation/tasks.jsonl"),"task_ids":[t["id"] for t in tasks],
+              "concurrency":args.concurrency,"sampling":"thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027", "api":args.api}
+    identity = read_json(RUN/"active-server.json")
+    assert not identity.get("stopped"), "No managed benchmark server is active"
+    manifest["server"] = {k:v for k,v in identity.items() if k not in {"created", "pid"}}
+    if (folder/"manifest.json").exists():
+        assert read_json(folder/"manifest.json")==manifest,"Cannot resume with different settings"
+    write_json(folder/"manifest.json",manifest)
+    output=folder/"tasks.jsonl"
+    old=records(output) if output.exists() else []
+    done={r["id"] for r in old}
+    todo=[t for t in tasks if t["id"] not in done]
+    start=time.monotonic()
+    with output.open("a") as f, concurrent.futures.ThreadPoolExecutor(args.concurrency) as pool:
+        futures=[pool.submit(execute,t,args.api) for t in todo]
+        for future in concurrent.futures.as_completed(futures):
+            result=future.result()
+            f.write(json.dumps(result)+"\n");f.flush()
+            print(result["id"],"correct=",result["correct"],"tokens=",result["output_tokens"],"seconds=",round(result["model_seconds"],2),"error=",result["error"],flush=True)
+    elapsed=time.monotonic()-start
+    rows=records(output)
+    for row in rows:
+        if row["truncated"]:
+            row["correct"] = False
+    lookup={t["id"]:t for t in tasks}
+    pending=[r for r in rows if r["suite"]=="ifbench" and r["correct"] is None and not r["truncated"]]
+    if pending:
+        env=dict(os.environ,NLTK_DATA=str(RUN/"nltk_data"))
+        r=subprocess.run([str(RUN/"eval-venv/bin/python"),str(ROOT/"swift15/ifbench_score.py")],
+                         input=json.dumps([{"task":lookup[r["id"]],"response":r["response"]} for r in pending]),
+                         text=True,capture_output=True,check=True,env=env)
+        scores=json.loads(r.stdout)
+        for row,score in zip(pending,scores):row.update(score)
+    from common import write_records
+    write_records(folder/"scored.jsonl",rows)
+    groups=collections.defaultdict(list)
+    for row in rows:groups[row["suite"]].append(row)
+    summary={"created":stamp(),"concurrency":args.concurrency,"resumed":bool(old),
+             "new_run_wall_seconds":elapsed,"new_tasks":len(todo),"suites":{k:aggregate(v) for k,v in groups.items()}}
+    summary["quality_comparison_ready"] = not any(r.get("token_counts_incomplete") for r in rows)
+    summary["truncation_policy"] = "count_as_wrong"
+    summary["truncated_task_ids"] = [r["id"] for r in rows if r["truncated"]]
+    if not old:
+        correct=sum(bool(r["correct"]) for r in rows)
+        summary.update(suite_wall_seconds=elapsed,wall_seconds_per_correct=elapsed/correct if correct else None)
+    write_json(folder/"summary.json",summary)
+    print(json.dumps(summary,indent=2),flush=True)
+
+
+if __name__=="__main__":main()

evaluation/code/swift15/generate.py

@@ -0,0 +1,88 @@
+"""Generate fresh Swift responses; resumable by prompt hash and model identity."""
+import os
+os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1")
+os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+import argparse
+import json
+import time
+from common import RUN, BASELINE, records, read_json, write_json, sha256, stamp
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default=str(BASELINE))
+    ap.add_argument("--limit", type=int)
+    ap.add_argument("--chunk", type=int, default=128)
+    args = ap.parse_args()
+    from transformers import AutoTokenizer
+    from vllm import LLM
+    data = RUN / "calibration"
+    prompts = records(data / "prompts.jsonl")
+    if args.limit:
+        prompts = prompts[:args.limit]
+    manifest = {"model": args.model, "config_sha256": sha256(os.path.join(args.model, "config.json")),
+                "prompt_manifest_sha256": sha256(data / "manifest.json"),
+                "thinking_cap": 2048, "nonthinking_cap": 1024, "seed": 15027,
+                "temperature": 1.0, "top_p": .95, "top_k": 20,
+                "max_model_len": 8192, "kv_cache_dtype": "fp8", "int8_activations": False}
+    mp = data / "generation-manifest.json"
+    if mp.exists():
+        assert read_json(mp) == manifest, "Generation settings changed; use a separate run directory"
+    else:
+        write_json(mp, manifest)
+    output = data / "gen.jsonl"
+    done = {r["id"]: r for r in records(output)} if output.exists() else {}
+    for p in prompts:
+        if p["id"] in done:
+            assert p["prompt_sha256"] == done[p["id"]]["prompt_sha256"]
+    todo = [p for p in prompts if p["id"] not in done]
+    if not todo:
+        print("All requested examples already generated")
+        return
+    tok = AutoTokenizer.from_pretrained(args.model)
+    llm = LLM(model=args.model, gpu_memory_utilization=.93, max_model_len=8192,
+              max_num_seqs=32, max_num_batched_tokens=2048, kv_cache_dtype="fp8",
+              mamba_ssm_cache_dtype="float16", language_model_only=True,
+              enable_prefix_caching=False,
+              compilation_config={"max_cudagraph_capture_size": 32, "custom_ops": ["+rms_norm", "+silu_and_mul"]})
+    base = llm.get_default_sampling_params()
+    t0 = time.monotonic()
+    token_count = 0
+    with output.open("a") as f:
+        for start in range(0, len(todo), args.chunk):
+            batch, inputs, params = [], [], []
+            for p in todo[start:start + args.chunk]:
+                kwargs = {"tools": p["tools"]} if p.get("tools") else {}
+                text = tok.apply_chat_template(p["messages"], tokenize=False, add_generation_prompt=True,
+                                               enable_thinking=p["think"], **kwargs)
+                ids = tok.encode(text, add_special_tokens=False)
+                if len(ids) > 6000:
+                    raise ValueError(f"Prompt {p['id']} exceeds calibration context: {len(ids)}")
+                sp = base.clone()
+                sp.max_tokens = 2048 if p["think"] else 1024
+                sp.temperature, sp.top_p, sp.top_k = 1.0, .95, 20
+                sp.seed = 15027 + p["id"]
+                batch.append((p, ids))
+                inputs.append({"prompt_token_ids": ids})
+                params.append(sp)
+            results = llm.generate(inputs, params, use_tqdm=False)
+            for (p, ids), result in zip(batch, results):
+                c = result.outputs[0]
+                r = {k: p[k] for k in ["id", "src", "think", "split", "prompt_sha256"]}
+                r.update(prompt_ids=ids, output_ids=list(c.token_ids), finish=c.finish_reason)
+                f.write(json.dumps(r) + "\n")
+                token_count += len(c.token_ids)
+            f.flush()
+            elapsed = time.monotonic() - t0
+            print(f"{start+len(batch)}/{len(todo)} examples; {token_count} output tokens; {token_count/elapsed:.1f} tok/s; {elapsed/60:.1f} min", flush=True)
+    rows = records(output)
+    write_json(data / "generation-summary.json", {
+        "completed_at": stamp(), "examples": len(rows), "output_tokens": sum(len(r["output_ids"]) for r in rows),
+        "truncated": sum(r["finish"] == "length" for r in rows),
+        "note": "Calibration caps are not evaluation budgets; truncation rates are disclosed.",
+    })
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/ifbench_score.py

@@ -0,0 +1,21 @@
+"""Official IFBench strict prompt-level scoring in the isolated eval environment."""
+import json
+import sys
+from ifbench import instructions_registry
+
+
+def score(task, response):
+    passed = []
+    for name, kwargs in zip(task["instruction_id_list"], task["kwargs"]):
+        checker = instructions_registry.INSTRUCTION_DICT[name](name)
+        checker.build_description(**{k: v for k, v in kwargs.items() if v is not None})
+        needed = checker.get_instruction_args()
+        if needed and "prompt" in needed:
+            checker.build_description(prompt=task["messages"][-1]["content"])
+        passed.append(bool(response.strip()) and bool(checker.check_following(response)))
+    return {"correct": all(passed), "instructions": passed}
+
+
+if __name__ == "__main__":
+    rows = json.load(sys.stdin)
+    print(json.dumps([score(r["task"], r["response"]) for r in rows]))

evaluation/code/swift15/measure.py

@@ -0,0 +1,94 @@
+"""Frozen teacher-forced perplexity and fixed-output serving throughput."""
+import argparse
+import collections
+import concurrent.futures
+import json
+import math
+import statistics
+import re
+import time
+import urllib.request
+from common import RUN, records, read_json, write_json, sha256
+from evaluate import stream, post, key
+
+
+def perplexity(api):
+    windows = records(RUN / "evaluation/perplexity.jsonl")
+    rows = []
+    for i, window in enumerate(windows):
+        with post(api, "/completions", {"model": "qwen3.8-27b", "prompt": window["text"],
+                  "max_tokens": 1, "temperature": 0, "prompt_logprobs": 0}) as response:
+            result = json.load(response)
+        values = []
+        for entry in result["choices"][0]["prompt_logprobs"][1:]:
+            if entry is None or len(entry) != 1:
+                raise ValueError("Expected exactly the observed token's log probability")
+            lp = next(iter(entry.values()))
+            lp = lp["logprob"] if isinstance(lp, dict) else lp
+            assert math.isfinite(lp)
+            values.append(lp)
+        assert values
+        rows.append({"id": i, "language": window["language"], "tokens": len(values), "logprob_sum": sum(values)})
+    groups = collections.defaultdict(list)
+    for r in rows:
+        groups[r["language"]].append(r)
+        groups["all"].append(r)
+    return {"input_sha256": sha256(RUN / "evaluation/perplexity.jsonl"), "windows": rows,
+            "scores": {name: {"tokens": sum(r["tokens"] for r in group),
+                                "ppl": math.exp(-sum(r["logprob_sum"] for r in group)/sum(r["tokens"] for r in group))}
+                       for name, group in groups.items()}}
+
+
+def metrics(api):
+    req = urllib.request.Request(api.removesuffix("/v1") + "/metrics", headers={"Authorization": "Bearer " + key()})
+    with urllib.request.urlopen(req, timeout=10) as response:
+        return response.read().decode()
+
+
+def throughput(api, concurrency):
+    prompts = ["Explain how a hash table handles collisions, with examples.",
+               "Write a Python implementation of merge sort and explain its complexity.",
+               "Describe how to design a reliable background job queue.",
+               "Explain photosynthesis and the role of chlorophyll."]
+    def one(i):
+        return stream(api, {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": prompts[i % len(prompts)]}],
+                            "temperature": 0, "seed": 15027, "max_tokens": 512, "ignore_eos": True,
+                            "chat_template_kwargs": {"enable_thinking": False}})
+    one(0)  # identical warm-up for every model
+    before = metrics(api)
+    start = time.monotonic()
+    with concurrent.futures.ThreadPoolExecutor(concurrency) as pool:
+        rows = list(pool.map(one, range(max(8, concurrency*2))))
+    seconds = time.monotonic()-start
+    after = metrics(api)
+    def counters(text):
+        totals = collections.Counter()
+        for line in text.splitlines():
+            match = re.match(r'(vllm:spec_decode_num_(?:drafts|draft_tokens|accepted_tokens)_total)(?:\{[^}]*\})?\s+([\d.eE+-]+)',line)
+            if match:
+                totals[match[1]] += float(match[2])
+        return totals
+    delta = counters(after)
+    delta.subtract(counters(before))
+    assert all(r["usage"]["completion_tokens"] == 512 for r in rows), "Fixed-length benchmark stopped early"
+    return {"concurrency": concurrency, "requests": len(rows), "output_tokens_per_request": 512,
+            "wall_seconds": seconds, "aggregate_output_tps": sum(r["usage"]["completion_tokens"] for r in rows)/seconds,
+            "median_decode_tps": statistics.median(r["decode_tps"] for r in rows),
+            "median_ttft_seconds": statistics.median(r["ttft_seconds"] for r in rows),
+            "sampling": "greedy; ignore_eos for this throughput test only", "calls": rows,
+            "speculation_counter_deltas": dict(delta),
+            "decode_tps_note": "Client stream timing estimate; speculative decoding may deliver several tokens in one event.",
+            "metrics_before": before, "metrics_after": after}
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("tag")
+    ap.add_argument("--api", default="http://127.0.0.1:18021/v1")
+    ap.add_argument("--kind", choices=["ppl", "speed"], required=True)
+    ap.add_argument("--concurrency", type=int, default=1)
+    args = ap.parse_args()
+    result = perplexity(args.api) if args.kind == "ppl" else throughput(args.api, args.concurrency)
+    target = RUN / "results" / args.tag / ("perplexity.json" if args.kind == "ppl" else f"speed-c{args.concurrency}.json")
+    write_json(target, result)
+    print(target, flush=True)

evaluation/code/swift15/quantize.py

@@ -0,0 +1,183 @@
+"""Calibrate INT4 heads from pristine BF16 tensors and publish a local fast variant."""
+import argparse
+import collections
+import copy
+import json
+import math
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+from compressed_tensors.compressors.pack_quantized.base import pack_to_int32
+from common import ROOT, RUN, SOURCE, BASELINE, FAST, read_json, write_json, records, sha256, stamp
+from checkpoint import clone, audit
+sys.path.insert(0, str(ROOT / "drafter"))
+from gptq_utils import accumulate_hessian, gptq_quantize, rtn_quantize, dequant
+
+
+def original(key):
+    index = read_json(SOURCE / "model.safetensors.index.json")["weight_map"]
+    with safe_open(SOURCE / index[key], "pt") as f:
+        return f.get_tensor(key)
+
+
+def vocabulary():
+    from transformers import AutoTokenizer
+    rows = records(RUN / "calibration/gen.jsonl")
+    counts, held = collections.Counter(), collections.Counter()
+    for r in rows:
+        (held if r["split"] == "holdout" else counts).update(r["output_ids"])
+    assert counts and held
+    tok = AutoTokenizer.from_pretrained(BASELINE)
+    special = set(tok.all_special_ids)
+    total = sum(counts.values())
+    selected = set(special)
+    mass = sum(counts[t] for t in selected)
+    for token, count in counts.most_common():
+        if token not in selected:
+            selected.add(token)
+            mass += count
+        if mass / total >= .998 and len(selected) >= 16384:
+            break
+        if len(selected) >= 65536:
+            break
+    target_size = min(65536, max(16384, math.ceil(len(selected) / 128) * 128))
+    for token in list(counts) + torch.load(BASELINE / "mtp_draft_vocab_ids.pt", weights_only=True).tolist():
+        if len(selected) >= target_size:
+            break
+        selected.add(token)
+    assert len(selected) == target_size
+    ids = sorted(selected)
+    report = {"size": len(ids), "calibration_tokens": total,
+              "calibration_coverage": sum(counts[t] for t in ids) / total,
+              "holdout_tokens": sum(held.values()),
+              "holdout_coverage": sum(held[t] for t in ids) / sum(held.values()),
+              "selection": "99.8% calibration output-token coverage, min 16384 and max 65536; rounded to 128 rows; all special ids included; reference IDs only pad if calibration vocabulary is too small"}
+    write_json(RUN / "calibration/draft_vocab_ids.json", ids)
+    write_json(RUN / "calibration/vocabulary-report.json", report)
+    print(json.dumps(report), flush=True)
+
+
+def replace_tensors(model, replacements, bits):
+    index = read_json(model / "model.safetensors.index.json")
+    grouped = collections.defaultdict(dict)
+    for module, tensors in replacements.items():
+        shard = index["weight_map"][module + ".weight_packed"]
+        grouped[shard][module] = tensors
+    for shard, modules in grouped.items():
+        # Never open a shared inode for writing: replace via a new file.
+        with safe_open(model / shard, "pt") as f:
+            data = {k: f.get_tensor(k) for k in f.keys()
+                    if not any(k.startswith(m + ".") for m in modules)}
+        for m, tensors in modules.items():
+            data.update({m + "." + k: v for k, v in tensors.items()})
+        temporary = model / (shard + ".tmp")
+        save_file(data, temporary, metadata={"format": "pt"})
+        os.replace(temporary, model / shard)
+        del data
+    config = read_json(model / "config.json")
+    for group in bits:
+        config["quantization_config"]["config_groups"][group]["weights"]["num_bits"] = bits[group]
+    write_json(model / "config.json", config)
+    if (model / "quantization_config.json").exists():
+        write_json(model / "quantization_config.json", config["quantization_config"])
+
+
+def packed(q, scale):
+    return {"weight_packed": pack_to_int32(q.cpu(), 4, packed_dim=1).contiguous(),
+            "weight_scale": scale.cpu().to(torch.float16).contiguous(),
+            "weight_shape": torch.tensor(list(q.shape), dtype=torch.int64)}
+
+
+def output_head(calib_rows):
+    clone(BASELINE, FAST)
+    hiddens = np.load(RUN / "calibration/hidden.npy", mmap_mode="r")
+    seqs = read_json(RUN / "calibration/seqs.json")
+    rng = np.random.default_rng(15027)
+    pools = {"calibration": [], "holdout": []}
+    for seq in seqs:
+        # Include output-producing states; exclude prompt body and final nonpredicting row.
+        start = seq["off"] + max(0, seq["n_prompt"] - 1)
+        end = seq["off"] + seq["n"] - 1
+        if end > start:
+            pools[seq["split"]].append(np.arange(start, end, dtype=np.int64))
+    selected = {}
+    for split, size in [("calibration", calib_rows), ("holdout", 1024)]:
+        pool = np.concatenate(pools[split])
+        selected[split] = np.sort(rng.choice(pool, size=min(size, len(pool)), replace=False))
+    assert not np.intersect1d(selected["calibration"], selected["holdout"]).size
+    W = original("lm_head.weight").cuda()
+    H = torch.zeros(W.shape[1], W.shape[1], device="cuda")
+    seen = 0
+    for start in range(0, len(selected["calibration"]), 8192):
+        x = torch.from_numpy(np.array(hiddens[selected["calibration"][start:start+8192]])).view(torch.bfloat16).cuda()
+        H, seen = accumulate_hessian(H, x, seen)
+    del x
+    held = torch.from_numpy(np.array(hiddens[selected["holdout"]])).view(torch.bfloat16).cuda()
+
+    @torch.no_grad()
+    def kl(q, scale):
+        dq = dequant(q.cuda(), scale.cuda()).to(torch.bfloat16)
+        total = 0.
+        for start in range(0, len(held), 64):
+            x = held[start:start+64]
+            p = (x @ W.t()).float().log_softmax(-1)
+            lp = (x @ dq.t()).float().log_softmax(-1)
+            total += (p.exp() * (p-lp)).sum().item()
+        del dq
+        return total / len(held)
+
+    q, s = rtn_quantize(W, 4, 128)
+    rtn_kl = kl(q, s)
+    del q, s
+    torch.cuda.empty_cache()
+    qs, scales = [], []
+    for start in range(0, W.shape[0], 8192):
+        q, s = gptq_quantize(W[start:start+8192], H, bits=4, group=128)
+        qs.append(q.cpu()); scales.append(s.cpu())
+        print("lm_head rows", start + len(q), "/", W.shape[0], flush=True)
+        del q, s
+    q, s = torch.cat(qs), torch.cat(scales)
+    gptq_kl = kl(q, s)
+    assert math.isfinite(gptq_kl), gptq_kl
+    report = {"calibration_rows": seen, "holdout_rows": len(held),
+              "rtn_int4_kl": rtn_kl, "gptq_int4_kl": gptq_kl,
+              "original": "pristine source lm_head BF16", "row_split": "whole examples before generation"}
+    print(json.dumps(report), flush=True)
+    # A worse result must be reviewed rather than silently promoted.
+    if gptq_kl > rtn_kl:
+        raise RuntimeError("GPTQ failed to improve held-out KL over round-to-nearest")
+    replace_tensors(FAST, {"lm_head": packed(q, s)}, {"group_1": 4})
+    write_json(RUN / "calibration/lm-head-report.json", report)
+
+
+def mtp():
+    hessians = torch.load(RUN / "calibration/mtp_hessians.pt", weights_only=True, map_location="cpu")
+    replacements, report = {}, {}
+    for module, hessian in hessians.items():
+        W = original(module + ".weight").cuda()
+        q, scale = gptq_quantize(W, hessian.cuda(), bits=4, group=128)
+        error = ((dequant(q, scale) - W.float()).norm() / W.float().norm()).item()
+        assert math.isfinite(error)
+        report[module] = {"relative_weight_error": error, "shape": list(W.shape)}
+        replacements[module] = packed(q, scale)
+        print(module, error, flush=True)
+        del W, q, scale
+        torch.cuda.empty_cache()
+    assert len(replacements) == 8, list(replacements)
+    replace_tensors(FAST, replacements, {"group_3": 4})
+    write_json(RUN / "calibration/mtp-report.json", report)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("stage", choices=["vocabulary", "lm-head", "mtp"])
+    ap.add_argument("--calib-rows", type=int, default=300000)
+    args = ap.parse_args()
+    if args.stage == "vocabulary": vocabulary()
+    elif args.stage == "lm-head": output_head(args.calib_rows)
+    else: mtp()

evaluation/code/swift15/reference.py

@@ -0,0 +1,41 @@
+"""Repair the reference's embedding/index mismatch in a separate evaluation copy."""
+import shutil
+import os
+import subprocess
+import sys
+from common import ROOT, RUN, read_json, write_json, stamp
+from checkpoint import clone, audit
+
+SOURCE = ROOT / "models/Qwen3.8-27B-W4A16-AutoRound-fast"
+TARGET = ROOT / "models/Qwen3.8-27B-W4A16-AutoRound-fast-eval"
+
+
+def main():
+    if (TARGET / "embedding-repair.json").exists():
+        audit(TARGET)
+        return
+    clone(SOURCE, TARGET)
+    index = read_json(TARGET / "model.safetensors.index.json")
+    prefix = "model.language_model.embed_tokens"
+    shard = index["weight_map"][prefix + ".weight_packed"]
+    from safetensors import safe_open
+    with safe_open(TARGET / shard, "pt") as file:
+        assert prefix + ".weight" in file.keys()
+        assert prefix + ".weight_packed" not in file.keys()
+    # The legacy converter writes in place; detach this shard from source inodes.
+    temporary = TARGET / (shard + ".private")
+    shutil.copyfile(TARGET / shard, temporary)
+    os.replace(temporary, TARGET / shard)
+    for suffix in ["weight_packed", "weight_scale", "weight_shape"]:
+        del index["weight_map"][prefix + "." + suffix]
+    index["weight_map"][prefix + ".weight"] = shard
+    write_json(TARGET / "model.safetensors.index.json", index)
+    subprocess.run([sys.executable, str(ROOT / "prepare/quant_embed.py"), str(TARGET)], check=True)
+    audit(TARGET)
+    write_json(TARGET / "embedding-repair.json", {"created": stamp(), "source": str(SOURCE),
+        "reason": "Source index declares INT8 embeddings but its shard contains BF16 embeddings.",
+        "change": "Apply the repository's standard INT8 group128 embedding conversion in an isolated copy; source preserved."})
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/release.py

@@ -0,0 +1,133 @@
+"""Prepare an auditable local release; this script never uploads anything."""
+import json
+import shutil
+from pathlib import Path
+from safetensors import safe_open
+from common import ROOT, RUN, SOURCE, BASELINE, FAST, read_json, write_json, sha256, stamp
+from checkpoint import audit
+
+
+def verify_body():
+    import torch
+    source = read_json(SOURCE / "model.safetensors.index.json")["weight_map"]
+    target = read_json(FAST / "model.safetensors.index.json")["weight_map"]
+    count = 0
+    for name, shard in source.items():
+        if name.startswith(("mtp.", "lm_head.", "model.language_model.embed_tokens.")):
+            continue
+        a, b = SOURCE / shard, FAST / target[name]
+        if a.stat().st_ino != b.stat().st_ino or a.stat().st_dev != b.stat().st_dev:
+            with safe_open(a,"pt") as left, safe_open(b,"pt") as right:
+                assert torch.equal(left.get_tensor(name),right.get_tensor(name)), f"Unintended body change: {name}"
+        count += 1
+    return count
+
+
+def main():
+    unchanged = verify_body()
+    index = read_json(FAST / "model.safetensors.index.json")
+    sizes = {"F64": 8, "F32": 4, "F16": 2, "BF16": 2, "I64": 8, "I32": 4, "I16": 2, "I8": 1, "U8": 1, "BOOL": 1}
+    import math
+    total = 0
+    for shard in set(index["weight_map"].values()):
+        with safe_open(FAST / shard, "pt") as f:
+            for name in f.keys():
+                tensor = f.get_slice(name)
+                total += math.prod(tensor.get_shape()) * sizes[tensor.get_dtype()]
+    index.setdefault("metadata", {})["total_size"] = total
+    write_json(FAST / "model.safetensors.index.json", index)
+    result = audit(FAST)
+    provenance = FAST / "hyperqwen_provenance"
+    provenance.mkdir(exist_ok=True)
+    for name in ["LICENSE", "LICENSE-APACHE-2.0", "NOTICE"]:
+        assert (SOURCE / name).exists(), f"Missing upstream {name}"
+        shutil.copy2(SOURCE / name, FAST / name)
+    for name in ["README.md", "QUANTIZATION_MANIFEST.json", "UPLOAD_MANIFEST.json", "recipe.yaml"]:
+        if (SOURCE / name).exists():
+            shutil.copy2(SOURCE / name, provenance / ("upstream-" + name))
+    for name in ["manifest.json", "generation-manifest.json", "generation-summary.json", "vocabulary-report.json", "lm-head-report.json", "mtp-report.json", "mtp_hessians.pt.json"]:
+        shutil.copy2(RUN / "calibration" / name, provenance / name)
+    for source in list((ROOT/"swift15").glob("*.py")) + [ROOT/"swift15/README.md", ROOT/"drafter/gptq_utils.py",
+            ROOT/"drafter/capture.py", ROOT/"drafter/train_mtp.py", ROOT/"prepare/build_draft_vocab.py", ROOT/"prepare/quant_heads_stream.py"]:
+        destination = provenance/"recipe"/source.relative_to(ROOT)
+        destination.parent.mkdir(parents=True,exist_ok=True)
+        shutil.copy2(source,destination)
+    for name in ["runtime-compatibility.json", "preflight-validation.json"]:
+        if (RUN / name).exists():
+            shutil.copy2(RUN / name, provenance / name)
+    mtp_replay = read_json(RUN / "calibration/mtp_hessians.pt.json")
+    recipe = {"created": stamp(), "source": read_json(RUN / "source.json"), "variant": "fast",
+              "preflight_only": "preflight" in RUN.parts,
+              "unchanged_body_tensors_verified": unchanged,
+              "body": "unchanged asymmetric AWQ INT4 group128", "embedding_bits": 8,
+              "lm_head_bits": 4, "mtp_bits": 4, "group_size": 128, "act_order": False,
+              "gptq": {"damping": .01, "lm_head_rows": read_json(RUN / "calibration/lm-head-report.json")["calibration_rows"],
+                       "mtp_examples": len(mtp_replay["examples"]), "mtp_depths": mtp_replay["depths"]},
+              "pristine_inputs": "Heads quantized directly from source BF16 tensors, not from INT8 tensors",
+              "calibration": read_json(RUN / "calibration/manifest.json"),
+              "runtime": read_json(RUN / "environment.json"),
+              "script_hashes": {str(p.relative_to(ROOT)): sha256(p) for p in list((ROOT / "swift15").glob("*.py")) +
+                  [ROOT / "drafter/gptq_utils.py", ROOT / "drafter/capture.py", ROOT / "drafter/train_mtp.py", ROOT / "prepare/build_draft_vocab.py"]}}
+    write_json(FAST / "hyperqwen-build.json", recipe)
+    write_json(FAST / "QUANTIZATION_MANIFEST.json", recipe)
+    # The inherited upload list describes the upstream checkpoint, not this one.
+    if (FAST / "UPLOAD_MANIFEST.json").exists():
+        (FAST / "UPLOAD_MANIFEST.json").unlink()
+    with (FAST / "NOTICE").open("a") as out:
+        out.write("\nHyperQwen-compatible local derivative: INT8 embeddings; calibrated GPTQ INT4 output and MTP linear weights; reduced MTP draft vocabulary. The upstream AWQ body is unchanged. Quantization recipe and source attribution are included in hyperqwen-build.json.\n")
+    report = RUN / "REPORT.md"
+    measured = report.read_text() if report.exists() else "Evaluation is pending. No throughput or quality claim has been established for this derivative."
+    card = """---
+license: other
+license_name: swift-open-license-1.0
+license_link: LICENSE
+base_model: ukisai/Swift-1.5-Qwen3.8-27b
+base_model_relation: quantized
+library_name: vllm
+pipeline_tag: text-generation
+tags:
+- compressed-tensors
+- awq
+- gptq
+- hyperqwen
+---
+
+# Swift 1.5 Qwen3.8 27B — HyperQwen fast derivative
+
+Local release candidate. This is an independently prepared quantization of UkisAI's
+Swift 1.5, not an official UkisAI or HyperQwen release.
+
+The original asymmetric AWQ INT4 body is unchanged. Embeddings use INT8 group128;
+the full output head and eight MTP linear matrices use calibrated GPTQ INT4 group128.
+GPTQ starts from the upstream BF16 head tensors. The MTP draft head uses a subset
+of the output head selected using fresh Swift responses. This subset only limits
+draft proposals; the target retains its full vocabulary. This is not fine-tuning.
+
+Requires the patched HyperQwen runtime, including INT8 embeddings and reduced MTP
+vocabulary support. The tested package versions and recipe are in hyperqwen-build.json.
+Do not assume an unpatched Transformers/vLLM installation can load this checkpoint.
+Multi-user serving can use the W4A16 body without MTP. Optional INT8 activations are
+a separate runtime setting and require asymmetric Marlin support; they are not a
+property of the stored checkpoint.
+
+The upstream vision tensors are retained, but the initial evaluation is text-only.
+Calibration sources, pinned revisions and whole-example holdout separation are
+documented under hyperqwen_provenance. Raw benchmark answers and calibration
+examples are not included. See LICENSE, LICENSE-APACHE-2.0 and NOTICE for the
+upstream license and attribution. See hyperqwen_provenance/upstream-README.md for
+the original model information.
+
+## Local evaluation
+
+""" + measured + "\n"
+    if recipe["preflight_only"]:
+        card = card.replace("Local release candidate.", "Integration-test checkpoint only; exclude this small-subset preflight model from publication and final comparisons.")
+    (FAST / "README.md").write_text(card)
+    files = sorted(p.relative_to(FAST).as_posix() for p in FAST.rglob("*") if p.is_file()
+                   and ".bak" not in p.name and not p.name.endswith(".tmp") and ".cache" not in p.parts)
+    write_json(RUN / "release-files.json", files)
+    print("Prepared local release:", FAST, ";", len(files), "files; no upload performed")
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/report.py

@@ -0,0 +1,68 @@
+"""Generate the comparison only from completed measurements."""
+import math
+from common import RUN, read_json, stamp
+
+
+def interval(correct, n):
+    z = 1.96
+    p = correct / n
+    center = (p + z*z/(2*n))/(1+z*z/n)
+    radius = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/(1+z*z/n)
+    return f"{100*p:.1f}% ({100*(center-radius):.1f}–{100*(center+radius):.1f})"
+
+
+def main():
+    lines = ["# Local Swift / HyperQwen comparison", "", "Updated: " + stamp(), "",
+             "RTX 3090 measurements. A task is one independently scored problem; tool tasks include every model call. "
+             "Output tokens include reasoning. Latency is observed serving time, not a GPU-kernel compute measurement. "
+             "Quality runs use bounded budgets and one seed (15027). Thinking tasks use temperature 1.0, top_p 0.95, top_k 20; nonthinking tasks use greedy decoding. These are not publisher leaderboard scores.", "",
+             "The Qwen reference is the existing AutoRound fast checkpoint; Swift 1.0 is the existing AWQ/INT8-head checkpoint. "
+             "The Swift 1.5 INT8-head baseline and INT4-head fast derivative share the same upstream AWQ body. "
+             "A BF16 reference measurement is outside this four-checkpoint comparison.", "",
+             "## Fixed-output speed and perplexity", "",
+             "512 output tokens/request, concurrency 1; natural task lengths are reported separately.", "",
+             "| Model | Decode tok/s (median) | TTFT (s) | EN PPL | DA PPL | Code PPL |",
+             "|---|---:|---:|---:|---:|---:|"]
+    tags = ["qwen-fast", "swift10", "swift15-baseline", "swift15-fast"]
+    for tag in tags:
+        directory = RUN / "results" / tag
+        speed = read_json(directory / "speed-c1.json") if (directory / "speed-c1.json").exists() else {}
+        ppl = read_json(directory / "perplexity.json").get("scores", {}) if (directory / "perplexity.json").exists() else {}
+        def f(v): return "pending" if v is None else f"{v:.3f}"
+        lines.append("| " + " | ".join([tag, f(speed.get("median_decode_tps")), f(speed.get("median_ttft_seconds")),
+                                      *[f(ppl.get(k, {}).get("ppl")) for k in ["en", "da", "code"]]]) + " |")
+    lines += ["", "## Swift 1.5 multi-user serving", "",
+              "Same fast checkpoint, no MTP, concurrency eight. INT8 applies to MLP activations only.", "",
+              "| Runtime | Aggregate output tok/s | Combined PPL |", "|---|---:|---:|"]
+    for tag in ["swift15-fast-batch", "swift15-fast-batch-int8"]:
+        directory = RUN / "results" / tag
+        if not (directory / "speed-c8.json").exists(): continue
+        speed = read_json(directory / "speed-c8.json")
+        ppl = read_json(directory / "perplexity.json")["scores"]["all"]["ppl"] if (directory / "perplexity.json").exists() else None
+        lines.append(f"| {tag} | {speed['aggregate_output_tps']:.2f} | {ppl if ppl is not None else 'pending'} |")
+    for mode in ["pilot", "full"]:
+        lines += ["", "## " + ("Sequential task latency pilot" if mode == "pilot" else "Quality suite (concurrency recorded per run)"), "",
+                  "Accuracy includes all attempts. Truncated responses count as wrong; their count is reported separately. Parentheses show descriptive 95% Wilson intervals. Small differences are inconclusive.", "",
+                  "| Model | Task suite | Correct/total | Accuracy (95% CI) | Output tokens/task | Total tokens/task | Request seconds/task | Truncated | Errors |",
+                  "|---|---|---:|---:|---:|---:|---:|---:|---:|"]
+        for tag in tags:
+            path = RUN / "results" / (tag + "-" + mode) / "summary.json"
+            if not path.exists(): continue
+            summary = read_json(path)
+            for suite, values in summary["suites"].items():
+                v = values
+                accuracy = interval(v['correct'],v['attempted'])
+                lines.append(f"| {tag} | {suite} | {v['correct']}/{v['attempted']} | {accuracy} | "
+                             f"{v['mean_output_tokens']:.1f} | {v['mean_total_tokens']:.1f} | {v['mean_model_seconds']:.2f} | {v['truncated']} | {v['errors']} |")
+    lines += ["", "Full-suite request latencies overlap and must not be summed as GPU time. "
+              "Per-run JSON summaries also report wall-clock makespan and wall seconds per correct task, including failed attempts. "
+              "Resumed runs preserve attempt timings and omit a misleading whole-suite makespan.", "",
+              "LiveCodeBench is a fixed 100-problem, stdin-only v6 subset with a custom deterministic judge, "
+              "not an official full LCB score. IFBench uses the official instruction verifier. "
+              "The 20-task latency pilot has five tasks per category and is too small to establish quality equivalence.", ""]
+    (RUN / "REPORT.md").write_text("\n".join(lines))
+    print(RUN / "REPORT.md")
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/run.py

@@ -0,0 +1,162 @@
+"""Resumable build and four-checkpoint evaluation, one GPU owner at a time."""
+import argparse
+import fcntl
+import importlib.metadata
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+from common import ROOT, RUN, SOURCE, BASELINE, FAST, read_json, write_json, records, sha256, stamp
+from serve import server
+
+PYTHON = str(ROOT / "venv/bin/python")
+
+
+def stage(name, command, outputs, env=None):
+    markers = RUN / "stages"
+    marker = markers / (name + ".json")
+    signature = {"command": command, "env": env or {}}
+    signature["script_hashes"] = {part: sha256(ROOT / part) for part in command if part.endswith(".py") and (ROOT / part).is_file()}
+    if marker.exists():
+        old = read_json(marker)
+        assert old["signature"] == signature, f"Stage settings changed: {name}"
+        assert all(p.exists() for p in outputs), f"Stage output missing: {name}"
+        print("Already complete:", name, flush=True)
+        return
+    write_json(RUN / "status.json", {"stage": name, "state": "running", "started": stamp()})
+    print("Starting:", name, flush=True)
+    log = RUN / "logs" / (name + ".log")
+    log.parent.mkdir(parents=True, exist_ok=True)
+    started = time.monotonic()
+    with log.open("a") as out:
+        process = subprocess.Popen(command, cwd=ROOT, env=dict(os.environ, OMP_NUM_THREADS="8", **(env or {})),
+                                   stdout=out, stderr=subprocess.STDOUT, start_new_session=True)
+        try:
+            code = process.wait()
+            if code:
+                raise subprocess.CalledProcessError(code, command)
+        except BaseException:
+            try: os.killpg(process.pid, signal.SIGTERM)
+            except ProcessLookupError: pass
+            try: process.wait(timeout=45)
+            except subprocess.TimeoutExpired:
+                try: os.killpg(process.pid, signal.SIGKILL)
+                except ProcessLookupError: pass
+                process.wait(timeout=15)
+            raise
+    assert all(p.exists() for p in outputs), f"Missing outputs after {name}"
+    write_json(marker, {"signature": signature, "completed": stamp(), "seconds": time.monotonic()-started,
+                        "outputs": {str(p.relative_to(ROOT)): p.stat().st_size for p in outputs}})
+
+
+def build():
+    cal = RUN / "calibration"
+    stage("baseline", [PYTHON, "swift15/checkpoint.py", "baseline"], [BASELINE / "hyperqwen-build.json"])
+    stage("generation", [PYTHON, "-u", "swift15/generate.py"], [cal / "generation-summary.json"])
+    assert len(records(cal / "gen.jsonl")) == len(records(cal / "prompts.jsonl")), "Incomplete generation"
+    stage("vocabulary", [PYTHON, "swift15/quantize.py", "vocabulary"], [cal / "draft_vocab_ids.json"])
+    stage("capture", [PYTHON, "-u", "drafter/capture.py"], [cal / "hidden.npy", cal / "seqs.json"],
+          {"MODEL": str(BASELINE), "CALIBRATION_DATA": str(cal)})
+    stage("lm-head", [PYTHON, "-u", "swift15/quantize.py", "lm-head"], [cal / "lm-head-report.json"])
+    stage("draft-head", [PYTHON, "prepare/build_draft_vocab.py", str(FAST), "--ids", str(cal / "draft_vocab_ids.json")],
+          [FAST / "mtp_draft_vocab_ids.pt"])
+    stage("mtp-hessians", [PYTHON, "-u", "drafter/train_mtp.py", "--model", str(BASELINE),
+          "--original-model", str(SOURCE), "--data", str(cal), "--out", str(cal / "mtp-replay"),
+          "--eval-only", "1", "--dump-hessians", str(cal / "mtp_hessians.pt"),
+          "--max-seqs", "400", "--val-frac", ".4", "--depths", "2", "--micro-tokens", "1",
+          "--head-chunk", "128", "--draft-ids", str(cal / "draft_vocab_ids.json")], [cal / "mtp_hessians.pt"])
+    stage("mtp-quant", [PYTHON, "-u", "swift15/quantize.py", "mtp"], [cal / "mtp-report.json"])
+    stage("release", [PYTHON, "swift15/release.py"], [FAST / "hyperqwen-build.json", FAST / "hyperqwen-audit.json"])
+
+
+def evaluate(full):
+    evaluation_manifest = read_json(RUN / "evaluation/manifest.json")
+    quality_concurrency = int(evaluation_manifest.get(
+        "quality_concurrency", os.environ.get("SWIFT15_EVAL_CONCURRENCY", "8")))
+    if quality_concurrency < 1:
+        raise ValueError("Quality concurrency must be positive")
+    reference = ROOT / "models/Qwen3.8-27B-W4A16-AutoRound-fast-eval"
+    stage("reference-embedding-repair", [PYTHON, "swift15/reference.py"], [reference / "embedding-repair.json"])
+    models = {"qwen-fast": reference,
+              "swift10": ROOT / "models/Swift-Qwen3.8-27b-W4A16-AWQ",
+              "swift15-baseline": BASELINE, "swift15-fast": FAST}
+    for tag, model in models.items():
+        finished = RUN / "stages" / (tag + ("-full" if full else "-pilot") + ".json")
+        if finished.exists():
+            continue
+        with server(model, tag):
+            stage(tag + "-speed", [PYTHON, "swift15/measure.py", tag, "--kind", "speed"],
+                  [RUN / "results" / tag / "speed-c1.json"])
+            stage(tag + "-ppl", [PYTHON, "swift15/measure.py", tag, "--kind", "ppl"],
+                  [RUN / "results" / tag / "perplexity.json"])
+            stage(tag + "-pilot-tasks", [PYTHON, "-u", "swift15/evaluate.py", tag + "-pilot", "--pilot"],
+                  [RUN / "results" / (tag + "-pilot") / "summary.json"])
+            if full:
+                stage(tag + "-full-tasks", [PYTHON, "-u", "swift15/evaluate.py", tag + "-full", "--concurrency", str(quality_concurrency)],
+                      [RUN / "results" / (tag + "-full") / "summary.json"])
+        write_json(finished, {"completed": stamp()})
+        subprocess.run([PYTHON, "swift15/report.py"], cwd=ROOT, check=True)
+    if os.environ.get("SWIFT15_SINGLE_USER_ONLY") == "1":
+        return
+    stage("runtime-compat", [PYTHON, "swift15/runtime_compat.py"], [RUN / "runtime-compatibility.json"])
+    for tag, int8 in [("swift15-fast-batch", False), ("swift15-fast-batch-int8", True)]:
+        finished = RUN / "stages" / (tag + ".json")
+        if finished.exists():
+            continue
+        with server(FAST, tag, batch=True, int8=int8):
+            stage(tag + "-speed", [PYTHON, "swift15/measure.py", tag, "--kind", "speed", "--concurrency", "8"],
+                  [RUN / "results" / tag / "speed-c8.json"])
+            stage(tag + "-ppl", [PYTHON, "swift15/measure.py", tag, "--kind", "ppl"],
+                  [RUN / "results" / tag / "perplexity.json"])
+            stage(tag + "-tasks", [PYTHON, "-u", "swift15/evaluate.py", tag, "--pilot", "--concurrency", "8"],
+                  [RUN / "results" / tag / "summary.json"])
+        write_json(finished, {"completed": stamp(), "activation_dtype": "INT8 MLP only" if int8 else "BF16"})
+    subprocess.run([PYTHON, "swift15/report.py"], cwd=ROOT, check=True)
+    if (RUN / "calibration/manifest.json").exists():
+        subprocess.run([PYTHON, "swift15/release.py"], cwd=ROOT, check=True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("mode", choices=["build", "pilot", "full"])
+    ap.add_argument("--evaluation-only", action="store_true", help="Evaluate existing checkpoints in a separately prepared run directory")
+    ap.add_argument("--wait-for-pid", type=int, help="Wait for an already-running calibration process before acquiring the GPU")
+    args = ap.parse_args()
+    def stop(signum, frame):
+        raise KeyboardInterrupt("Pipeline terminated")
+    signal.signal(signal.SIGTERM, stop)
+    RUN.mkdir(parents=True, exist_ok=True)
+    with (RUN / "pipeline.lock").open("w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        if args.wait_for_pid:
+            print("Waiting for existing calibration process", args.wait_for_pid, flush=True)
+            while os.path.exists(f"/proc/{args.wait_for_pid}"):
+                time.sleep(5)
+        hardware = subprocess.check_output(["nvidia-smi", "--query-gpu=name,uuid,memory.total,power.limit,driver_version", "--format=csv"], text=True)
+        versions = {name: importlib.metadata.version(name) for name in ["torch", "vllm", "transformers", "safetensors", "compressed-tensors"]}
+        commit = subprocess.check_output(["git", "rev-parse", "HEAD"],cwd=ROOT,text=True).strip()
+        write_json(RUN / "environment.json", {"started": stamp(), "hardware": hardware, "versions": versions,
+                   "hyperqwen_commit":commit,"python":sys.version,
+                   "patches_sha256":{p.name:sha256(p) for p in (ROOT/"patches").glob("*.patch")}})
+        try:
+            if not args.evaluation_only:
+                build()
+            if args.mode != "build":
+                evaluate(full=args.mode == "full")
+            truncated = {}
+            for path in (RUN / "results").glob("*/summary.json"):
+                result = read_json(path)
+                if result.get("truncated_task_ids"):
+                    truncated[path.parent.name] = result["truncated_task_ids"]
+            write_json(RUN / "status.json", {"state": "complete",
+                       "mode": args.mode, "completed": stamp(), "truncated_counted_as_wrong": truncated})
+        except BaseException as error:
+            previous = read_json(RUN / "status.json") if (RUN / "status.json").exists() else {}
+            write_json(RUN / "status.json", dict(previous, state="failed", error=repr(error), failed=stamp()))
+            raise
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/runtime_compat.py

@@ -0,0 +1,41 @@
+"""Apply HyperQwen's two asymmetric INT4/INT8 Marlin guards idempotently.
+
+Reference: https://github.com/syv-ai/HyperQwen/blob/main/patches/marlin-int8-asym-zp.patch
+Backups and before/after hashes are recorded. No kernels are replaced.
+"""
+import importlib.util
+import shutil
+from common import RUN, write_json, sha256
+from pathlib import Path
+
+
+def main():
+    root = Path(importlib.util.find_spec("vllm").origin).parent
+    changes = [
+        ("model_executor/kernels/linear/mixed_precision/marlin.py",
+         'assert c.weight_type == scalar_types.uint4b8, (\n                "W8A8 is not supported',
+         'assert c.weight_type in (scalar_types.uint4b8, scalar_types.uint4), (\n                "W8A8 is not supported'),
+        ("model_executor/layers/quantization/utils/marlin_utils.py",
+         'assert wtype == scalar_types.uint4b8, (\n            "W8A8-INT8 is not supported',
+         'assert wtype in (scalar_types.uint4b8, scalar_types.uint4), (\n            "W8A8-INT8 is not supported')]
+    plan = []
+    for relative, old, new in changes:
+        path = root / relative
+        text = path.read_text()
+        assert text.count(old) == 1 or text.count(new) == 1, f"Unexpected installed code: {path}"
+        plan.append((path, old, new, text))
+    report = []
+    for path, old, new, text in plan:
+        before = sha256(path)
+        backup = path.with_name(path.name + ".swift15-before")
+        if old in text:
+            assert not backup.exists(), f"Backup exists but patch is absent: {backup}"
+            shutil.copy2(path, backup)
+            path.write_text(text.replace(old, new))
+        report.append({"path": str(path), "before": before, "after": sha256(path), "backup": str(backup)})
+    write_json(RUN / "runtime-compatibility.json", report)
+    print("Asymmetric INT4 + INT8-activation guards installed")
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/serve.py

@@ -0,0 +1,103 @@
+"""Own one benchmark server process group; leave unrelated services alone."""
+import contextlib
+import json
+import os
+import signal
+import subprocess
+import time
+import urllib.request
+from common import ROOT, RUN, write_json, sha256, stamp
+from evaluate import key
+
+
+@contextlib.contextmanager
+def server(model, tag, batch=False, int8=False):
+    context_len = int(os.environ.get("SWIFT15_CONTEXT_LEN", "16384"))
+    speculation = os.environ.get("SWIFT15_SPEC", "mtp")
+    if speculation not in {"mtp", "off"}:
+        raise ValueError("SWIFT15_SPEC must be mtp or off")
+    write_json(RUN / "status.json", {"stage": tag + "-server", "state": "starting", "started": stamp()})
+    log = RUN / "logs" / (tag + "-server.log")
+    log.parent.mkdir(parents=True, exist_ok=True)
+    env = dict(os.environ, MODEL=str(model), HOST="127.0.0.1", PORT="18021",
+               VISION="0", CTX="fast", SPEC=speculation, MAX_LEN=str(context_len), MAX_SEQS="8",
+               GPU_UTIL="0.93", API_SERVERS="1", PREFIX_CACHE="0", INT8_ACT="int8" if int8 else "",
+               INT8_LAYERS="mlp", OMP_NUM_THREADS="8", EXTRA_ARGS="--generation-config vllm",
+               VLLM_OFFLOAD_KEEP_SHM="1")
+    for name in ["VLLM_MARLIN_INPUT_DTYPE", "VLLM_MARLIN_INT8_INCLUDE_RE", "VLLM_PREFILL_ATTN"]:
+        env.pop(name, None)
+    if not int8 and (ROOT / ".env").exists():
+        for line in (ROOT / ".env").read_text().splitlines():
+            if line.startswith("INT8_ACT=") and line.split("=", 1)[1].strip(' "'):
+                raise RuntimeError("Local .env enables INT8_ACT; remove that default before the controlled W4A16 comparison")
+    launcher = ROOT / ("batch/start_qwen.sh" if batch else "single-user/start_qwen.sh")
+    if batch:
+        env["EXTRA_ARGS"] += " --attention-backend FLASH_ATTN --kv-cache-dtype bfloat16 --no-enable-prefix-caching"
+    local_profile = os.environ.get("SWIFT15_SERVING_PROFILE") == "local-single-user" and not batch
+    if local_profile:
+        # Match the launcher's .env semantics, retaining all local runtime knobs.
+        env = dict(os.environ)
+        for line in (ROOT / ".env").read_text().splitlines():
+            if not line or line.startswith("#"):
+                continue
+            name, sep, value = line.removeprefix("export ").partition("=")
+            if sep and name.isidentifier() and not env.get(name):
+                env[name] = value.strip('"')
+        context_len = 150000
+        env.update(MODEL=str(model), HOST="127.0.0.1", PORT="18021",
+                   CTX="long", MAX_LEN=str(context_len))
+        if env.get("SPEC", "mtp") != "mtp":
+            raise ValueError("The FP8 local profile requires SPEC=mtp")
+        speculation = "mtp"
+    import socket
+    with socket.socket() as s:
+        if s.connect_ex(("127.0.0.1", 18021)) == 0:
+            raise RuntimeError("Benchmark port 18021 is occupied; refusing to use or stop another server")
+    with log.open("w") as out:
+        p = subprocess.Popen(["bash", str(launcher)], cwd=ROOT, env=env, stdout=out, stderr=subprocess.STDOUT,
+                             start_new_session=True)
+        identity = {"created": stamp(), "model": str(model), "config_sha256": sha256(model / "config.json"),
+                    "index_sha256": sha256(model / "model.safetensors.index.json"), "launcher_sha256": sha256(launcher),
+                    "tokenizer_sha256": sha256(model / "tokenizer.json"),
+                    "chat_template_sha256": sha256(model / "chat_template.jinja") if (model / "chat_template.jinja").exists() else sha256(model / "tokenizer_config.json"),
+                    "shards": {p.name: {"size": p.stat().st_size, "mtime_ns": p.stat().st_mtime_ns} for p in model.glob("*.safetensors")},
+                    "mode": "batch" if batch else "single-user", "int8_activations": int8,
+                    "speculation": speculation,
+                    "max_model_len": context_len, "max_num_seqs": 8, "kv_cache_dtype": "bfloat16",
+                    "prefix_cache": False, "gpu_memory_utilization": .93, "pid": p.pid}
+        if local_profile:
+            identity.update(serving_profile="local-single-user", kv_cache_dtype="fp8",
+                            max_num_seqs=int(env.get("MAX_SEQS") or 8),
+                            prefix_cache=env.get("PREFIX_CACHE") == "1",
+                            vision=env.get("VISION") == "1",
+                            gpu_memory_utilization=float(env.get("GPU_UTIL") or .93),
+                            draft_tokens=int(env.get("DRAFT_TOKENS") or 3),
+                            int8_activations=bool(env.get("INT8_ACT")),
+                            local_env_sha256=sha256(ROOT / ".env"),
+                            extra_args=env.get("EXTRA_ARGS", ""))
+        write_json(RUN / "active-server.json", identity)
+        write_json(RUN / "results" / tag / "server.json", identity)
+        try:
+            deadline = time.monotonic() + 900
+            while time.monotonic() < deadline:
+                if p.poll() is not None:
+                    raise RuntimeError(f"Server exited {p.returncode}; see {log}")
+                try:
+                    req = urllib.request.Request("http://127.0.0.1:18021/health", headers={"Authorization": "Bearer " + key()})
+                    with urllib.request.urlopen(req, timeout=3) as response:
+                        if response.status == 200:
+                            break
+                except OSError:
+                    time.sleep(2)
+            else:
+                raise TimeoutError(f"Server startup timed out; see {log}")
+            yield identity
+        finally:
+            try: os.killpg(p.pid, signal.SIGTERM)
+            except ProcessLookupError: pass
+            try: p.wait(timeout=45)
+            except subprocess.TimeoutExpired:
+                try: os.killpg(p.pid, signal.SIGKILL)
+                except ProcessLookupError: pass
+                p.wait(timeout=15)
+            write_json(RUN / "active-server.json", dict(identity, stopped=stamp(), returncode=p.returncode))

evaluation/code/swift15/setup.py

@@ -0,0 +1,42 @@
+"""Download pinned public inputs and install the isolated official verifier."""
+import hashlib
+import io
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+from huggingface_hub import snapshot_download
+from common import ROOT, RUN, SOURCE, write_json, stamp
+
+REPO = "ukisai/Swift-1.5-Qwen3.8-27b-W4A16-AWQ"
+REVISION = "9dba8a05877150d587215a519ce6befec3c9978a"
+IFBENCH = "1c40f0c10d9b5c5c2f10a175a28007ebb64f7f4d"
+
+
+def main():
+    RUN.mkdir(parents=True, exist_ok=True)
+    snapshot_download(REPO, revision=REVISION, local_dir=SOURCE,
+                      allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.txt", "LICENSE*", "NOTICE", "recipe.yaml", "README.md"])
+    if not (RUN / "source.json").exists():
+        write_json(RUN / "source.json", {"repo": REPO, "revision": REVISION, "downloaded_at": stamp()})
+    env = RUN / "eval-venv"
+    if not (env / "bin/python").exists():
+        subprocess.run([sys.executable, "-m", "venv", str(env)], check=True)
+    subprocess.run([str(env / "bin/pip"), "install", "https://github.com/allenai/IFBench/archive/" + IFBENCH + ".zip"], check=True)
+    url = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt_tab.zip"
+    payload = urllib.request.urlopen(url).read()
+    with zipfile.ZipFile(io.BytesIO(payload)) as archive:
+        assert all(not n.startswith("/") and ".." not in n.split("/") for n in archive.namelist())
+        archive.extractall(RUN / "nltk_data/tokenizers")
+    subprocess.run(["docker", "image", "inspect", "python:3.12-slim"], check=True, stdout=subprocess.DEVNULL)
+    image = subprocess.check_output(["docker", "image", "inspect", "python:3.12-slim", "--format", "{{.Id}}"], text=True).strip()
+    write_json(RUN / "evaluation/runtime.json", {"code_image": image, "ifbench_revision": IFBENCH,
+               "punkt_tab_sha256": hashlib.sha256(payload).hexdigest()})
+    subprocess.run([sys.executable, "swift15/corpus.py"], cwd=ROOT, check=True)
+    subprocess.run([sys.executable, "swift15/eval_data.py"], cwd=ROOT, check=True)
+    print("Setup complete. Existing patched HyperQwen venv and Python Docker image are required.")
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/smoke.py

@@ -0,0 +1,55 @@
+"""Exercise real MTP serving, streaming, prompt logprobs, tools and scoring."""
+import argparse
+import json
+import os
+import subprocess
+from pathlib import Path
+from common import ROOT, RUN, records, write_json
+from serve import server
+from evaluate import execute, stream, post
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("model", type=Path)
+    ap.add_argument("--tag", default="smoke")
+    ap.add_argument("--batch-int8", action="store_true")
+    ap.add_argument("--check-harness", action="store_true")
+    args = ap.parse_args()
+    api = "http://127.0.0.1:18021/v1"
+    tasks = records(RUN / "evaluation/tasks.jsonl")
+    results = []
+    with server(args.model.resolve(), args.tag, batch=args.batch_int8, int8=args.batch_int8):
+        warm = stream(api, {"model": "qwen3.8-27b", "messages": [{"role":"user","content":"Reply with the word ready."}],
+                            "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}})
+        assert warm["usage"]["completion_tokens"] > 0
+        with post(api, "/completions", {"model":"qwen3.8-27b", "prompt":"A simple test of language model probabilities.",
+                                       "max_tokens":1,"temperature":0,"prompt_logprobs":0}) as response:
+            scored = json.load(response)["choices"][0]["prompt_logprobs"]
+            assert scored and all(len(entry)==1 for entry in scored[1:])
+        for suite in ["gsm8k", "tools", "ifbench", "livecodebench"]:
+            task = next(t for t in tasks if t["suite"] == suite)
+            result = execute(task, api)
+            assert result["output_tokens"] > 0, result
+            if result["error"] and result["error"] != "incorrect_tool_call":
+                raise RuntimeError(result["error"])
+            if result.get("code_score", {}).get("reason") == "sandbox_error":
+                raise RuntimeError(result["code_score"])
+            if suite == "ifbench" and result["correct"] is None:
+                scored = subprocess.run([str(RUN/"eval-venv/bin/python"),str(ROOT/"swift15/ifbench_score.py")],
+                    input=json.dumps([{"task":task,"response":result["response"]}]), text=True,capture_output=True,check=True,
+                    env=dict(os.environ,NLTK_DATA=str(RUN/"nltk_data")))
+                result.update(json.loads(scored.stdout)[0])
+            results.append(result)
+            print(suite, "correct:",result["correct"],"tokens:",result["output_tokens"],flush=True)
+        if args.check_harness:
+            subprocess.run([str(ROOT/"venv/bin/python"),str(ROOT/"swift15/evaluate.py"),args.tag+"-harness",
+                            "--pilot","--suites","tools","--concurrency","2"],check=True,cwd=ROOT)
+            subprocess.run([str(ROOT/"venv/bin/python"),str(ROOT/"swift15/measure.py"),args.tag,
+                            "--kind","speed","--concurrency","8" if args.batch_int8 else "1"],check=True,cwd=ROOT)
+    write_json(RUN/"results"/args.tag/"smoke.json",{"warmup":warm,"tasks":results,
+               "note":"Integration checks only; this small sample is not a quality benchmark."})
+
+
+if __name__ == "__main__":
+    main()

evaluation/code/swift15/test_workflow.py

@@ -0,0 +1,66 @@
+"""Regression checks for file isolation, calibration separation and task scoring."""
+import json
+import tempfile
+import unittest
+from pathlib import Path
+import torch
+from safetensors.torch import save_file, load_file
+from checkpoint import clone
+from quantize import replace_tensors
+from evaluate import gsm_score, json_equal, aggregate
+from code_runner import equal
+from gptq_utils import accumulate_hessian
+
+
+class WorkflowTests(unittest.TestCase):
+    def test_capped_answers_count_as_wrong_even_if_partial_answer_matches(self):
+        row={"input_tokens":10,"output_tokens":32768,"truncated":True,"error":None,"correct":True,"model_seconds":200}
+        result=aggregate([row])
+        self.assertEqual(result["accuracy"],0)
+        self.assertEqual(result["correct"],0)
+        self.assertEqual(result["attempted"],1)
+        self.assertEqual(result["truncated_counted_as_wrong"],1)
+
+    def test_hessian_remains_fp32_inside_bf16_autocast(self):
+        torch.manual_seed(42)
+        x = torch.randn(64,128)
+        h = torch.zeros(128,128)
+        with torch.autocast("cpu",dtype=torch.bfloat16):
+            h,n = accumulate_hessian(h,x,0)
+        reference = 2/len(x) * (x.T @ x)
+        self.assertEqual(n,64)
+        self.assertTrue(torch.allclose(h,reference,rtol=1e-5,atol=1e-6))
+
+    def test_quant_export_does_not_mutate_hardlinked_baseline(self):
+        with tempfile.TemporaryDirectory() as td:
+            source, target = Path(td)/"source", Path(td)/"fast"
+            source.mkdir()
+            old = torch.ones(2, 4, dtype=torch.int32)
+            save_file({"lm_head.weight_packed": old, "untouched.weight": torch.ones(2)}, source/"model.safetensors")
+            (source/"model.safetensors.index.json").write_text(json.dumps({"weight_map":{"lm_head.weight_packed":"model.safetensors"}}))
+            (source/"config.json").write_text(json.dumps({"quantization_config":{"config_groups":{"group_1":{"weights":{"num_bits":8}}}}}))
+            clone(source,target)
+            self.assertEqual((source/"model.safetensors").stat().st_ino,(target/"model.safetensors").stat().st_ino)
+            replace_tensors(target,{"lm_head":{"weight_packed":torch.zeros_like(old)}},{"group_1":4})
+            self.assertTrue(torch.equal(load_file(source/"model.safetensors")["lm_head.weight_packed"],old))
+            self.assertEqual(load_file(target/"model.safetensors")["lm_head.weight_packed"].sum().item(),0)
+            self.assertTrue(torch.equal(load_file(target/"model.safetensors")["untouched.weight"],torch.ones(2)))
+
+    def test_scoring_rejects_wrong_types_and_nonfinite_numbers(self):
+        self.assertFalse(json_equal({"x":True},{"x":1}))
+        self.assertFalse(json_equal(float("nan"),1))
+        self.assertFalse(equal("9007199254740993","9007199254740992"))
+        self.assertTrue(equal("1.0000001\n2","1 2"))
+        self.assertTrue(gsm_score("Final answer: 1,250", "1250"))
+        self.assertFalse(gsm_score("Final answer: 1251", "1250"))
+
+    def test_failures_remain_in_time_and_quality_denominators(self):
+        common={"input_tokens":10,"output_tokens":20,"truncated":False,"error":None}
+        result=aggregate([dict(common,correct=True,model_seconds=2),dict(common,correct=False,model_seconds=8)])
+        self.assertEqual(result["accuracy"],.5)
+        self.assertEqual(result["mean_model_seconds"],5)
+        self.assertEqual(result["summed_request_seconds_per_correct"],10)
+
+
+if __name__ == "__main__":
+    unittest.main()

evaluation/environment.json

@@ -0,0 +1,41 @@
+{
+  "started": "2026-09-28T10:01:00Z",
+  "hardware": "name, uuid, memory.total [MiB], power.limit [W], driver_version\nNVIDIA GeForce RTX 3090, GPU-9724ba4e-72d2-f72e-69e7-c21e552fe118, 24576 MiB, 250.00 W, 580.173.02\n",
+  "versions": {
+    "torch": "2.13.0",
+    "vllm": "0.27.1",
+    "transformers": "5.15.0",
+    "safetensors": "0.8.0",
+    "compressed-tensors": "0.17.0"
+  },
+  "hyperqwen_commit": "253c76aea0a240bf7cb2bd3ed92b672c0f258d0d",
+  "python": "3.12.8 | packaged by conda-forge | (main, Dec  5 2024, 14:24:40) [GCC 13.3.0]",
+  "patches_sha256": {
+    "spec-decode-int8-kv.patch": "3cfd31304353237af5277c861dc2b43ad33d0f9bff2700594300eff150f64382",
+    "qwen3_5-mtp-draft-vocab.patch": "292ef662d17bcc10556b787d5bb5f2cd12d3d3fc1f3bd2fe982485ce0d916298",
+    "marlin-int8-layer-select.patch": "833405e2ed2916529eb20184c38243c843378935522b74bab8c39707bf3ee800",
+    "offload-wsl2-devptr.patch": "8c6f9e3e5723571d1f33925794435e55062b684b2706d9284d1c5b94778d34db",
+    "marlin-repack-staged-sm80.patch": "1588e2f10b5f82194d449483e766c0b39c84ba522d1623d39caa9502040f1fab",
+    "qwen3_5-embed-quant.patch": "0a1b9ca06798c1aef582995de5a0beb3ad9a22a54cdbd2361986563a9c7a980e",
+    "dflash2-lookup-drafting.patch": "5df09ef03b592d4a2c3b47dd8d2dbf8862fa7383d68aef3dd6ef1d97d29ce196",
+    "triton-prefill-attn-int8.patch": "6bd36db0226dce7b92ebd1231b6f9718a25da4a933ab4b2a6a8be77860364fae",
+    "dflash2-backport.patch": "2e937ef748c1942ab04ace05de884e7c24041d34b23e5be8428cac6939ac6875",
+    "sampler-small-topk-fast-softmax.patch": "8828646ce1916c065282529d9c1bb52064f668397d8e1f33652aef2f308292b6",
+    "dflash2-ngram-chains.patch": "555b5a75d9023c99b2cd17634ac5b1dc8505f9275d857b1eba08c2d9cbf2df6e",
+    "spec-decode-int4-kv-mq3d.patch": "b93b186deba1513ad9c4803ee2c574c923b21ba5e88e23bca8b8594b6a6b20a0",
+    "mamba-align-checkpoint-order.patch": "515d9bf76e860c95d832d615bdab4a97b71e2b0412b4ccc2445f11535ddddf45",
+    "offload-dflash-eagle-groups.patch": "f2791af64d8066b31e4250d866c4322f45670d75522aff221307281c97c73156",
+    "marlin-int8-negative-scales.patch": "4cb8a064c706cc62dc77224479898da58864fbfff6b65012d90aa8326035300d",
+    "vision-tower-cpu-offload.patch": "81dff64a1177058783dcf7d4e8552f1547f74fa8a68019e349dacdb9d66e968e",
+    "dflash2-prewarm.patch": "e1e8012fa2c304c6948c19fc0071332f80eebcff802b402255445a30d553ab92",
+    "hybrid-kv-groups-v2-cudagraph.patch": "143825dd9744ab81dabcb438ec0bf974b6e0be8d0d7b0ffbbdfbf7c16a549664",
+    "marlin-tune-table.patch": "1cac17f12e4ce389b0cb0fc733ceceb0a8cf60c02696ee378dc1d8d0c1ef8914",
+    "int4-kv-per-token-head.patch": "c03ff10c8c997b355f04fc521827ee2fa99c332ce4127cc2d878b749b79ca3d9",
+    "spec-sampler-prewarm.patch": "18ed9608baa5a09ed2ccbb214d62763da3ce7bcf09c6aad9003778ea4e56d18f",
+    "vllm-pr50021-gdn-spec-bounds.patch": "cd6e00270fa28e37a8c7ad11f965662f4153b2fcaf840f2c4043710e7b078655",
+    "spec-decode-attn.patch": "007791047a1d60143f8e3fe4e0a56dabadfc0134cb288847338b76ba7d1f9fe7",
+    "xgrammar-spec-terminated.patch": "37589b9a45d5ece37cc16e82b195362719011e52ea31b2e70cdc89845c825040",
+    "hybrid-sw-block-promote.patch": "10876ac706546e74fe74b8964b90576a7b36f90a2f35a856997c558d7817f154",
+    "speed-knobs-envs.patch": "841ec93021b1b90f90313a1880a791bbfbdb79cbe34d9b99fdb4c3a7e52ed7c0"
+  }
+}

evaluation/manifest.json

@@ -0,0 +1,72 @@
+{
+  "seed": 15027,
+  "sources": [
+    {
+      "dataset": "openai/gsm8k",
+      "split": "main/test",
+      "file_sha256": "ee7b8da9e381df27b9e3f7758a159ab2bdaa4dbaa910546cbbc47e0cb44e4f59",
+      "selection": "first 200, same as HyperQwen"
+    },
+    {
+      "dataset": "allenai/IFBench_test",
+      "file_sha256": "80037e4d99c39a55c1e2e7d5a863d8d9edeb5ebe136ba8c1e849f8c015027c6a",
+      "note": "Upstream names this split train, but it is the benchmark test set; never used for calibration."
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test.jsonl",
+      "sha256": "2bd02b38beb48e8c46b5b9987095d999ff38cd8efc255ea5d58974317c48f63f"
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test2.jsonl",
+      "sha256": "095df7c5daf15f882c51a9deb84085cff1e073495a5dbcf95015a564d485f3a3"
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test3.jsonl",
+      "sha256": "28ed26cc83363ce3f1fe2d5fad9f8393077beb1907b167a31bd3b32f80801b79"
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test4.jsonl",
+      "sha256": "d711138ddaebfcf5f8ec6a4283ee677298c0f5c5d374a235af92aaf0584510da"
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test5.jsonl",
+      "sha256": "7f77571c2a6df0c2a72a3277650309f67e01e0008e18117e624633df53f81214"
+    },
+    {
+      "dataset": "livecodebench/code_generation_lite",
+      "file": "test6.jsonl",
+      "sha256": "bb4c364f71921c4495a6ad15abe1a927350b720009f4933e2e71f8af0f6fd1f5"
+    }
+  ],
+  "tasks": {
+    "gsm8k": 200,
+    "ifbench": 300,
+    "livecodebench": 100,
+    "tools": 30
+  },
+  "task_file_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "ppl_file_sha256": "57c83ffe7c0dfba2b869f6a379d2df3f0bc0d055069b49569a2154336151a718",
+  "protocol": "Local single-user launcher and .env, FP8 KV, 150000 context, 128000 output ceiling. Full quality concurrency two; speed and latency pilot concurrency one. Truncated answers count as wrong.",
+  "lcb_scoring": "100 stratified stdin-only v6-era tasks; all supplied public/private tests; whitespace token comparison with numeric tolerance; not the full official LCB runner.",
+  "max_output_tokens_per_call": 128000,
+  "context_len": 150000,
+  "quality_concurrency": 2,
+  "supersedes": "<WORKSPACE>/runs/swift15/evaluation/tasks.jsonl",
+  "truncation_policy": "count_as_wrong",
+  "sampling": {
+    "thinking": {
+      "temperature": 1.0,
+      "top_p": 0.95,
+      "top_k": 20,
+      "reasoning_effort": "xhigh"
+    },
+    "nonthinking": "greedy",
+    "seed": 15027
+  },
+  "serving_profile": "local-single-user"
+}

evaluation/pilot-ids.json

@@ -0,0 +1,22 @@
+[
+  "gsm8k-105",
+  "gsm8k-152",
+  "gsm8k-109",
+  "gsm8k-27",
+  "gsm8k-60",
+  "ifbench-268",
+  "ifbench-129",
+  "ifbench-76",
+  "ifbench-21",
+  "ifbench-130",
+  "lcb-atcoder-abc377_b",
+  "lcb-atcoder-abc390_d",
+  "lcb-atcoder-abc325_f",
+  "lcb-atcoder-abc385_e",
+  "lcb-atcoder-abc368_e",
+  "json-26",
+  "tool-16",
+  "tool-15",
+  "json-22",
+  "json-24"
+]

evaluation/qwen-fast-full/manifest.json

@@ -0,0 +1,693 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-0",
+    "gsm8k-1",
+    "gsm8k-2",
+    "gsm8k-3",
+    "gsm8k-4",
+    "gsm8k-5",
+    "gsm8k-6",
+    "gsm8k-7",
+    "gsm8k-8",
+    "gsm8k-9",
+    "gsm8k-10",
+    "gsm8k-11",
+    "gsm8k-12",
+    "gsm8k-13",
+    "gsm8k-14",
+    "gsm8k-15",
+    "gsm8k-16",
+    "gsm8k-17",
+    "gsm8k-18",
+    "gsm8k-19",
+    "gsm8k-20",
+    "gsm8k-21",
+    "gsm8k-22",
+    "gsm8k-23",
+    "gsm8k-24",
+    "gsm8k-25",
+    "gsm8k-26",
+    "gsm8k-27",
+    "gsm8k-28",
+    "gsm8k-29",
+    "gsm8k-30",
+    "gsm8k-31",
+    "gsm8k-32",
+    "gsm8k-33",
+    "gsm8k-34",
+    "gsm8k-35",
+    "gsm8k-36",
+    "gsm8k-37",
+    "gsm8k-38",
+    "gsm8k-39",
+    "gsm8k-40",
+    "gsm8k-41",
+    "gsm8k-42",
+    "gsm8k-43",
+    "gsm8k-44",
+    "gsm8k-45",
+    "gsm8k-46",
+    "gsm8k-47",
+    "gsm8k-48",
+    "gsm8k-49",
+    "gsm8k-50",
+    "gsm8k-51",
+    "gsm8k-52",
+    "gsm8k-53",
+    "gsm8k-54",
+    "gsm8k-55",
+    "gsm8k-56",
+    "gsm8k-57",
+    "gsm8k-58",
+    "gsm8k-59",
+    "gsm8k-60",
+    "gsm8k-61",
+    "gsm8k-62",
+    "gsm8k-63",
+    "gsm8k-64",
+    "gsm8k-65",
+    "gsm8k-66",
+    "gsm8k-67",
+    "gsm8k-68",
+    "gsm8k-69",
+    "gsm8k-70",
+    "gsm8k-71",
+    "gsm8k-72",
+    "gsm8k-73",
+    "gsm8k-74",
+    "gsm8k-75",
+    "gsm8k-76",
+    "gsm8k-77",
+    "gsm8k-78",
+    "gsm8k-79",
+    "gsm8k-80",
+    "gsm8k-81",
+    "gsm8k-82",
+    "gsm8k-83",
+    "gsm8k-84",
+    "gsm8k-85",
+    "gsm8k-86",
+    "gsm8k-87",
+    "gsm8k-88",
+    "gsm8k-89",
+    "gsm8k-90",
+    "gsm8k-91",
+    "gsm8k-92",
+    "gsm8k-93",
+    "gsm8k-94",
+    "gsm8k-95",
+    "gsm8k-96",
+    "gsm8k-97",
+    "gsm8k-98",
+    "gsm8k-99",
+    "gsm8k-100",
+    "gsm8k-101",
+    "gsm8k-102",
+    "gsm8k-103",
+    "gsm8k-104",
+    "gsm8k-105",
+    "gsm8k-106",
+    "gsm8k-107",
+    "gsm8k-108",
+    "gsm8k-109",
+    "gsm8k-110",
+    "gsm8k-111",
+    "gsm8k-112",
+    "gsm8k-113",
+    "gsm8k-114",
+    "gsm8k-115",
+    "gsm8k-116",
+    "gsm8k-117",
+    "gsm8k-118",
+    "gsm8k-119",
+    "gsm8k-120",
+    "gsm8k-121",
+    "gsm8k-122",
+    "gsm8k-123",
+    "gsm8k-124",
+    "gsm8k-125",
+    "gsm8k-126",
+    "gsm8k-127",
+    "gsm8k-128",
+    "gsm8k-129",
+    "gsm8k-130",
+    "gsm8k-131",
+    "gsm8k-132",
+    "gsm8k-133",
+    "gsm8k-134",
+    "gsm8k-135",
+    "gsm8k-136",
+    "gsm8k-137",
+    "gsm8k-138",
+    "gsm8k-139",
+    "gsm8k-140",
+    "gsm8k-141",
+    "gsm8k-142",
+    "gsm8k-143",
+    "gsm8k-144",
+    "gsm8k-145",
+    "gsm8k-146",
+    "gsm8k-147",
+    "gsm8k-148",
+    "gsm8k-149",
+    "gsm8k-150",
+    "gsm8k-151",
+    "gsm8k-152",
+    "gsm8k-153",
+    "gsm8k-154",
+    "gsm8k-155",
+    "gsm8k-156",
+    "gsm8k-157",
+    "gsm8k-158",
+    "gsm8k-159",
+    "gsm8k-160",
+    "gsm8k-161",
+    "gsm8k-162",
+    "gsm8k-163",
+    "gsm8k-164",
+    "gsm8k-165",
+    "gsm8k-166",
+    "gsm8k-167",
+    "gsm8k-168",
+    "gsm8k-169",
+    "gsm8k-170",
+    "gsm8k-171",
+    "gsm8k-172",
+    "gsm8k-173",
+    "gsm8k-174",
+    "gsm8k-175",
+    "gsm8k-176",
+    "gsm8k-177",
+    "gsm8k-178",
+    "gsm8k-179",
+    "gsm8k-180",
+    "gsm8k-181",
+    "gsm8k-182",
+    "gsm8k-183",
+    "gsm8k-184",
+    "gsm8k-185",
+    "gsm8k-186",
+    "gsm8k-187",
+    "gsm8k-188",
+    "gsm8k-189",
+    "gsm8k-190",
+    "gsm8k-191",
+    "gsm8k-192",
+    "gsm8k-193",
+    "gsm8k-194",
+    "gsm8k-195",
+    "gsm8k-196",
+    "gsm8k-197",
+    "gsm8k-198",
+    "gsm8k-199",
+    "ifbench-0",
+    "ifbench-1",
+    "ifbench-2",
+    "ifbench-3",
+    "ifbench-4",
+    "ifbench-5",
+    "ifbench-6",
+    "ifbench-7",
+    "ifbench-8",
+    "ifbench-9",
+    "ifbench-10",
+    "ifbench-11",
+    "ifbench-12",
+    "ifbench-13",
+    "ifbench-14",
+    "ifbench-15",
+    "ifbench-16",
+    "ifbench-17",
+    "ifbench-18",
+    "ifbench-19",
+    "ifbench-20",
+    "ifbench-21",
+    "ifbench-22",
+    "ifbench-23",
+    "ifbench-24",
+    "ifbench-25",
+    "ifbench-26",
+    "ifbench-27",
+    "ifbench-28",
+    "ifbench-29",
+    "ifbench-30",
+    "ifbench-31",
+    "ifbench-32",
+    "ifbench-33",
+    "ifbench-34",
+    "ifbench-35",
+    "ifbench-36",
+    "ifbench-37",
+    "ifbench-38",
+    "ifbench-39",
+    "ifbench-40",
+    "ifbench-41",
+    "ifbench-42",
+    "ifbench-43",
+    "ifbench-44",
+    "ifbench-45",
+    "ifbench-46",
+    "ifbench-47",
+    "ifbench-48",
+    "ifbench-49",
+    "ifbench-50",
+    "ifbench-51",
+    "ifbench-52",
+    "ifbench-53",
+    "ifbench-54",
+    "ifbench-55",
+    "ifbench-56",
+    "ifbench-57",
+    "ifbench-58",
+    "ifbench-59",
+    "ifbench-60",
+    "ifbench-61",
+    "ifbench-62",
+    "ifbench-63",
+    "ifbench-64",
+    "ifbench-65",
+    "ifbench-66",
+    "ifbench-67",
+    "ifbench-68",
+    "ifbench-69",
+    "ifbench-70",
+    "ifbench-71",
+    "ifbench-72",
+    "ifbench-73",
+    "ifbench-74",
+    "ifbench-75",
+    "ifbench-76",
+    "ifbench-77",
+    "ifbench-78",
+    "ifbench-79",
+    "ifbench-80",
+    "ifbench-81",
+    "ifbench-82",
+    "ifbench-83",
+    "ifbench-84",
+    "ifbench-85",
+    "ifbench-86",
+    "ifbench-87",
+    "ifbench-88",
+    "ifbench-89",
+    "ifbench-90",
+    "ifbench-91",
+    "ifbench-92",
+    "ifbench-93",
+    "ifbench-94",
+    "ifbench-95",
+    "ifbench-96",
+    "ifbench-97",
+    "ifbench-98",
+    "ifbench-99",
+    "ifbench-100",
+    "ifbench-101",
+    "ifbench-102",
+    "ifbench-103",
+    "ifbench-104",
+    "ifbench-105",
+    "ifbench-106",
+    "ifbench-107",
+    "ifbench-108",
+    "ifbench-109",
+    "ifbench-110",
+    "ifbench-111",
+    "ifbench-112",
+    "ifbench-113",
+    "ifbench-114",
+    "ifbench-115",
+    "ifbench-116",
+    "ifbench-117",
+    "ifbench-118",
+    "ifbench-119",
+    "ifbench-120",
+    "ifbench-121",
+    "ifbench-122",
+    "ifbench-123",
+    "ifbench-124",
+    "ifbench-125",
+    "ifbench-126",
+    "ifbench-127",
+    "ifbench-128",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-131",
+    "ifbench-132",
+    "ifbench-133",
+    "ifbench-134",
+    "ifbench-135",
+    "ifbench-136",
+    "ifbench-137",
+    "ifbench-138",
+    "ifbench-139",
+    "ifbench-140",
+    "ifbench-141",
+    "ifbench-142",
+    "ifbench-143",
+    "ifbench-144",
+    "ifbench-145",
+    "ifbench-146",
+    "ifbench-147",
+    "ifbench-148",
+    "ifbench-149",
+    "ifbench-150",
+    "ifbench-151",
+    "ifbench-152",
+    "ifbench-153",
+    "ifbench-154",
+    "ifbench-155",
+    "ifbench-156",
+    "ifbench-157",
+    "ifbench-158",
+    "ifbench-159",
+    "ifbench-160",
+    "ifbench-161",
+    "ifbench-162",
+    "ifbench-163",
+    "ifbench-164",
+    "ifbench-165",
+    "ifbench-166",
+    "ifbench-167",
+    "ifbench-168",
+    "ifbench-169",
+    "ifbench-170",
+    "ifbench-171",
+    "ifbench-172",
+    "ifbench-173",
+    "ifbench-174",
+    "ifbench-175",
+    "ifbench-176",
+    "ifbench-177",
+    "ifbench-178",
+    "ifbench-179",
+    "ifbench-180",
+    "ifbench-181",
+    "ifbench-182",
+    "ifbench-183",
+    "ifbench-184",
+    "ifbench-185",
+    "ifbench-186",
+    "ifbench-187",
+    "ifbench-188",
+    "ifbench-189",
+    "ifbench-190",
+    "ifbench-191",
+    "ifbench-192",
+    "ifbench-193",
+    "ifbench-194",
+    "ifbench-195",
+    "ifbench-196",
+    "ifbench-197",
+    "ifbench-198",
+    "ifbench-199",
+    "ifbench-200",
+    "ifbench-201",
+    "ifbench-202",
+    "ifbench-203",
+    "ifbench-204",
+    "ifbench-205",
+    "ifbench-206",
+    "ifbench-207",
+    "ifbench-208",
+    "ifbench-209",
+    "ifbench-210",
+    "ifbench-211",
+    "ifbench-212",
+    "ifbench-213",
+    "ifbench-214",
+    "ifbench-215",
+    "ifbench-216",
+    "ifbench-217",
+    "ifbench-218",
+    "ifbench-219",
+    "ifbench-220",
+    "ifbench-221",
+    "ifbench-222",
+    "ifbench-223",
+    "ifbench-224",
+    "ifbench-225",
+    "ifbench-226",
+    "ifbench-227",
+    "ifbench-228",
+    "ifbench-229",
+    "ifbench-230",
+    "ifbench-231",
+    "ifbench-232",
+    "ifbench-233",
+    "ifbench-234",
+    "ifbench-235",
+    "ifbench-236",
+    "ifbench-237",
+    "ifbench-238",
+    "ifbench-239",
+    "ifbench-240",
+    "ifbench-241",
+    "ifbench-242",
+    "ifbench-243",
+    "ifbench-244",
+    "ifbench-245",
+    "ifbench-246",
+    "ifbench-247",
+    "ifbench-248",
+    "ifbench-249",
+    "ifbench-250",
+    "ifbench-251",
+    "ifbench-252",
+    "ifbench-253",
+    "ifbench-254",
+    "ifbench-255",
+    "ifbench-256",
+    "ifbench-257",
+    "ifbench-258",
+    "ifbench-259",
+    "ifbench-260",
+    "ifbench-261",
+    "ifbench-262",
+    "ifbench-263",
+    "ifbench-264",
+    "ifbench-265",
+    "ifbench-266",
+    "ifbench-267",
+    "ifbench-268",
+    "ifbench-269",
+    "ifbench-270",
+    "ifbench-271",
+    "ifbench-272",
+    "ifbench-273",
+    "ifbench-274",
+    "ifbench-275",
+    "ifbench-276",
+    "ifbench-277",
+    "ifbench-278",
+    "ifbench-279",
+    "ifbench-280",
+    "ifbench-281",
+    "ifbench-282",
+    "ifbench-283",
+    "ifbench-284",
+    "ifbench-285",
+    "ifbench-286",
+    "ifbench-287",
+    "ifbench-288",
+    "ifbench-289",
+    "ifbench-290",
+    "ifbench-291",
+    "ifbench-292",
+    "ifbench-293",
+    "ifbench-294",
+    "ifbench-295",
+    "ifbench-296",
+    "ifbench-297",
+    "ifbench-298",
+    "ifbench-299",
+    "lcb-atcoder-abc322_b",
+    "lcb-atcoder-abc381_a",
+    "lcb-atcoder-abc393_a",
+    "lcb-atcoder-abc321_b",
+    "lcb-atcoder-abc359_a",
+    "lcb-atcoder-abc329_b",
+    "lcb-atcoder-abc353_a",
+    "lcb-atcoder-abc355_b",
+    "lcb-atcoder-abc326_b",
+    "lcb-codeforces-1873_B",
+    "lcb-atcoder-abc356_a",
+    "lcb-atcoder-abc356_b",
+    "lcb-atcoder-abc375_a",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc311_b",
+    "lcb-atcoder-abc378_b",
+    "lcb-atcoder-abc309_b",
+    "lcb-atcoder-abc325_a",
+    "lcb-atcoder-abc301_b",
+    "lcb-atcoder-abc391_a",
+    "lcb-atcoder-abc399_b",
+    "lcb-atcoder-abc354_a",
+    "lcb-atcoder-abc352_b",
+    "lcb-atcoder-abc382_b",
+    "lcb-atcoder-abc332_b",
+    "lcb-atcoder-abc343_b",
+    "lcb-atcoder-abc361_a",
+    "lcb-atcoder-abc362_a",
+    "lcb-atcoder-abc328_a",
+    "lcb-atcoder-abc393_b",
+    "lcb-atcoder-abc352_a",
+    "lcb-atcoder-abc310_a",
+    "lcb-atcoder-abc365_a",
+    "lcb-atcoder-abc371_b",
+    "lcb-atcoder-abc367_c",
+    "lcb-atcoder-abc334_b",
+    "lcb-atcoder-abc385_c",
+    "lcb-atcoder-abc307_c",
+    "lcb-atcoder-abc338_c",
+    "lcb-atcoder-abc303_d",
+    "lcb-atcoder-abc342_c",
+    "lcb-atcoder-abc319_d",
+    "lcb-atcoder-abc315_d",
+    "lcb-atcoder-abc309_c",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc343_d",
+    "lcb-atcoder-abc397_b",
+    "lcb-atcoder-abc370_c",
+    "lcb-atcoder-abc375_c",
+    "lcb-atcoder-abc368_c",
+    "lcb-atcoder-abc325_b",
+    "lcb-atcoder-abc323_c",
+    "lcb-atcoder-abc377_c",
+    "lcb-atcoder-abc383_d",
+    "lcb-atcoder-arc189_a",
+    "lcb-codeforces-1883_C",
+    "lcb-atcoder-abc371_c",
+    "lcb-atcoder-abc380_c",
+    "lcb-atcoder-abc378_c",
+    "lcb-atcoder-abc366_c",
+    "lcb-atcoder-abc397_c",
+    "lcb-atcoder-abc339_c",
+    "lcb-atcoder-abc324_c",
+    "lcb-atcoder-abc355_c",
+    "lcb-atcoder-abc358_c",
+    "lcb-atcoder-abc321_d",
+    "lcb-atcoder-abc334_c",
+    "lcb-atcoder-arc195_c",
+    "lcb-atcoder-arc184_c",
+    "lcb-atcoder-abc377_e",
+    "lcb-atcoder-arc186_e",
+    "lcb-atcoder-abc398_f",
+    "lcb-atcoder-abc330_e",
+    "lcb-atcoder-abc396_e",
+    "lcb-atcoder-arc192_b",
+    "lcb-atcoder-abc391_f",
+    "lcb-atcoder-arc195_b",
+    "lcb-atcoder-abc384_g",
+    "lcb-atcoder-abc343_e",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc333_e",
+    "lcb-atcoder-abc341_e",
+    "lcb-atcoder-abc400_g",
+    "lcb-atcoder-abc362_d",
+    "lcb-codeforces-1899_D",
+    "lcb-atcoder-abc363_f",
+    "lcb-atcoder-abc382_d",
+    "lcb-atcoder-abc331_e",
+    "lcb-atcoder-abc351_e",
+    "lcb-atcoder-arc194_b",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-arc188_d",
+    "lcb-atcoder-abc368_e",
+    "lcb-atcoder-abc301_e",
+    "lcb-atcoder-arc194_e",
+    "lcb-atcoder-abc379_e",
+    "lcb-atcoder-abc360_e",
+    "lcb-atcoder-abc305_e",
+    "lcb-atcoder-arc186_a",
+    "lcb-atcoder-abc362_e",
+    "tool-0",
+    "tool-1",
+    "tool-2",
+    "tool-3",
+    "tool-4",
+    "tool-5",
+    "tool-6",
+    "tool-7",
+    "tool-8",
+    "tool-9",
+    "tool-10",
+    "tool-11",
+    "tool-12",
+    "tool-13",
+    "tool-14",
+    "tool-15",
+    "tool-16",
+    "tool-17",
+    "tool-18",
+    "tool-19",
+    "json-20",
+    "json-21",
+    "json-22",
+    "json-23",
+    "json-24",
+    "json-25",
+    "json-26",
+    "json-27",
+    "json-28",
+    "json-29"
+  ],
+  "concurrency": 2,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Qwen3.8-27B-W4A16-AutoRound-fast-eval",
+    "config_sha256": "5a3e7312dee3a6c5909beb854b05ac818be836683730e4facba81f547b77fd62",
+    "index_sha256": "316f7411552d8e80df6018b5f01c521e2b1a60c81b5564ed84366e50c74e383f",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00005-of-00007.safetensors": {
+        "size": 699275200,
+        "mtime_ns": 1789678521251603880
+      },
+      "model-00001-of-00007.safetensors": {
+        "size": 3211003472,
+        "mtime_ns": 1789678522484562485
+      },
+      "model-00006-of-00007.safetensors": {
+        "size": 1291274752,
+        "mtime_ns": 1790442052555391180
+      },
+      "model-00003-of-00007.safetensors": {
+        "size": 3195027104,
+        "mtime_ns": 1789678524567492723
+      },
+      "model-00002-of-00007.safetensors": {
+        "size": 3195027104,
+        "mtime_ns": 1789678522534560808
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 327162544,
+        "mtime_ns": 1789678527579392230
+      },
+      "model-00007-of-00007.safetensors": {
+        "size": 655565120,
+        "mtime_ns": 1789678527180405514
+      },
+      "model-00004-of-00007.safetensors": {
+        "size": 3217442568,
+        "mtime_ns": 1789678522650556917
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/qwen-fast-full/summary.json

@@ -0,0 +1,89 @@
+{
+  "created": "2026-09-27T06:45:29Z",
+  "concurrency": 2,
+  "resumed": false,
+  "new_run_wall_seconds": 34295.263359012024,
+  "new_tasks": 630,
+  "suites": {
+    "gsm8k": {
+      "attempted": 200,
+      "correct": 195,
+      "accuracy": 0.975,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 405.4,
+      "mean_input_tokens": 93.27,
+      "mean_total_tokens": 498.67,
+      "mean_model_seconds": 3.4959702649945394,
+      "median_model_seconds": 3.070697635994293,
+      "p95_model_seconds": 6.074415434035473,
+      "summed_request_seconds_per_correct": 3.5856105281995276,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 300,
+      "correct": 222,
+      "accuracy": 0.74,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 1,
+      "errors": 0,
+      "truncated": 1,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 10429.793333333333,
+      "mean_input_tokens": 125.44,
+      "mean_total_tokens": 10555.233333333334,
+      "mean_model_seconds": 114.29685370972729,
+      "median_model_seconds": 59.345528290461516,
+      "p95_model_seconds": 438.60096366197104,
+      "summed_request_seconds_per_correct": 154.4552077158477,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 100,
+      "correct": 90,
+      "accuracy": 0.9,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 1,
+      "errors": 0,
+      "truncated": 1,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 24492.16,
+      "mean_input_tokens": 653.75,
+      "mean_total_tokens": 25145.91,
+      "mean_model_seconds": 331.0521950917202,
+      "median_model_seconds": 90.47728537750663,
+      "p95_model_seconds": 1340.8850151840015,
+      "summed_request_seconds_per_correct": 367.8357723241336,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 30,
+      "correct": 29,
+      "accuracy": 0.9666666666666667,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 40.833333333333336,
+      "mean_input_tokens": 481.3333333333333,
+      "mean_total_tokens": 522.1666666666666,
+      "mean_model_seconds": 1.0324668154586107,
+      "median_model_seconds": 1.2877807764452882,
+      "p95_model_seconds": 1.3691221890621819,
+      "summed_request_seconds_per_correct": 1.068069119439942,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [
+    "ifbench-107",
+    "lcb-atcoder-arc184_c"
+  ],
+  "suite_wall_seconds": 34295.263359012024,
+  "wall_seconds_per_correct": 63.983700296664225
+}

evaluation/qwen-fast-pilot/manifest.json

@@ -0,0 +1,83 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-27",
+    "gsm8k-60",
+    "gsm8k-105",
+    "gsm8k-109",
+    "gsm8k-152",
+    "ifbench-21",
+    "ifbench-76",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-268",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-abc368_e",
+    "tool-15",
+    "tool-16",
+    "json-22",
+    "json-24",
+    "json-26"
+  ],
+  "concurrency": 1,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Qwen3.8-27B-W4A16-AutoRound-fast-eval",
+    "config_sha256": "5a3e7312dee3a6c5909beb854b05ac818be836683730e4facba81f547b77fd62",
+    "index_sha256": "316f7411552d8e80df6018b5f01c521e2b1a60c81b5564ed84366e50c74e383f",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00005-of-00007.safetensors": {
+        "size": 699275200,
+        "mtime_ns": 1789678521251603880
+      },
+      "model-00001-of-00007.safetensors": {
+        "size": 3211003472,
+        "mtime_ns": 1789678522484562485
+      },
+      "model-00006-of-00007.safetensors": {
+        "size": 1291274752,
+        "mtime_ns": 1790442052555391180
+      },
+      "model-00003-of-00007.safetensors": {
+        "size": 3195027104,
+        "mtime_ns": 1789678524567492723
+      },
+      "model-00002-of-00007.safetensors": {
+        "size": 3195027104,
+        "mtime_ns": 1789678522534560808
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 327162544,
+        "mtime_ns": 1789678527579392230
+      },
+      "model-00007-of-00007.safetensors": {
+        "size": 655565120,
+        "mtime_ns": 1789678527180405514
+      },
+      "model-00004-of-00007.safetensors": {
+        "size": 3217442568,
+        "mtime_ns": 1789678522650556917
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/qwen-fast-pilot/summary.json

@@ -0,0 +1,86 @@
+{
+  "created": "2026-09-26T21:13:49Z",
+  "concurrency": 1,
+  "resumed": false,
+  "new_run_wall_seconds": 4374.499300124997,
+  "new_tasks": 20,
+  "suites": {
+    "gsm8k": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 275.2,
+      "mean_input_tokens": 77.4,
+      "mean_total_tokens": 352.6,
+      "mean_model_seconds": 2.2795148026081735,
+      "median_model_seconds": 2.2747438669903204,
+      "p95_model_seconds": 2.7609281560289674,
+      "summed_request_seconds_per_correct": 2.2795148026081735,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 5,
+      "correct": 4,
+      "accuracy": 0.8,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 5641,
+      "mean_input_tokens": 134.6,
+      "mean_total_tokens": 5775.6,
+      "mean_model_seconds": 60.683199993602464,
+      "median_model_seconds": 33.55660462501692,
+      "p95_model_seconds": 188.0719450309989,
+      "summed_request_seconds_per_correct": 75.85399999200308,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 63239.6,
+      "mean_input_tokens": 742.4,
+      "mean_total_tokens": 63982,
+      "mean_model_seconds": 807.4264700495871,
+      "median_model_seconds": 839.0481605409877,
+      "p95_model_seconds": 1524.661622893007,
+      "summed_request_seconds_per_correct": 807.4264700495871,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 32.8,
+      "mean_input_tokens": 330.6,
+      "mean_total_tokens": 363.4,
+      "mean_model_seconds": 0.6404867220087909,
+      "median_model_seconds": 0.4934498239890672,
+      "p95_model_seconds": 1.0912616860005073,
+      "summed_request_seconds_per_correct": 0.6404867220087909,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [],
+  "suite_wall_seconds": 4374.499300124997,
+  "wall_seconds_per_correct": 230.23680526973666
+}

evaluation/qwen-fast/perplexity.json

@@ -0,0 +1,659 @@
+{
+  "input_sha256": "57c83ffe7c0dfba2b869f6a379d2df3f0bc0d055069b49569a2154336151a718",
+  "windows": [
+    {
+      "id": 0,
+      "language": "en",
+      "tokens": 283,
+      "logprob_sum": -592.9908143195717
+    },
+    {
+      "id": 1,
+      "language": "en",
+      "tokens": 313,
+      "logprob_sum": -694.9407490367958
+    },
+    {
+      "id": 2,
+      "language": "en",
+      "tokens": 284,
+      "logprob_sum": -731.2198149585729
+    },
+    {
+      "id": 3,
+      "language": "en",
+      "tokens": 286,
+      "logprob_sum": -760.9622869625673
+    },
+    {
+      "id": 4,
+      "language": "en",
+      "tokens": 279,
+      "logprob_sum": -585.9759548976872
+    },
+    {
+      "id": 5,
+      "language": "en",
+      "tokens": 251,
+      "logprob_sum": -624.1354073971652
+    },
+    {
+      "id": 6,
+      "language": "en",
+      "tokens": 258,
+      "logprob_sum": -606.1599571104307
+    },
+    {
+      "id": 7,
+      "language": "en",
+      "tokens": 258,
+      "logprob_sum": -601.0718723634782
+    },
+    {
+      "id": 8,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -629.9130605636892
+    },
+    {
+      "id": 9,
+      "language": "en",
+      "tokens": 278,
+      "logprob_sum": -605.0941830798547
+    },
+    {
+      "id": 10,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -713.5701698549092
+    },
+    {
+      "id": 11,
+      "language": "en",
+      "tokens": 288,
+      "logprob_sum": -759.5278018228346
+    },
+    {
+      "id": 12,
+      "language": "en",
+      "tokens": 277,
+      "logprob_sum": -623.4307517245397
+    },
+    {
+      "id": 13,
+      "language": "en",
+      "tokens": 291,
+      "logprob_sum": -699.0976419129438
+    },
+    {
+      "id": 14,
+      "language": "en",
+      "tokens": 260,
+      "logprob_sum": -672.0876630296007
+    },
+    {
+      "id": 15,
+      "language": "en",
+      "tokens": 250,
+      "logprob_sum": -761.0354307279922
+    },
+    {
+      "id": 16,
+      "language": "en",
+      "tokens": 252,
+      "logprob_sum": -754.8780198292225
+    },
+    {
+      "id": 17,
+      "language": "en",
+      "tokens": 272,
+      "logprob_sum": -788.3398015776256
+    },
+    {
+      "id": 18,
+      "language": "en",
+      "tokens": 240,
+      "logprob_sum": -668.2802309252293
+    },
+    {
+      "id": 19,
+      "language": "en",
+      "tokens": 244,
+      "logprob_sum": -561.2377929796639
+    },
+    {
+      "id": 20,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -682.5377916035941
+    },
+    {
+      "id": 21,
+      "language": "en",
+      "tokens": 299,
+      "logprob_sum": -738.1227494915511
+    },
+    {
+      "id": 22,
+      "language": "en",
+      "tokens": 318,
+      "logprob_sum": -823.5101455471158
+    },
+    {
+      "id": 23,
+      "language": "en",
+      "tokens": 263,
+      "logprob_sum": -677.3689384464233
+    },
+    {
+      "id": 24,
+      "language": "en",
+      "tokens": 239,
+      "logprob_sum": -711.9839977540105
+    },
+    {
+      "id": 25,
+      "language": "en",
+      "tokens": 293,
+      "logprob_sum": -493.05563317897304
+    },
+    {
+      "id": 26,
+      "language": "en",
+      "tokens": 303,
+      "logprob_sum": -678.5333020727912
+    },
+    {
+      "id": 27,
+      "language": "en",
+      "tokens": 276,
+      "logprob_sum": -623.330151065903
+    },
+    {
+      "id": 28,
+      "language": "en",
+      "tokens": 282,
+      "logprob_sum": -729.7613100720046
+    },
+    {
+      "id": 29,
+      "language": "en",
+      "tokens": 298,
+      "logprob_sum": -691.1857994162092
+    },
+    {
+      "id": 30,
+      "language": "en",
+      "tokens": 338,
+      "logprob_sum": -478.5514251983941
+    },
+    {
+      "id": 31,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -639.465224127729
+    },
+    {
+      "id": 32,
+      "language": "en",
+      "tokens": 310,
+      "logprob_sum": -813.8291402243576
+    },
+    {
+      "id": 33,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -798.3652476328898
+    },
+    {
+      "id": 34,
+      "language": "en",
+      "tokens": 302,
+      "logprob_sum": -693.1609880588003
+    },
+    {
+      "id": 35,
+      "language": "en",
+      "tokens": 286,
+      "logprob_sum": -523.3296906485411
+    },
+    {
+      "id": 36,
+      "language": "en",
+      "tokens": 244,
+      "logprob_sum": -581.809817118934
+    },
+    {
+      "id": 37,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -495.3885064668866
+    },
+    {
+      "id": 38,
+      "language": "en",
+      "tokens": 315,
+      "logprob_sum": -599.4133885340561
+    },
+    {
+      "id": 39,
+      "language": "en",
+      "tokens": 283,
+      "logprob_sum": -647.8761923574384
+    },
+    {
+      "id": 40,
+      "language": "da",
+      "tokens": 319,
+      "logprob_sum": -518.2542151119087
+    },
+    {
+      "id": 41,
+      "language": "da",
+      "tokens": 334,
+      "logprob_sum": -796.8319925991527
+    },
+    {
+      "id": 42,
+      "language": "da",
+      "tokens": 321,
+      "logprob_sum": -657.0615293114461
+    },
+    {
+      "id": 43,
+      "language": "da",
+      "tokens": 355,
+      "logprob_sum": -765.0516552211648
+    },
+    {
+      "id": 44,
+      "language": "da",
+      "tokens": 372,
+      "logprob_sum": -793.2305373175477
+    },
+    {
+      "id": 45,
+      "language": "da",
+      "tokens": 360,
+      "logprob_sum": -591.0126144723617
+    },
+    {
+      "id": 46,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -1261.7429924435055
+    },
+    {
+      "id": 47,
+      "language": "da",
+      "tokens": 372,
+      "logprob_sum": -597.0048053516152
+    },
+    {
+      "id": 48,
+      "language": "da",
+      "tokens": 356,
+      "logprob_sum": -806.2568797468084
+    },
+    {
+      "id": 49,
+      "language": "da",
+      "tokens": 269,
+      "logprob_sum": -1434.5953341651184
+    },
+    {
+      "id": 50,
+      "language": "da",
+      "tokens": 334,
+      "logprob_sum": -586.6079792070204
+    },
+    {
+      "id": 51,
+      "language": "da",
+      "tokens": 364,
+      "logprob_sum": -621.1433348721202
+    },
+    {
+      "id": 52,
+      "language": "da",
+      "tokens": 364,
+      "logprob_sum": -678.3656660186107
+    },
+    {
+      "id": 53,
+      "language": "da",
+      "tokens": 378,
+      "logprob_sum": -453.8299491084478
+    },
+    {
+      "id": 54,
+      "language": "da",
+      "tokens": 400,
+      "logprob_sum": -827.0148732287234
+    },
+    {
+      "id": 55,
+      "language": "da",
+      "tokens": 319,
+      "logprob_sum": -1556.6795706446283
+    },
+    {
+      "id": 56,
+      "language": "da",
+      "tokens": 329,
+      "logprob_sum": -621.5343871314999
+    },
+    {
+      "id": 57,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -865.1125200971237
+    },
+    {
+      "id": 58,
+      "language": "da",
+      "tokens": 336,
+      "logprob_sum": -1569.0122353932675
+    },
+    {
+      "id": 59,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -990.5655090794025
+    },
+    {
+      "id": 60,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -484.93386594452477
+    },
+    {
+      "id": 61,
+      "language": "da",
+      "tokens": 349,
+      "logprob_sum": -868.9878758826126
+    },
+    {
+      "id": 62,
+      "language": "da",
+      "tokens": 349,
+      "logprob_sum": -569.4126007331457
+    },
+    {
+      "id": 63,
+      "language": "da",
+      "tokens": 356,
+      "logprob_sum": -669.0766140862625
+    },
+    {
+      "id": 64,
+      "language": "da",
+      "tokens": 388,
+      "logprob_sum": -739.1665408044555
+    },
+    {
+      "id": 65,
+      "language": "da",
+      "tokens": 342,
+      "logprob_sum": -1576.0850733005718
+    },
+    {
+      "id": 66,
+      "language": "da",
+      "tokens": 376,
+      "logprob_sum": -680.5289953708525
+    },
+    {
+      "id": 67,
+      "language": "da",
+      "tokens": 388,
+      "logprob_sum": -563.0824029279489
+    },
+    {
+      "id": 68,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -680.2388399759147
+    },
+    {
+      "id": 69,
+      "language": "da",
+      "tokens": 335,
+      "logprob_sum": -643.2929824862158
+    },
+    {
+      "id": 70,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -742.9423492672252
+    },
+    {
+      "id": 71,
+      "language": "da",
+      "tokens": 371,
+      "logprob_sum": -1402.213066599099
+    },
+    {
+      "id": 72,
+      "language": "da",
+      "tokens": 371,
+      "logprob_sum": -730.0679423760885
+    },
+    {
+      "id": 73,
+      "language": "da",
+      "tokens": 325,
+      "logprob_sum": -584.7675743625477
+    },
+    {
+      "id": 74,
+      "language": "da",
+      "tokens": 381,
+      "logprob_sum": -1003.3340749706913
+    },
+    {
+      "id": 75,
+      "language": "da",
+      "tokens": 265,
+      "logprob_sum": -1381.0563661176711
+    },
+    {
+      "id": 76,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -895.6867733937834
+    },
+    {
+      "id": 77,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -603.0461025548739
+    },
+    {
+      "id": 78,
+      "language": "da",
+      "tokens": 346,
+      "logprob_sum": -860.3451435220468
+    },
+    {
+      "id": 79,
+      "language": "da",
+      "tokens": 359,
+      "logprob_sum": -592.5020944437811
+    },
+    {
+      "id": 80,
+      "language": "code",
+      "tokens": 320,
+      "logprob_sum": -276.6699116966555
+    },
+    {
+      "id": 81,
+      "language": "code",
+      "tokens": 282,
+      "logprob_sum": -420.23796508818486
+    },
+    {
+      "id": 82,
+      "language": "code",
+      "tokens": 278,
+      "logprob_sum": -263.582931014716
+    },
+    {
+      "id": 83,
+      "language": "code",
+      "tokens": 273,
+      "logprob_sum": -418.2609401268255
+    },
+    {
+      "id": 84,
+      "language": "code",
+      "tokens": 253,
+      "logprob_sum": -199.9309399436836
+    },
+    {
+      "id": 85,
+      "language": "code",
+      "tokens": 239,
+      "logprob_sum": -465.6562252798758
+    },
+    {
+      "id": 86,
+      "language": "code",
+      "tokens": 272,
+      "logprob_sum": -374.48760858155583
+    },
+    {
+      "id": 87,
+      "language": "code",
+      "tokens": 253,
+      "logprob_sum": -291.8236611021075
+    },
+    {
+      "id": 88,
+      "language": "code",
+      "tokens": 317,
+      "logprob_sum": -284.14727575928714
+    },
+    {
+      "id": 89,
+      "language": "code",
+      "tokens": 298,
+      "logprob_sum": -303.1649754933601
+    },
+    {
+      "id": 90,
+      "language": "code",
+      "tokens": 280,
+      "logprob_sum": -329.05116950601234
+    },
+    {
+      "id": 91,
+      "language": "code",
+      "tokens": 265,
+      "logprob_sum": -394.77499111452255
+    },
+    {
+      "id": 92,
+      "language": "code",
+      "tokens": 305,
+      "logprob_sum": -219.57371839839698
+    },
+    {
+      "id": 93,
+      "language": "code",
+      "tokens": 310,
+      "logprob_sum": -331.6206139813031
+    },
+    {
+      "id": 94,
+      "language": "code",
+      "tokens": 278,
+      "logprob_sum": -251.4822827990156
+    },
+    {
+      "id": 95,
+      "language": "code",
+      "tokens": 301,
+      "logprob_sum": -308.4679439706525
+    },
+    {
+      "id": 96,
+      "language": "code",
+      "tokens": 311,
+      "logprob_sum": -184.7737082277912
+    },
+    {
+      "id": 97,
+      "language": "code",
+      "tokens": 312,
+      "logprob_sum": -226.99020833792702
+    },
+    {
+      "id": 98,
+      "language": "code",
+      "tokens": 325,
+      "logprob_sum": -197.7390554841084
+    },
+    {
+      "id": 99,
+      "language": "code",
+      "tokens": 291,
+      "logprob_sum": -325.6971021090387
+    },
+    {
+      "id": 100,
+      "language": "code",
+      "tokens": 321,
+      "logprob_sum": -453.9610157664529
+    },
+    {
+      "id": 101,
+      "language": "code",
+      "tokens": 318,
+      "logprob_sum": -451.3294726509339
+    },
+    {
+      "id": 102,
+      "language": "code",
+      "tokens": 316,
+      "logprob_sum": -502.03988517310944
+    },
+    {
+      "id": 103,
+      "language": "code",
+      "tokens": 307,
+      "logprob_sum": -297.40668610171537
+    },
+    {
+      "id": 104,
+      "language": "code",
+      "tokens": 254,
+      "logprob_sum": -418.13921085138236
+    },
+    {
+      "id": 105,
+      "language": "code",
+      "tokens": 275,
+      "logprob_sum": -457.8816399213333
+    }
+  ],
+  "scores": {
+    "en": {
+      "tokens": 11175,
+      "ppl": 10.764397834031014
+    },
+    "all": {
+      "tokens": 32646,
+      "ppl": 8.143312831984629
+    },
+    "da": {
+      "tokens": 13917,
+      "ppl": 10.913529905515743
+    },
+    "code": {
+      "tokens": 7554,
+      "ppl": 3.1422587970654146
+    }
+  }
+}

evaluation/qwen-fast/server.json

@@ -0,0 +1,57 @@
+{
+  "created": "2026-09-26T19:58:25Z",
+  "model": "<WORKSPACE>/models/Qwen3.8-27B-W4A16-AutoRound-fast-eval",
+  "config_sha256": "5a3e7312dee3a6c5909beb854b05ac818be836683730e4facba81f547b77fd62",
+  "index_sha256": "316f7411552d8e80df6018b5f01c521e2b1a60c81b5564ed84366e50c74e383f",
+  "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+  "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+  "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+  "shards": {
+    "model-00005-of-00007.safetensors": {
+      "size": 699275200,
+      "mtime_ns": 1789678521251603880
+    },
+    "model-00001-of-00007.safetensors": {
+      "size": 3211003472,
+      "mtime_ns": 1789678522484562485
+    },
+    "model-00006-of-00007.safetensors": {
+      "size": 1291274752,
+      "mtime_ns": 1790442052555391180
+    },
+    "model-00003-of-00007.safetensors": {
+      "size": 3195027104,
+      "mtime_ns": 1789678524567492723
+    },
+    "model-00002-of-00007.safetensors": {
+      "size": 3195027104,
+      "mtime_ns": 1789678522534560808
+    },
+    "model_extra_tensors.safetensors": {
+      "size": 327162544,
+      "mtime_ns": 1789678527579392230
+    },
+    "model-00007-of-00007.safetensors": {
+      "size": 655565120,
+      "mtime_ns": 1789678527180405514
+    },
+    "model-00004-of-00007.safetensors": {
+      "size": 3217442568,
+      "mtime_ns": 1789678522650556917
+    }
+  },
+  "mode": "single-user",
+  "int8_activations": false,
+  "speculation": "mtp",
+  "max_model_len": 150000,
+  "max_num_seqs": 8,
+  "kv_cache_dtype": "fp8",
+  "prefix_cache": true,
+  "gpu_memory_utilization": 0.93,
+  "pid": 926860,
+  "serving_profile": "local-single-user",
+  "vision": true,
+  "draft_tokens": 3,
+  "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+  "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+}

evaluation/qwen-fast/speed-c1.json

@@ -0,0 +1,16 @@
+{
+  "concurrency": 1,
+  "requests": 8,
+  "output_tokens_per_request": 512,
+  "wall_seconds": 37.67895263497485,
+  "aggregate_output_tps": 108.7079049059861,
+  "median_decode_tps": 112.12193584833635,
+  "median_ttft_seconds": 0.07377005548914894,
+  "sampling": "greedy; ignore_eos for this throughput test only",
+  "speculation_counter_deltas": {
+    "vllm:spec_decode_num_drafts_total": 1320.0,
+    "vllm:spec_decode_num_draft_tokens_total": 3960.0,
+    "vllm:spec_decode_num_accepted_tokens_total": 2782.0
+  },
+  "decode_tps_note": "Client stream timing estimate; speculative decoding may deliver several tokens in one event."
+}

evaluation/swift10-full/manifest.json

@@ -0,0 +1,697 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-0",
+    "gsm8k-1",
+    "gsm8k-2",
+    "gsm8k-3",
+    "gsm8k-4",
+    "gsm8k-5",
+    "gsm8k-6",
+    "gsm8k-7",
+    "gsm8k-8",
+    "gsm8k-9",
+    "gsm8k-10",
+    "gsm8k-11",
+    "gsm8k-12",
+    "gsm8k-13",
+    "gsm8k-14",
+    "gsm8k-15",
+    "gsm8k-16",
+    "gsm8k-17",
+    "gsm8k-18",
+    "gsm8k-19",
+    "gsm8k-20",
+    "gsm8k-21",
+    "gsm8k-22",
+    "gsm8k-23",
+    "gsm8k-24",
+    "gsm8k-25",
+    "gsm8k-26",
+    "gsm8k-27",
+    "gsm8k-28",
+    "gsm8k-29",
+    "gsm8k-30",
+    "gsm8k-31",
+    "gsm8k-32",
+    "gsm8k-33",
+    "gsm8k-34",
+    "gsm8k-35",
+    "gsm8k-36",
+    "gsm8k-37",
+    "gsm8k-38",
+    "gsm8k-39",
+    "gsm8k-40",
+    "gsm8k-41",
+    "gsm8k-42",
+    "gsm8k-43",
+    "gsm8k-44",
+    "gsm8k-45",
+    "gsm8k-46",
+    "gsm8k-47",
+    "gsm8k-48",
+    "gsm8k-49",
+    "gsm8k-50",
+    "gsm8k-51",
+    "gsm8k-52",
+    "gsm8k-53",
+    "gsm8k-54",
+    "gsm8k-55",
+    "gsm8k-56",
+    "gsm8k-57",
+    "gsm8k-58",
+    "gsm8k-59",
+    "gsm8k-60",
+    "gsm8k-61",
+    "gsm8k-62",
+    "gsm8k-63",
+    "gsm8k-64",
+    "gsm8k-65",
+    "gsm8k-66",
+    "gsm8k-67",
+    "gsm8k-68",
+    "gsm8k-69",
+    "gsm8k-70",
+    "gsm8k-71",
+    "gsm8k-72",
+    "gsm8k-73",
+    "gsm8k-74",
+    "gsm8k-75",
+    "gsm8k-76",
+    "gsm8k-77",
+    "gsm8k-78",
+    "gsm8k-79",
+    "gsm8k-80",
+    "gsm8k-81",
+    "gsm8k-82",
+    "gsm8k-83",
+    "gsm8k-84",
+    "gsm8k-85",
+    "gsm8k-86",
+    "gsm8k-87",
+    "gsm8k-88",
+    "gsm8k-89",
+    "gsm8k-90",
+    "gsm8k-91",
+    "gsm8k-92",
+    "gsm8k-93",
+    "gsm8k-94",
+    "gsm8k-95",
+    "gsm8k-96",
+    "gsm8k-97",
+    "gsm8k-98",
+    "gsm8k-99",
+    "gsm8k-100",
+    "gsm8k-101",
+    "gsm8k-102",
+    "gsm8k-103",
+    "gsm8k-104",
+    "gsm8k-105",
+    "gsm8k-106",
+    "gsm8k-107",
+    "gsm8k-108",
+    "gsm8k-109",
+    "gsm8k-110",
+    "gsm8k-111",
+    "gsm8k-112",
+    "gsm8k-113",
+    "gsm8k-114",
+    "gsm8k-115",
+    "gsm8k-116",
+    "gsm8k-117",
+    "gsm8k-118",
+    "gsm8k-119",
+    "gsm8k-120",
+    "gsm8k-121",
+    "gsm8k-122",
+    "gsm8k-123",
+    "gsm8k-124",
+    "gsm8k-125",
+    "gsm8k-126",
+    "gsm8k-127",
+    "gsm8k-128",
+    "gsm8k-129",
+    "gsm8k-130",
+    "gsm8k-131",
+    "gsm8k-132",
+    "gsm8k-133",
+    "gsm8k-134",
+    "gsm8k-135",
+    "gsm8k-136",
+    "gsm8k-137",
+    "gsm8k-138",
+    "gsm8k-139",
+    "gsm8k-140",
+    "gsm8k-141",
+    "gsm8k-142",
+    "gsm8k-143",
+    "gsm8k-144",
+    "gsm8k-145",
+    "gsm8k-146",
+    "gsm8k-147",
+    "gsm8k-148",
+    "gsm8k-149",
+    "gsm8k-150",
+    "gsm8k-151",
+    "gsm8k-152",
+    "gsm8k-153",
+    "gsm8k-154",
+    "gsm8k-155",
+    "gsm8k-156",
+    "gsm8k-157",
+    "gsm8k-158",
+    "gsm8k-159",
+    "gsm8k-160",
+    "gsm8k-161",
+    "gsm8k-162",
+    "gsm8k-163",
+    "gsm8k-164",
+    "gsm8k-165",
+    "gsm8k-166",
+    "gsm8k-167",
+    "gsm8k-168",
+    "gsm8k-169",
+    "gsm8k-170",
+    "gsm8k-171",
+    "gsm8k-172",
+    "gsm8k-173",
+    "gsm8k-174",
+    "gsm8k-175",
+    "gsm8k-176",
+    "gsm8k-177",
+    "gsm8k-178",
+    "gsm8k-179",
+    "gsm8k-180",
+    "gsm8k-181",
+    "gsm8k-182",
+    "gsm8k-183",
+    "gsm8k-184",
+    "gsm8k-185",
+    "gsm8k-186",
+    "gsm8k-187",
+    "gsm8k-188",
+    "gsm8k-189",
+    "gsm8k-190",
+    "gsm8k-191",
+    "gsm8k-192",
+    "gsm8k-193",
+    "gsm8k-194",
+    "gsm8k-195",
+    "gsm8k-196",
+    "gsm8k-197",
+    "gsm8k-198",
+    "gsm8k-199",
+    "ifbench-0",
+    "ifbench-1",
+    "ifbench-2",
+    "ifbench-3",
+    "ifbench-4",
+    "ifbench-5",
+    "ifbench-6",
+    "ifbench-7",
+    "ifbench-8",
+    "ifbench-9",
+    "ifbench-10",
+    "ifbench-11",
+    "ifbench-12",
+    "ifbench-13",
+    "ifbench-14",
+    "ifbench-15",
+    "ifbench-16",
+    "ifbench-17",
+    "ifbench-18",
+    "ifbench-19",
+    "ifbench-20",
+    "ifbench-21",
+    "ifbench-22",
+    "ifbench-23",
+    "ifbench-24",
+    "ifbench-25",
+    "ifbench-26",
+    "ifbench-27",
+    "ifbench-28",
+    "ifbench-29",
+    "ifbench-30",
+    "ifbench-31",
+    "ifbench-32",
+    "ifbench-33",
+    "ifbench-34",
+    "ifbench-35",
+    "ifbench-36",
+    "ifbench-37",
+    "ifbench-38",
+    "ifbench-39",
+    "ifbench-40",
+    "ifbench-41",
+    "ifbench-42",
+    "ifbench-43",
+    "ifbench-44",
+    "ifbench-45",
+    "ifbench-46",
+    "ifbench-47",
+    "ifbench-48",
+    "ifbench-49",
+    "ifbench-50",
+    "ifbench-51",
+    "ifbench-52",
+    "ifbench-53",
+    "ifbench-54",
+    "ifbench-55",
+    "ifbench-56",
+    "ifbench-57",
+    "ifbench-58",
+    "ifbench-59",
+    "ifbench-60",
+    "ifbench-61",
+    "ifbench-62",
+    "ifbench-63",
+    "ifbench-64",
+    "ifbench-65",
+    "ifbench-66",
+    "ifbench-67",
+    "ifbench-68",
+    "ifbench-69",
+    "ifbench-70",
+    "ifbench-71",
+    "ifbench-72",
+    "ifbench-73",
+    "ifbench-74",
+    "ifbench-75",
+    "ifbench-76",
+    "ifbench-77",
+    "ifbench-78",
+    "ifbench-79",
+    "ifbench-80",
+    "ifbench-81",
+    "ifbench-82",
+    "ifbench-83",
+    "ifbench-84",
+    "ifbench-85",
+    "ifbench-86",
+    "ifbench-87",
+    "ifbench-88",
+    "ifbench-89",
+    "ifbench-90",
+    "ifbench-91",
+    "ifbench-92",
+    "ifbench-93",
+    "ifbench-94",
+    "ifbench-95",
+    "ifbench-96",
+    "ifbench-97",
+    "ifbench-98",
+    "ifbench-99",
+    "ifbench-100",
+    "ifbench-101",
+    "ifbench-102",
+    "ifbench-103",
+    "ifbench-104",
+    "ifbench-105",
+    "ifbench-106",
+    "ifbench-107",
+    "ifbench-108",
+    "ifbench-109",
+    "ifbench-110",
+    "ifbench-111",
+    "ifbench-112",
+    "ifbench-113",
+    "ifbench-114",
+    "ifbench-115",
+    "ifbench-116",
+    "ifbench-117",
+    "ifbench-118",
+    "ifbench-119",
+    "ifbench-120",
+    "ifbench-121",
+    "ifbench-122",
+    "ifbench-123",
+    "ifbench-124",
+    "ifbench-125",
+    "ifbench-126",
+    "ifbench-127",
+    "ifbench-128",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-131",
+    "ifbench-132",
+    "ifbench-133",
+    "ifbench-134",
+    "ifbench-135",
+    "ifbench-136",
+    "ifbench-137",
+    "ifbench-138",
+    "ifbench-139",
+    "ifbench-140",
+    "ifbench-141",
+    "ifbench-142",
+    "ifbench-143",
+    "ifbench-144",
+    "ifbench-145",
+    "ifbench-146",
+    "ifbench-147",
+    "ifbench-148",
+    "ifbench-149",
+    "ifbench-150",
+    "ifbench-151",
+    "ifbench-152",
+    "ifbench-153",
+    "ifbench-154",
+    "ifbench-155",
+    "ifbench-156",
+    "ifbench-157",
+    "ifbench-158",
+    "ifbench-159",
+    "ifbench-160",
+    "ifbench-161",
+    "ifbench-162",
+    "ifbench-163",
+    "ifbench-164",
+    "ifbench-165",
+    "ifbench-166",
+    "ifbench-167",
+    "ifbench-168",
+    "ifbench-169",
+    "ifbench-170",
+    "ifbench-171",
+    "ifbench-172",
+    "ifbench-173",
+    "ifbench-174",
+    "ifbench-175",
+    "ifbench-176",
+    "ifbench-177",
+    "ifbench-178",
+    "ifbench-179",
+    "ifbench-180",
+    "ifbench-181",
+    "ifbench-182",
+    "ifbench-183",
+    "ifbench-184",
+    "ifbench-185",
+    "ifbench-186",
+    "ifbench-187",
+    "ifbench-188",
+    "ifbench-189",
+    "ifbench-190",
+    "ifbench-191",
+    "ifbench-192",
+    "ifbench-193",
+    "ifbench-194",
+    "ifbench-195",
+    "ifbench-196",
+    "ifbench-197",
+    "ifbench-198",
+    "ifbench-199",
+    "ifbench-200",
+    "ifbench-201",
+    "ifbench-202",
+    "ifbench-203",
+    "ifbench-204",
+    "ifbench-205",
+    "ifbench-206",
+    "ifbench-207",
+    "ifbench-208",
+    "ifbench-209",
+    "ifbench-210",
+    "ifbench-211",
+    "ifbench-212",
+    "ifbench-213",
+    "ifbench-214",
+    "ifbench-215",
+    "ifbench-216",
+    "ifbench-217",
+    "ifbench-218",
+    "ifbench-219",
+    "ifbench-220",
+    "ifbench-221",
+    "ifbench-222",
+    "ifbench-223",
+    "ifbench-224",
+    "ifbench-225",
+    "ifbench-226",
+    "ifbench-227",
+    "ifbench-228",
+    "ifbench-229",
+    "ifbench-230",
+    "ifbench-231",
+    "ifbench-232",
+    "ifbench-233",
+    "ifbench-234",
+    "ifbench-235",
+    "ifbench-236",
+    "ifbench-237",
+    "ifbench-238",
+    "ifbench-239",
+    "ifbench-240",
+    "ifbench-241",
+    "ifbench-242",
+    "ifbench-243",
+    "ifbench-244",
+    "ifbench-245",
+    "ifbench-246",
+    "ifbench-247",
+    "ifbench-248",
+    "ifbench-249",
+    "ifbench-250",
+    "ifbench-251",
+    "ifbench-252",
+    "ifbench-253",
+    "ifbench-254",
+    "ifbench-255",
+    "ifbench-256",
+    "ifbench-257",
+    "ifbench-258",
+    "ifbench-259",
+    "ifbench-260",
+    "ifbench-261",
+    "ifbench-262",
+    "ifbench-263",
+    "ifbench-264",
+    "ifbench-265",
+    "ifbench-266",
+    "ifbench-267",
+    "ifbench-268",
+    "ifbench-269",
+    "ifbench-270",
+    "ifbench-271",
+    "ifbench-272",
+    "ifbench-273",
+    "ifbench-274",
+    "ifbench-275",
+    "ifbench-276",
+    "ifbench-277",
+    "ifbench-278",
+    "ifbench-279",
+    "ifbench-280",
+    "ifbench-281",
+    "ifbench-282",
+    "ifbench-283",
+    "ifbench-284",
+    "ifbench-285",
+    "ifbench-286",
+    "ifbench-287",
+    "ifbench-288",
+    "ifbench-289",
+    "ifbench-290",
+    "ifbench-291",
+    "ifbench-292",
+    "ifbench-293",
+    "ifbench-294",
+    "ifbench-295",
+    "ifbench-296",
+    "ifbench-297",
+    "ifbench-298",
+    "ifbench-299",
+    "lcb-atcoder-abc322_b",
+    "lcb-atcoder-abc381_a",
+    "lcb-atcoder-abc393_a",
+    "lcb-atcoder-abc321_b",
+    "lcb-atcoder-abc359_a",
+    "lcb-atcoder-abc329_b",
+    "lcb-atcoder-abc353_a",
+    "lcb-atcoder-abc355_b",
+    "lcb-atcoder-abc326_b",
+    "lcb-codeforces-1873_B",
+    "lcb-atcoder-abc356_a",
+    "lcb-atcoder-abc356_b",
+    "lcb-atcoder-abc375_a",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc311_b",
+    "lcb-atcoder-abc378_b",
+    "lcb-atcoder-abc309_b",
+    "lcb-atcoder-abc325_a",
+    "lcb-atcoder-abc301_b",
+    "lcb-atcoder-abc391_a",
+    "lcb-atcoder-abc399_b",
+    "lcb-atcoder-abc354_a",
+    "lcb-atcoder-abc352_b",
+    "lcb-atcoder-abc382_b",
+    "lcb-atcoder-abc332_b",
+    "lcb-atcoder-abc343_b",
+    "lcb-atcoder-abc361_a",
+    "lcb-atcoder-abc362_a",
+    "lcb-atcoder-abc328_a",
+    "lcb-atcoder-abc393_b",
+    "lcb-atcoder-abc352_a",
+    "lcb-atcoder-abc310_a",
+    "lcb-atcoder-abc365_a",
+    "lcb-atcoder-abc371_b",
+    "lcb-atcoder-abc367_c",
+    "lcb-atcoder-abc334_b",
+    "lcb-atcoder-abc385_c",
+    "lcb-atcoder-abc307_c",
+    "lcb-atcoder-abc338_c",
+    "lcb-atcoder-abc303_d",
+    "lcb-atcoder-abc342_c",
+    "lcb-atcoder-abc319_d",
+    "lcb-atcoder-abc315_d",
+    "lcb-atcoder-abc309_c",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc343_d",
+    "lcb-atcoder-abc397_b",
+    "lcb-atcoder-abc370_c",
+    "lcb-atcoder-abc375_c",
+    "lcb-atcoder-abc368_c",
+    "lcb-atcoder-abc325_b",
+    "lcb-atcoder-abc323_c",
+    "lcb-atcoder-abc377_c",
+    "lcb-atcoder-abc383_d",
+    "lcb-atcoder-arc189_a",
+    "lcb-codeforces-1883_C",
+    "lcb-atcoder-abc371_c",
+    "lcb-atcoder-abc380_c",
+    "lcb-atcoder-abc378_c",
+    "lcb-atcoder-abc366_c",
+    "lcb-atcoder-abc397_c",
+    "lcb-atcoder-abc339_c",
+    "lcb-atcoder-abc324_c",
+    "lcb-atcoder-abc355_c",
+    "lcb-atcoder-abc358_c",
+    "lcb-atcoder-abc321_d",
+    "lcb-atcoder-abc334_c",
+    "lcb-atcoder-arc195_c",
+    "lcb-atcoder-arc184_c",
+    "lcb-atcoder-abc377_e",
+    "lcb-atcoder-arc186_e",
+    "lcb-atcoder-abc398_f",
+    "lcb-atcoder-abc330_e",
+    "lcb-atcoder-abc396_e",
+    "lcb-atcoder-arc192_b",
+    "lcb-atcoder-abc391_f",
+    "lcb-atcoder-arc195_b",
+    "lcb-atcoder-abc384_g",
+    "lcb-atcoder-abc343_e",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc333_e",
+    "lcb-atcoder-abc341_e",
+    "lcb-atcoder-abc400_g",
+    "lcb-atcoder-abc362_d",
+    "lcb-codeforces-1899_D",
+    "lcb-atcoder-abc363_f",
+    "lcb-atcoder-abc382_d",
+    "lcb-atcoder-abc331_e",
+    "lcb-atcoder-abc351_e",
+    "lcb-atcoder-arc194_b",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-arc188_d",
+    "lcb-atcoder-abc368_e",
+    "lcb-atcoder-abc301_e",
+    "lcb-atcoder-arc194_e",
+    "lcb-atcoder-abc379_e",
+    "lcb-atcoder-abc360_e",
+    "lcb-atcoder-abc305_e",
+    "lcb-atcoder-arc186_a",
+    "lcb-atcoder-abc362_e",
+    "tool-0",
+    "tool-1",
+    "tool-2",
+    "tool-3",
+    "tool-4",
+    "tool-5",
+    "tool-6",
+    "tool-7",
+    "tool-8",
+    "tool-9",
+    "tool-10",
+    "tool-11",
+    "tool-12",
+    "tool-13",
+    "tool-14",
+    "tool-15",
+    "tool-16",
+    "tool-17",
+    "tool-18",
+    "tool-19",
+    "json-20",
+    "json-21",
+    "json-22",
+    "json-23",
+    "json-24",
+    "json-25",
+    "json-26",
+    "json-27",
+    "json-28",
+    "json-29"
+  ],
+  "concurrency": 2,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Swift-Qwen3.8-27b-W4A16-AWQ",
+    "config_sha256": "13bcc57aaa12c046cf379d0680f6a8dd144db48696c294798daae0d485a1e255",
+    "index_sha256": "30bde9db796be88667575f52843af2a23538008cadfeb7d58cd1fd2201bd07b6",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00005-of-00007.safetensors": {
+        "size": 2179699760,
+        "mtime_ns": 1789502506919812372
+      },
+      "model-00001-of-00007.safetensors": {
+        "size": 2212761608,
+        "mtime_ns": 1789544289096628097
+      },
+      "model-00006-of-00007.safetensors": {
+        "size": 2186211856,
+        "mtime_ns": 1789502504446803803
+      },
+      "model-00003-of-00007.safetensors": {
+        "size": 2179699752,
+        "mtime_ns": 1789502419990506742
+      },
+      "model-00002-of-00007.safetensors": {
+        "size": 2186190664,
+        "mtime_ns": 1789502424392522448
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 212992352,
+        "mtime_ns": 1789544970849697727
+      },
+      "model-nonquant.safetensors": {
+        "size": 431364472,
+        "mtime_ns": 1789544293085638815
+      },
+      "model-00007-of-00007.safetensors": {
+        "size": 3071134144,
+        "mtime_ns": 1789544275606591269
+      },
+      "model-00004-of-00007.safetensors": {
+        "size": 2179699760,
+        "mtime_ns": 1789502463524660967
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/swift10-full/summary.json

@@ -0,0 +1,88 @@
+{
+  "created": "2026-09-27T13:16:04Z",
+  "concurrency": 2,
+  "resumed": false,
+  "new_run_wall_seconds": 20949.800895433058,
+  "new_tasks": 630,
+  "suites": {
+    "gsm8k": {
+      "attempted": 200,
+      "correct": 196,
+      "accuracy": 0.98,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 380.925,
+      "mean_input_tokens": 93.27,
+      "mean_total_tokens": 474.195,
+      "mean_model_seconds": 3.4486288186194725,
+      "median_model_seconds": 2.997326802520547,
+      "p95_model_seconds": 5.85073896299582,
+      "summed_request_seconds_per_correct": 3.5190089985912985,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 300,
+      "correct": 220,
+      "accuracy": 0.7333333333333333,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 4998.316666666667,
+      "mean_input_tokens": 125.44,
+      "mean_total_tokens": 5123.756666666667,
+      "mean_model_seconds": 55.58578871234068,
+      "median_model_seconds": 27.256246656470466,
+      "p95_model_seconds": 223.28189022897277,
+      "summed_request_seconds_per_correct": 75.79880278955547,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 100,
+      "correct": 89,
+      "accuracy": 0.89,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 1,
+      "errors": 0,
+      "truncated": 1,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 17271.87,
+      "mean_input_tokens": 653.75,
+      "mean_total_tokens": 17925.62,
+      "mean_model_seconds": 243.10798836177565,
+      "median_model_seconds": 61.351035209954716,
+      "p95_model_seconds": 1182.111728347023,
+      "summed_request_seconds_per_correct": 273.1550431031187,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 30,
+      "correct": 28,
+      "accuracy": 0.9333333333333333,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 40.333333333333336,
+      "mean_input_tokens": 481.3333333333333,
+      "mean_total_tokens": 521.6666666666666,
+      "mean_model_seconds": 0.994276461203117,
+      "median_model_seconds": 1.245341427042149,
+      "p95_model_seconds": 1.324655186966993,
+      "summed_request_seconds_per_correct": 1.065296208431911,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [
+    "lcb-atcoder-arc184_c"
+  ],
+  "suite_wall_seconds": 20949.800895433058,
+  "wall_seconds_per_correct": 39.30544258054983
+}

evaluation/swift10-pilot/manifest.json

@@ -0,0 +1,87 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-27",
+    "gsm8k-60",
+    "gsm8k-105",
+    "gsm8k-109",
+    "gsm8k-152",
+    "ifbench-21",
+    "ifbench-76",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-268",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-abc368_e",
+    "tool-15",
+    "tool-16",
+    "json-22",
+    "json-24",
+    "json-26"
+  ],
+  "concurrency": 1,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Swift-Qwen3.8-27b-W4A16-AWQ",
+    "config_sha256": "13bcc57aaa12c046cf379d0680f6a8dd144db48696c294798daae0d485a1e255",
+    "index_sha256": "30bde9db796be88667575f52843af2a23538008cadfeb7d58cd1fd2201bd07b6",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00005-of-00007.safetensors": {
+        "size": 2179699760,
+        "mtime_ns": 1789502506919812372
+      },
+      "model-00001-of-00007.safetensors": {
+        "size": 2212761608,
+        "mtime_ns": 1789544289096628097
+      },
+      "model-00006-of-00007.safetensors": {
+        "size": 2186211856,
+        "mtime_ns": 1789502504446803803
+      },
+      "model-00003-of-00007.safetensors": {
+        "size": 2179699752,
+        "mtime_ns": 1789502419990506742
+      },
+      "model-00002-of-00007.safetensors": {
+        "size": 2186190664,
+        "mtime_ns": 1789502424392522448
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 212992352,
+        "mtime_ns": 1789544970849697727
+      },
+      "model-nonquant.safetensors": {
+        "size": 431364472,
+        "mtime_ns": 1789544293085638815
+      },
+      "model-00007-of-00007.safetensors": {
+        "size": 3071134144,
+        "mtime_ns": 1789544275606591269
+      },
+      "model-00004-of-00007.safetensors": {
+        "size": 2179699760,
+        "mtime_ns": 1789502463524660967
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/swift10-pilot/summary.json

@@ -0,0 +1,86 @@
+{
+  "created": "2026-09-27T07:26:50Z",
+  "concurrency": 1,
+  "resumed": false,
+  "new_run_wall_seconds": 2322.5720736039802,
+  "new_tasks": 20,
+  "suites": {
+    "gsm8k": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 262.6,
+      "mean_input_tokens": 77.4,
+      "mean_total_tokens": 340,
+      "mean_model_seconds": 2.3007265538093633,
+      "median_model_seconds": 2.3031491080182604,
+      "p95_model_seconds": 3.3395710720214993,
+      "summed_request_seconds_per_correct": 2.3007265538093633,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 5,
+      "correct": 3,
+      "accuracy": 0.6,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 2349.2,
+      "mean_input_tokens": 134.6,
+      "mean_total_tokens": 2483.8,
+      "mean_model_seconds": 25.789347258815543,
+      "median_model_seconds": 11.154919354012236,
+      "p95_model_seconds": 89.14424104097998,
+      "summed_request_seconds_per_correct": 42.98224543135924,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 33674,
+      "mean_input_tokens": 742.4,
+      "mean_total_tokens": 34416.4,
+      "mean_model_seconds": 432.0463959699846,
+      "median_model_seconds": 277.599615551997,
+      "p95_model_seconds": 1155.1297870659619,
+      "summed_request_seconds_per_correct": 432.0463959699846,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 5,
+      "correct": 4,
+      "accuracy": 0.8,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 34.6,
+      "mean_input_tokens": 330.6,
+      "mean_total_tokens": 365.2,
+      "mean_model_seconds": 0.6769871199969202,
+      "median_model_seconds": 0.5235941839637235,
+      "p95_model_seconds": 1.125315964978654,
+      "summed_request_seconds_per_correct": 0.8462338999961503,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [],
+  "suite_wall_seconds": 2322.5720736039802,
+  "wall_seconds_per_correct": 136.62188668258707
+}

evaluation/swift10/perplexity.json

@@ -0,0 +1,659 @@
+{
+  "input_sha256": "57c83ffe7c0dfba2b869f6a379d2df3f0bc0d055069b49569a2154336151a718",
+  "windows": [
+    {
+      "id": 0,
+      "language": "en",
+      "tokens": 283,
+      "logprob_sum": -593.3995295942841
+    },
+    {
+      "id": 1,
+      "language": "en",
+      "tokens": 313,
+      "logprob_sum": -700.5331317378659
+    },
+    {
+      "id": 2,
+      "language": "en",
+      "tokens": 284,
+      "logprob_sum": -724.3817458400572
+    },
+    {
+      "id": 3,
+      "language": "en",
+      "tokens": 286,
+      "logprob_sum": -774.8060930467109
+    },
+    {
+      "id": 4,
+      "language": "en",
+      "tokens": 279,
+      "logprob_sum": -573.8085490163357
+    },
+    {
+      "id": 5,
+      "language": "en",
+      "tokens": 251,
+      "logprob_sum": -647.1657810044508
+    },
+    {
+      "id": 6,
+      "language": "en",
+      "tokens": 258,
+      "logprob_sum": -613.3406054319057
+    },
+    {
+      "id": 7,
+      "language": "en",
+      "tokens": 258,
+      "logprob_sum": -598.2336515752831
+    },
+    {
+      "id": 8,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -655.9989099322702
+    },
+    {
+      "id": 9,
+      "language": "en",
+      "tokens": 278,
+      "logprob_sum": -620.1450639520117
+    },
+    {
+      "id": 10,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -750.830778833566
+    },
+    {
+      "id": 11,
+      "language": "en",
+      "tokens": 288,
+      "logprob_sum": -780.6144631060088
+    },
+    {
+      "id": 12,
+      "language": "en",
+      "tokens": 277,
+      "logprob_sum": -614.716890099935
+    },
+    {
+      "id": 13,
+      "language": "en",
+      "tokens": 291,
+      "logprob_sum": -711.6267314220386
+    },
+    {
+      "id": 14,
+      "language": "en",
+      "tokens": 260,
+      "logprob_sum": -671.5577233368549
+    },
+    {
+      "id": 15,
+      "language": "en",
+      "tokens": 250,
+      "logprob_sum": -775.6550934422994
+    },
+    {
+      "id": 16,
+      "language": "en",
+      "tokens": 252,
+      "logprob_sum": -756.1478926279815
+    },
+    {
+      "id": 17,
+      "language": "en",
+      "tokens": 272,
+      "logprob_sum": -783.3444251654037
+    },
+    {
+      "id": 18,
+      "language": "en",
+      "tokens": 240,
+      "logprob_sum": -676.2792851193226
+    },
+    {
+      "id": 19,
+      "language": "en",
+      "tokens": 244,
+      "logprob_sum": -561.7464151983149
+    },
+    {
+      "id": 20,
+      "language": "en",
+      "tokens": 274,
+      "logprob_sum": -689.3211253857444
+    },
+    {
+      "id": 21,
+      "language": "en",
+      "tokens": 299,
+      "logprob_sum": -719.0637272568165
+    },
+    {
+      "id": 22,
+      "language": "en",
+      "tokens": 318,
+      "logprob_sum": -828.7790999828721
+    },
+    {
+      "id": 23,
+      "language": "en",
+      "tokens": 263,
+      "logprob_sum": -680.0565696356061
+    },
+    {
+      "id": 24,
+      "language": "en",
+      "tokens": 239,
+      "logprob_sum": -715.9142811552592
+    },
+    {
+      "id": 25,
+      "language": "en",
+      "tokens": 293,
+      "logprob_sum": -489.55537099707726
+    },
+    {
+      "id": 26,
+      "language": "en",
+      "tokens": 303,
+      "logprob_sum": -682.4112273987994
+    },
+    {
+      "id": 27,
+      "language": "en",
+      "tokens": 276,
+      "logprob_sum": -620.8538292451813
+    },
+    {
+      "id": 28,
+      "language": "en",
+      "tokens": 282,
+      "logprob_sum": -738.9185054079862
+    },
+    {
+      "id": 29,
+      "language": "en",
+      "tokens": 298,
+      "logprob_sum": -686.1773185838956
+    },
+    {
+      "id": 30,
+      "language": "en",
+      "tokens": 338,
+      "logprob_sum": -473.0945813098033
+    },
+    {
+      "id": 31,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -630.0955357118728
+    },
+    {
+      "id": 32,
+      "language": "en",
+      "tokens": 310,
+      "logprob_sum": -810.4693004913643
+    },
+    {
+      "id": 33,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -799.6141130025499
+    },
+    {
+      "id": 34,
+      "language": "en",
+      "tokens": 302,
+      "logprob_sum": -712.025332148276
+    },
+    {
+      "id": 35,
+      "language": "en",
+      "tokens": 286,
+      "logprob_sum": -538.7282043778414
+    },
+    {
+      "id": 36,
+      "language": "en",
+      "tokens": 244,
+      "logprob_sum": -593.2613173677673
+    },
+    {
+      "id": 37,
+      "language": "en",
+      "tokens": 280,
+      "logprob_sum": -498.5803779190901
+    },
+    {
+      "id": 38,
+      "language": "en",
+      "tokens": 315,
+      "logprob_sum": -604.9404760405196
+    },
+    {
+      "id": 39,
+      "language": "en",
+      "tokens": 283,
+      "logprob_sum": -652.8633822322918
+    },
+    {
+      "id": 40,
+      "language": "da",
+      "tokens": 319,
+      "logprob_sum": -517.1994196986234
+    },
+    {
+      "id": 41,
+      "language": "da",
+      "tokens": 334,
+      "logprob_sum": -793.5113881794969
+    },
+    {
+      "id": 42,
+      "language": "da",
+      "tokens": 321,
+      "logprob_sum": -649.114615170889
+    },
+    {
+      "id": 43,
+      "language": "da",
+      "tokens": 355,
+      "logprob_sum": -777.4246189849518
+    },
+    {
+      "id": 44,
+      "language": "da",
+      "tokens": 372,
+      "logprob_sum": -790.426270207965
+    },
+    {
+      "id": 45,
+      "language": "da",
+      "tokens": 360,
+      "logprob_sum": -605.4395699449469
+    },
+    {
+      "id": 46,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -1271.5344694513333
+    },
+    {
+      "id": 47,
+      "language": "da",
+      "tokens": 372,
+      "logprob_sum": -609.4069087250737
+    },
+    {
+      "id": 48,
+      "language": "da",
+      "tokens": 356,
+      "logprob_sum": -803.8936290113306
+    },
+    {
+      "id": 49,
+      "language": "da",
+      "tokens": 269,
+      "logprob_sum": -1456.3805292455363
+    },
+    {
+      "id": 50,
+      "language": "da",
+      "tokens": 334,
+      "logprob_sum": -580.4934725780004
+    },
+    {
+      "id": 51,
+      "language": "da",
+      "tokens": 364,
+      "logprob_sum": -613.1963399831075
+    },
+    {
+      "id": 52,
+      "language": "da",
+      "tokens": 364,
+      "logprob_sum": -667.4508653348885
+    },
+    {
+      "id": 53,
+      "language": "da",
+      "tokens": 378,
+      "logprob_sum": -454.4978262218378
+    },
+    {
+      "id": 54,
+      "language": "da",
+      "tokens": 400,
+      "logprob_sum": -832.7972447402817
+    },
+    {
+      "id": 55,
+      "language": "da",
+      "tokens": 319,
+      "logprob_sum": -1565.9580685560359
+    },
+    {
+      "id": 56,
+      "language": "da",
+      "tokens": 329,
+      "logprob_sum": -603.6377663023723
+    },
+    {
+      "id": 57,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -870.4163886572023
+    },
+    {
+      "id": 58,
+      "language": "da",
+      "tokens": 336,
+      "logprob_sum": -1593.2642377602024
+    },
+    {
+      "id": 59,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -991.8582370316872
+    },
+    {
+      "id": 60,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -496.14374373139617
+    },
+    {
+      "id": 61,
+      "language": "da",
+      "tokens": 349,
+      "logprob_sum": -864.6478007899059
+    },
+    {
+      "id": 62,
+      "language": "da",
+      "tokens": 349,
+      "logprob_sum": -564.6994042088133
+    },
+    {
+      "id": 63,
+      "language": "da",
+      "tokens": 356,
+      "logprob_sum": -674.2995392707544
+    },
+    {
+      "id": 64,
+      "language": "da",
+      "tokens": 388,
+      "logprob_sum": -746.9267507952973
+    },
+    {
+      "id": 65,
+      "language": "da",
+      "tokens": 342,
+      "logprob_sum": -1581.7085867874302
+    },
+    {
+      "id": 66,
+      "language": "da",
+      "tokens": 376,
+      "logprob_sum": -684.6263309357491
+    },
+    {
+      "id": 67,
+      "language": "da",
+      "tokens": 388,
+      "logprob_sum": -585.0814573411387
+    },
+    {
+      "id": 68,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -679.226675238438
+    },
+    {
+      "id": 69,
+      "language": "da",
+      "tokens": 335,
+      "logprob_sum": -651.8282408997134
+    },
+    {
+      "id": 70,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -744.8929153228212
+    },
+    {
+      "id": 71,
+      "language": "da",
+      "tokens": 371,
+      "logprob_sum": -1395.236346740101
+    },
+    {
+      "id": 72,
+      "language": "da",
+      "tokens": 371,
+      "logprob_sum": -710.4150057600818
+    },
+    {
+      "id": 73,
+      "language": "da",
+      "tokens": 325,
+      "logprob_sum": -587.0278204807751
+    },
+    {
+      "id": 74,
+      "language": "da",
+      "tokens": 381,
+      "logprob_sum": -1015.3197047918511
+    },
+    {
+      "id": 75,
+      "language": "da",
+      "tokens": 265,
+      "logprob_sum": -1385.7349133007228
+    },
+    {
+      "id": 76,
+      "language": "da",
+      "tokens": 348,
+      "logprob_sum": -905.108547452197
+    },
+    {
+      "id": 77,
+      "language": "da",
+      "tokens": 338,
+      "logprob_sum": -602.2395354474211
+    },
+    {
+      "id": 78,
+      "language": "da",
+      "tokens": 346,
+      "logprob_sum": -875.796343427799
+    },
+    {
+      "id": 79,
+      "language": "da",
+      "tokens": 359,
+      "logprob_sum": -594.3750530699126
+    },
+    {
+      "id": 80,
+      "language": "code",
+      "tokens": 320,
+      "logprob_sum": -274.7489258961019
+    },
+    {
+      "id": 81,
+      "language": "code",
+      "tokens": 282,
+      "logprob_sum": -430.6838706552262
+    },
+    {
+      "id": 82,
+      "language": "code",
+      "tokens": 278,
+      "logprob_sum": -260.0947107755287
+    },
+    {
+      "id": 83,
+      "language": "code",
+      "tokens": 273,
+      "logprob_sum": -411.2611253242603
+    },
+    {
+      "id": 84,
+      "language": "code",
+      "tokens": 253,
+      "logprob_sum": -197.21853261078218
+    },
+    {
+      "id": 85,
+      "language": "code",
+      "tokens": 239,
+      "logprob_sum": -461.1512319083995
+    },
+    {
+      "id": 86,
+      "language": "code",
+      "tokens": 272,
+      "logprob_sum": -377.2601323130457
+    },
+    {
+      "id": 87,
+      "language": "code",
+      "tokens": 253,
+      "logprob_sum": -282.7043757491738
+    },
+    {
+      "id": 88,
+      "language": "code",
+      "tokens": 317,
+      "logprob_sum": -288.68833832504083
+    },
+    {
+      "id": 89,
+      "language": "code",
+      "tokens": 298,
+      "logprob_sum": -290.93484990602883
+    },
+    {
+      "id": 90,
+      "language": "code",
+      "tokens": 280,
+      "logprob_sum": -305.8559028699442
+    },
+    {
+      "id": 91,
+      "language": "code",
+      "tokens": 265,
+      "logprob_sum": -387.24696855193054
+    },
+    {
+      "id": 92,
+      "language": "code",
+      "tokens": 305,
+      "logprob_sum": -197.82955161913992
+    },
+    {
+      "id": 93,
+      "language": "code",
+      "tokens": 310,
+      "logprob_sum": -351.8639510574991
+    },
+    {
+      "id": 94,
+      "language": "code",
+      "tokens": 278,
+      "logprob_sum": -236.71723654100515
+    },
+    {
+      "id": 95,
+      "language": "code",
+      "tokens": 301,
+      "logprob_sum": -317.9189716539063
+    },
+    {
+      "id": 96,
+      "language": "code",
+      "tokens": 311,
+      "logprob_sum": -186.151134865902
+    },
+    {
+      "id": 97,
+      "language": "code",
+      "tokens": 312,
+      "logprob_sum": -227.76212669502684
+    },
+    {
+      "id": 98,
+      "language": "code",
+      "tokens": 325,
+      "logprob_sum": -195.6774061962218
+    },
+    {
+      "id": 99,
+      "language": "code",
+      "tokens": 291,
+      "logprob_sum": -313.3808719475601
+    },
+    {
+      "id": 100,
+      "language": "code",
+      "tokens": 321,
+      "logprob_sum": -436.92633147341314
+    },
+    {
+      "id": 101,
+      "language": "code",
+      "tokens": 318,
+      "logprob_sum": -442.086307042553
+    },
+    {
+      "id": 102,
+      "language": "code",
+      "tokens": 316,
+      "logprob_sum": -522.1931145482638
+    },
+    {
+      "id": 103,
+      "language": "code",
+      "tokens": 307,
+      "logprob_sum": -305.42324257145077
+    },
+    {
+      "id": 104,
+      "language": "code",
+      "tokens": 254,
+      "logprob_sum": -407.6405011889765
+    },
+    {
+      "id": 105,
+      "language": "code",
+      "tokens": 275,
+      "logprob_sum": -499.1410040079517
+    }
+  ],
+  "scores": {
+    "en": {
+      "tokens": 11175,
+      "ppl": 10.953418316428
+    },
+    "all": {
+      "tokens": 32646,
+      "ppl": 8.214905986747409
+    },
+    "da": {
+      "tokens": 13917,
+      "ppl": 11.017187284986234
+    },
+    "code": {
+      "tokens": 7554,
+      "ppl": 3.1255271414775376
+    }
+  }
+}

evaluation/swift10/server.json

@@ -0,0 +1,61 @@
+{
+  "created": "2026-09-27T06:45:30Z",
+  "model": "<WORKSPACE>/models/Swift-Qwen3.8-27b-W4A16-AWQ",
+  "config_sha256": "13bcc57aaa12c046cf379d0680f6a8dd144db48696c294798daae0d485a1e255",
+  "index_sha256": "30bde9db796be88667575f52843af2a23538008cadfeb7d58cd1fd2201bd07b6",
+  "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+  "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523",
+  "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+  "shards": {
+    "model-00005-of-00007.safetensors": {
+      "size": 2179699760,
+      "mtime_ns": 1789502506919812372
+    },
+    "model-00001-of-00007.safetensors": {
+      "size": 2212761608,
+      "mtime_ns": 1789544289096628097
+    },
+    "model-00006-of-00007.safetensors": {
+      "size": 2186211856,
+      "mtime_ns": 1789502504446803803
+    },
+    "model-00003-of-00007.safetensors": {
+      "size": 2179699752,
+      "mtime_ns": 1789502419990506742
+    },
+    "model-00002-of-00007.safetensors": {
+      "size": 2186190664,
+      "mtime_ns": 1789502424392522448
+    },
+    "model_extra_tensors.safetensors": {
+      "size": 212992352,
+      "mtime_ns": 1789544970849697727
+    },
+    "model-nonquant.safetensors": {
+      "size": 431364472,
+      "mtime_ns": 1789544293085638815
+    },
+    "model-00007-of-00007.safetensors": {
+      "size": 3071134144,
+      "mtime_ns": 1789544275606591269
+    },
+    "model-00004-of-00007.safetensors": {
+      "size": 2179699760,
+      "mtime_ns": 1789502463524660967
+    }
+  },
+  "mode": "single-user",
+  "int8_activations": false,
+  "speculation": "mtp",
+  "max_model_len": 150000,
+  "max_num_seqs": 8,
+  "kv_cache_dtype": "fp8",
+  "prefix_cache": true,
+  "gpu_memory_utilization": 0.93,
+  "pid": 1665267,
+  "serving_profile": "local-single-user",
+  "vision": true,
+  "draft_tokens": 3,
+  "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+  "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+}

evaluation/swift10/speed-c1.json

@@ -0,0 +1,16 @@
+{
+  "concurrency": 1,
+  "requests": 8,
+  "output_tokens_per_request": 512,
+  "wall_seconds": 39.691121668962296,
+  "aggregate_output_tps": 103.19688201714375,
+  "median_decode_tps": 105.94877596587726,
+  "median_ttft_seconds": 0.07828654450713657,
+  "sampling": "greedy; ignore_eos for this throughput test only",
+  "speculation_counter_deltas": {
+    "vllm:spec_decode_num_drafts_total": 1320.0,
+    "vllm:spec_decode_num_draft_tokens_total": 3960.0,
+    "vllm:spec_decode_num_accepted_tokens_total": 2776.0
+  },
+  "decode_tps_note": "Client stream timing estimate; speculative decoding may deliver several tokens in one event."
+}

evaluation/swift15-baseline-full/manifest.json

@@ -0,0 +1,693 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-0",
+    "gsm8k-1",
+    "gsm8k-2",
+    "gsm8k-3",
+    "gsm8k-4",
+    "gsm8k-5",
+    "gsm8k-6",
+    "gsm8k-7",
+    "gsm8k-8",
+    "gsm8k-9",
+    "gsm8k-10",
+    "gsm8k-11",
+    "gsm8k-12",
+    "gsm8k-13",
+    "gsm8k-14",
+    "gsm8k-15",
+    "gsm8k-16",
+    "gsm8k-17",
+    "gsm8k-18",
+    "gsm8k-19",
+    "gsm8k-20",
+    "gsm8k-21",
+    "gsm8k-22",
+    "gsm8k-23",
+    "gsm8k-24",
+    "gsm8k-25",
+    "gsm8k-26",
+    "gsm8k-27",
+    "gsm8k-28",
+    "gsm8k-29",
+    "gsm8k-30",
+    "gsm8k-31",
+    "gsm8k-32",
+    "gsm8k-33",
+    "gsm8k-34",
+    "gsm8k-35",
+    "gsm8k-36",
+    "gsm8k-37",
+    "gsm8k-38",
+    "gsm8k-39",
+    "gsm8k-40",
+    "gsm8k-41",
+    "gsm8k-42",
+    "gsm8k-43",
+    "gsm8k-44",
+    "gsm8k-45",
+    "gsm8k-46",
+    "gsm8k-47",
+    "gsm8k-48",
+    "gsm8k-49",
+    "gsm8k-50",
+    "gsm8k-51",
+    "gsm8k-52",
+    "gsm8k-53",
+    "gsm8k-54",
+    "gsm8k-55",
+    "gsm8k-56",
+    "gsm8k-57",
+    "gsm8k-58",
+    "gsm8k-59",
+    "gsm8k-60",
+    "gsm8k-61",
+    "gsm8k-62",
+    "gsm8k-63",
+    "gsm8k-64",
+    "gsm8k-65",
+    "gsm8k-66",
+    "gsm8k-67",
+    "gsm8k-68",
+    "gsm8k-69",
+    "gsm8k-70",
+    "gsm8k-71",
+    "gsm8k-72",
+    "gsm8k-73",
+    "gsm8k-74",
+    "gsm8k-75",
+    "gsm8k-76",
+    "gsm8k-77",
+    "gsm8k-78",
+    "gsm8k-79",
+    "gsm8k-80",
+    "gsm8k-81",
+    "gsm8k-82",
+    "gsm8k-83",
+    "gsm8k-84",
+    "gsm8k-85",
+    "gsm8k-86",
+    "gsm8k-87",
+    "gsm8k-88",
+    "gsm8k-89",
+    "gsm8k-90",
+    "gsm8k-91",
+    "gsm8k-92",
+    "gsm8k-93",
+    "gsm8k-94",
+    "gsm8k-95",
+    "gsm8k-96",
+    "gsm8k-97",
+    "gsm8k-98",
+    "gsm8k-99",
+    "gsm8k-100",
+    "gsm8k-101",
+    "gsm8k-102",
+    "gsm8k-103",
+    "gsm8k-104",
+    "gsm8k-105",
+    "gsm8k-106",
+    "gsm8k-107",
+    "gsm8k-108",
+    "gsm8k-109",
+    "gsm8k-110",
+    "gsm8k-111",
+    "gsm8k-112",
+    "gsm8k-113",
+    "gsm8k-114",
+    "gsm8k-115",
+    "gsm8k-116",
+    "gsm8k-117",
+    "gsm8k-118",
+    "gsm8k-119",
+    "gsm8k-120",
+    "gsm8k-121",
+    "gsm8k-122",
+    "gsm8k-123",
+    "gsm8k-124",
+    "gsm8k-125",
+    "gsm8k-126",
+    "gsm8k-127",
+    "gsm8k-128",
+    "gsm8k-129",
+    "gsm8k-130",
+    "gsm8k-131",
+    "gsm8k-132",
+    "gsm8k-133",
+    "gsm8k-134",
+    "gsm8k-135",
+    "gsm8k-136",
+    "gsm8k-137",
+    "gsm8k-138",
+    "gsm8k-139",
+    "gsm8k-140",
+    "gsm8k-141",
+    "gsm8k-142",
+    "gsm8k-143",
+    "gsm8k-144",
+    "gsm8k-145",
+    "gsm8k-146",
+    "gsm8k-147",
+    "gsm8k-148",
+    "gsm8k-149",
+    "gsm8k-150",
+    "gsm8k-151",
+    "gsm8k-152",
+    "gsm8k-153",
+    "gsm8k-154",
+    "gsm8k-155",
+    "gsm8k-156",
+    "gsm8k-157",
+    "gsm8k-158",
+    "gsm8k-159",
+    "gsm8k-160",
+    "gsm8k-161",
+    "gsm8k-162",
+    "gsm8k-163",
+    "gsm8k-164",
+    "gsm8k-165",
+    "gsm8k-166",
+    "gsm8k-167",
+    "gsm8k-168",
+    "gsm8k-169",
+    "gsm8k-170",
+    "gsm8k-171",
+    "gsm8k-172",
+    "gsm8k-173",
+    "gsm8k-174",
+    "gsm8k-175",
+    "gsm8k-176",
+    "gsm8k-177",
+    "gsm8k-178",
+    "gsm8k-179",
+    "gsm8k-180",
+    "gsm8k-181",
+    "gsm8k-182",
+    "gsm8k-183",
+    "gsm8k-184",
+    "gsm8k-185",
+    "gsm8k-186",
+    "gsm8k-187",
+    "gsm8k-188",
+    "gsm8k-189",
+    "gsm8k-190",
+    "gsm8k-191",
+    "gsm8k-192",
+    "gsm8k-193",
+    "gsm8k-194",
+    "gsm8k-195",
+    "gsm8k-196",
+    "gsm8k-197",
+    "gsm8k-198",
+    "gsm8k-199",
+    "ifbench-0",
+    "ifbench-1",
+    "ifbench-2",
+    "ifbench-3",
+    "ifbench-4",
+    "ifbench-5",
+    "ifbench-6",
+    "ifbench-7",
+    "ifbench-8",
+    "ifbench-9",
+    "ifbench-10",
+    "ifbench-11",
+    "ifbench-12",
+    "ifbench-13",
+    "ifbench-14",
+    "ifbench-15",
+    "ifbench-16",
+    "ifbench-17",
+    "ifbench-18",
+    "ifbench-19",
+    "ifbench-20",
+    "ifbench-21",
+    "ifbench-22",
+    "ifbench-23",
+    "ifbench-24",
+    "ifbench-25",
+    "ifbench-26",
+    "ifbench-27",
+    "ifbench-28",
+    "ifbench-29",
+    "ifbench-30",
+    "ifbench-31",
+    "ifbench-32",
+    "ifbench-33",
+    "ifbench-34",
+    "ifbench-35",
+    "ifbench-36",
+    "ifbench-37",
+    "ifbench-38",
+    "ifbench-39",
+    "ifbench-40",
+    "ifbench-41",
+    "ifbench-42",
+    "ifbench-43",
+    "ifbench-44",
+    "ifbench-45",
+    "ifbench-46",
+    "ifbench-47",
+    "ifbench-48",
+    "ifbench-49",
+    "ifbench-50",
+    "ifbench-51",
+    "ifbench-52",
+    "ifbench-53",
+    "ifbench-54",
+    "ifbench-55",
+    "ifbench-56",
+    "ifbench-57",
+    "ifbench-58",
+    "ifbench-59",
+    "ifbench-60",
+    "ifbench-61",
+    "ifbench-62",
+    "ifbench-63",
+    "ifbench-64",
+    "ifbench-65",
+    "ifbench-66",
+    "ifbench-67",
+    "ifbench-68",
+    "ifbench-69",
+    "ifbench-70",
+    "ifbench-71",
+    "ifbench-72",
+    "ifbench-73",
+    "ifbench-74",
+    "ifbench-75",
+    "ifbench-76",
+    "ifbench-77",
+    "ifbench-78",
+    "ifbench-79",
+    "ifbench-80",
+    "ifbench-81",
+    "ifbench-82",
+    "ifbench-83",
+    "ifbench-84",
+    "ifbench-85",
+    "ifbench-86",
+    "ifbench-87",
+    "ifbench-88",
+    "ifbench-89",
+    "ifbench-90",
+    "ifbench-91",
+    "ifbench-92",
+    "ifbench-93",
+    "ifbench-94",
+    "ifbench-95",
+    "ifbench-96",
+    "ifbench-97",
+    "ifbench-98",
+    "ifbench-99",
+    "ifbench-100",
+    "ifbench-101",
+    "ifbench-102",
+    "ifbench-103",
+    "ifbench-104",
+    "ifbench-105",
+    "ifbench-106",
+    "ifbench-107",
+    "ifbench-108",
+    "ifbench-109",
+    "ifbench-110",
+    "ifbench-111",
+    "ifbench-112",
+    "ifbench-113",
+    "ifbench-114",
+    "ifbench-115",
+    "ifbench-116",
+    "ifbench-117",
+    "ifbench-118",
+    "ifbench-119",
+    "ifbench-120",
+    "ifbench-121",
+    "ifbench-122",
+    "ifbench-123",
+    "ifbench-124",
+    "ifbench-125",
+    "ifbench-126",
+    "ifbench-127",
+    "ifbench-128",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-131",
+    "ifbench-132",
+    "ifbench-133",
+    "ifbench-134",
+    "ifbench-135",
+    "ifbench-136",
+    "ifbench-137",
+    "ifbench-138",
+    "ifbench-139",
+    "ifbench-140",
+    "ifbench-141",
+    "ifbench-142",
+    "ifbench-143",
+    "ifbench-144",
+    "ifbench-145",
+    "ifbench-146",
+    "ifbench-147",
+    "ifbench-148",
+    "ifbench-149",
+    "ifbench-150",
+    "ifbench-151",
+    "ifbench-152",
+    "ifbench-153",
+    "ifbench-154",
+    "ifbench-155",
+    "ifbench-156",
+    "ifbench-157",
+    "ifbench-158",
+    "ifbench-159",
+    "ifbench-160",
+    "ifbench-161",
+    "ifbench-162",
+    "ifbench-163",
+    "ifbench-164",
+    "ifbench-165",
+    "ifbench-166",
+    "ifbench-167",
+    "ifbench-168",
+    "ifbench-169",
+    "ifbench-170",
+    "ifbench-171",
+    "ifbench-172",
+    "ifbench-173",
+    "ifbench-174",
+    "ifbench-175",
+    "ifbench-176",
+    "ifbench-177",
+    "ifbench-178",
+    "ifbench-179",
+    "ifbench-180",
+    "ifbench-181",
+    "ifbench-182",
+    "ifbench-183",
+    "ifbench-184",
+    "ifbench-185",
+    "ifbench-186",
+    "ifbench-187",
+    "ifbench-188",
+    "ifbench-189",
+    "ifbench-190",
+    "ifbench-191",
+    "ifbench-192",
+    "ifbench-193",
+    "ifbench-194",
+    "ifbench-195",
+    "ifbench-196",
+    "ifbench-197",
+    "ifbench-198",
+    "ifbench-199",
+    "ifbench-200",
+    "ifbench-201",
+    "ifbench-202",
+    "ifbench-203",
+    "ifbench-204",
+    "ifbench-205",
+    "ifbench-206",
+    "ifbench-207",
+    "ifbench-208",
+    "ifbench-209",
+    "ifbench-210",
+    "ifbench-211",
+    "ifbench-212",
+    "ifbench-213",
+    "ifbench-214",
+    "ifbench-215",
+    "ifbench-216",
+    "ifbench-217",
+    "ifbench-218",
+    "ifbench-219",
+    "ifbench-220",
+    "ifbench-221",
+    "ifbench-222",
+    "ifbench-223",
+    "ifbench-224",
+    "ifbench-225",
+    "ifbench-226",
+    "ifbench-227",
+    "ifbench-228",
+    "ifbench-229",
+    "ifbench-230",
+    "ifbench-231",
+    "ifbench-232",
+    "ifbench-233",
+    "ifbench-234",
+    "ifbench-235",
+    "ifbench-236",
+    "ifbench-237",
+    "ifbench-238",
+    "ifbench-239",
+    "ifbench-240",
+    "ifbench-241",
+    "ifbench-242",
+    "ifbench-243",
+    "ifbench-244",
+    "ifbench-245",
+    "ifbench-246",
+    "ifbench-247",
+    "ifbench-248",
+    "ifbench-249",
+    "ifbench-250",
+    "ifbench-251",
+    "ifbench-252",
+    "ifbench-253",
+    "ifbench-254",
+    "ifbench-255",
+    "ifbench-256",
+    "ifbench-257",
+    "ifbench-258",
+    "ifbench-259",
+    "ifbench-260",
+    "ifbench-261",
+    "ifbench-262",
+    "ifbench-263",
+    "ifbench-264",
+    "ifbench-265",
+    "ifbench-266",
+    "ifbench-267",
+    "ifbench-268",
+    "ifbench-269",
+    "ifbench-270",
+    "ifbench-271",
+    "ifbench-272",
+    "ifbench-273",
+    "ifbench-274",
+    "ifbench-275",
+    "ifbench-276",
+    "ifbench-277",
+    "ifbench-278",
+    "ifbench-279",
+    "ifbench-280",
+    "ifbench-281",
+    "ifbench-282",
+    "ifbench-283",
+    "ifbench-284",
+    "ifbench-285",
+    "ifbench-286",
+    "ifbench-287",
+    "ifbench-288",
+    "ifbench-289",
+    "ifbench-290",
+    "ifbench-291",
+    "ifbench-292",
+    "ifbench-293",
+    "ifbench-294",
+    "ifbench-295",
+    "ifbench-296",
+    "ifbench-297",
+    "ifbench-298",
+    "ifbench-299",
+    "lcb-atcoder-abc322_b",
+    "lcb-atcoder-abc381_a",
+    "lcb-atcoder-abc393_a",
+    "lcb-atcoder-abc321_b",
+    "lcb-atcoder-abc359_a",
+    "lcb-atcoder-abc329_b",
+    "lcb-atcoder-abc353_a",
+    "lcb-atcoder-abc355_b",
+    "lcb-atcoder-abc326_b",
+    "lcb-codeforces-1873_B",
+    "lcb-atcoder-abc356_a",
+    "lcb-atcoder-abc356_b",
+    "lcb-atcoder-abc375_a",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc311_b",
+    "lcb-atcoder-abc378_b",
+    "lcb-atcoder-abc309_b",
+    "lcb-atcoder-abc325_a",
+    "lcb-atcoder-abc301_b",
+    "lcb-atcoder-abc391_a",
+    "lcb-atcoder-abc399_b",
+    "lcb-atcoder-abc354_a",
+    "lcb-atcoder-abc352_b",
+    "lcb-atcoder-abc382_b",
+    "lcb-atcoder-abc332_b",
+    "lcb-atcoder-abc343_b",
+    "lcb-atcoder-abc361_a",
+    "lcb-atcoder-abc362_a",
+    "lcb-atcoder-abc328_a",
+    "lcb-atcoder-abc393_b",
+    "lcb-atcoder-abc352_a",
+    "lcb-atcoder-abc310_a",
+    "lcb-atcoder-abc365_a",
+    "lcb-atcoder-abc371_b",
+    "lcb-atcoder-abc367_c",
+    "lcb-atcoder-abc334_b",
+    "lcb-atcoder-abc385_c",
+    "lcb-atcoder-abc307_c",
+    "lcb-atcoder-abc338_c",
+    "lcb-atcoder-abc303_d",
+    "lcb-atcoder-abc342_c",
+    "lcb-atcoder-abc319_d",
+    "lcb-atcoder-abc315_d",
+    "lcb-atcoder-abc309_c",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc343_d",
+    "lcb-atcoder-abc397_b",
+    "lcb-atcoder-abc370_c",
+    "lcb-atcoder-abc375_c",
+    "lcb-atcoder-abc368_c",
+    "lcb-atcoder-abc325_b",
+    "lcb-atcoder-abc323_c",
+    "lcb-atcoder-abc377_c",
+    "lcb-atcoder-abc383_d",
+    "lcb-atcoder-arc189_a",
+    "lcb-codeforces-1883_C",
+    "lcb-atcoder-abc371_c",
+    "lcb-atcoder-abc380_c",
+    "lcb-atcoder-abc378_c",
+    "lcb-atcoder-abc366_c",
+    "lcb-atcoder-abc397_c",
+    "lcb-atcoder-abc339_c",
+    "lcb-atcoder-abc324_c",
+    "lcb-atcoder-abc355_c",
+    "lcb-atcoder-abc358_c",
+    "lcb-atcoder-abc321_d",
+    "lcb-atcoder-abc334_c",
+    "lcb-atcoder-arc195_c",
+    "lcb-atcoder-arc184_c",
+    "lcb-atcoder-abc377_e",
+    "lcb-atcoder-arc186_e",
+    "lcb-atcoder-abc398_f",
+    "lcb-atcoder-abc330_e",
+    "lcb-atcoder-abc396_e",
+    "lcb-atcoder-arc192_b",
+    "lcb-atcoder-abc391_f",
+    "lcb-atcoder-arc195_b",
+    "lcb-atcoder-abc384_g",
+    "lcb-atcoder-abc343_e",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc333_e",
+    "lcb-atcoder-abc341_e",
+    "lcb-atcoder-abc400_g",
+    "lcb-atcoder-abc362_d",
+    "lcb-codeforces-1899_D",
+    "lcb-atcoder-abc363_f",
+    "lcb-atcoder-abc382_d",
+    "lcb-atcoder-abc331_e",
+    "lcb-atcoder-abc351_e",
+    "lcb-atcoder-arc194_b",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-arc188_d",
+    "lcb-atcoder-abc368_e",
+    "lcb-atcoder-abc301_e",
+    "lcb-atcoder-arc194_e",
+    "lcb-atcoder-abc379_e",
+    "lcb-atcoder-abc360_e",
+    "lcb-atcoder-abc305_e",
+    "lcb-atcoder-arc186_a",
+    "lcb-atcoder-abc362_e",
+    "tool-0",
+    "tool-1",
+    "tool-2",
+    "tool-3",
+    "tool-4",
+    "tool-5",
+    "tool-6",
+    "tool-7",
+    "tool-8",
+    "tool-9",
+    "tool-10",
+    "tool-11",
+    "tool-12",
+    "tool-13",
+    "tool-14",
+    "tool-15",
+    "tool-16",
+    "tool-17",
+    "tool-18",
+    "tool-19",
+    "json-20",
+    "json-21",
+    "json-22",
+    "json-23",
+    "json-24",
+    "json-25",
+    "json-26",
+    "json-27",
+    "json-28",
+    "json-29"
+  ],
+  "concurrency": 2,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen",
+    "config_sha256": "7a6b1bbea65bf844cd3d824845dd54d57fe8c92d46249233e1347e90c9f41b3f",
+    "index_sha256": "04a57f3004384efa2cb0aa67400454a478d3d0bd94cbad0dd27f199ee2b2e7e4",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00003-of-00006.safetensors": {
+        "size": 3966050624,
+        "mtime_ns": 1790420289637223015
+      },
+      "model-00002-of-00006.safetensors": {
+        "size": 2730582048,
+        "mtime_ns": 1790420535527155687
+      },
+      "model-mtp-bf16.safetensors": {
+        "size": 431364472,
+        "mtime_ns": 1790420539382140680
+      },
+      "model-00005-of-00006.safetensors": {
+        "size": 3993185016,
+        "mtime_ns": 1790420447135512937
+      },
+      "model-00006-of-00006.safetensors": {
+        "size": 248265904,
+        "mtime_ns": 1790420420446626139
+      },
+      "model-00001-of-00006.safetensors": {
+        "size": 1291264336,
+        "mtime_ns": 1790420521965208848
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 212992352,
+        "mtime_ns": 1790420543620124231
+      },
+      "model-00004-of-00006.safetensors": {
+        "size": 3966050704,
+        "mtime_ns": 1790420327914040554
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/swift15-baseline-full/summary.json

@@ -0,0 +1,88 @@
+{
+  "created": "2026-09-27T20:38:58Z",
+  "concurrency": 2,
+  "resumed": false,
+  "new_run_wall_seconds": 22960.825675303,
+  "new_tasks": 630,
+  "suites": {
+    "gsm8k": {
+      "attempted": 200,
+      "correct": 196,
+      "accuracy": 0.98,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 355.47,
+      "mean_input_tokens": 93.27,
+      "mean_total_tokens": 448.74,
+      "mean_model_seconds": 3.2202836997405395,
+      "median_model_seconds": 2.9490127855096944,
+      "p95_model_seconds": 5.4270519990241155,
+      "summed_request_seconds_per_correct": 3.2860037752454483,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 300,
+      "correct": 221,
+      "accuracy": 0.7366666666666667,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 5598.606666666667,
+      "mean_input_tokens": 125.44,
+      "mean_total_tokens": 5724.046666666667,
+      "mean_model_seconds": 63.119914838518014,
+      "median_model_seconds": 28.452186586917378,
+      "p95_model_seconds": 249.57361784903333,
+      "summed_request_seconds_per_correct": 85.68314231473033,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 100,
+      "correct": 89,
+      "accuracy": 0.89,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 1,
+      "errors": 0,
+      "truncated": 1,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 18712.25,
+      "mean_input_tokens": 653.75,
+      "mean_total_tokens": 19366,
+      "mean_model_seconds": 258.7643161940086,
+      "median_model_seconds": 62.81849751947448,
+      "p95_model_seconds": 1266.3205917470623,
+      "summed_request_seconds_per_correct": 290.74642268989726,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 30,
+      "correct": 30,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 40,
+      "mean_input_tokens": 481.3333333333333,
+      "mean_total_tokens": 521.3333333333334,
+      "mean_model_seconds": 1.0695485257388404,
+      "median_model_seconds": 1.3348065734608099,
+      "p95_model_seconds": 1.4240245381370187,
+      "summed_request_seconds_per_correct": 1.0695485257388404,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [
+    "lcb-atcoder-arc184_c"
+  ],
+  "suite_wall_seconds": 22960.825675303,
+  "wall_seconds_per_correct": 42.83736133452052
+}

evaluation/swift15-baseline-pilot/manifest.json

@@ -0,0 +1,83 @@
+{
+  "tasks_sha256": "809c2c6b124579d41c379a6a436e9649b5e1d1ddf44a147a3d70d9b4b19d68f4",
+  "task_ids": [
+    "gsm8k-27",
+    "gsm8k-60",
+    "gsm8k-105",
+    "gsm8k-109",
+    "gsm8k-152",
+    "ifbench-21",
+    "ifbench-76",
+    "ifbench-129",
+    "ifbench-130",
+    "ifbench-268",
+    "lcb-atcoder-abc377_b",
+    "lcb-atcoder-abc390_d",
+    "lcb-atcoder-abc385_e",
+    "lcb-atcoder-abc325_f",
+    "lcb-atcoder-abc368_e",
+    "tool-15",
+    "tool-16",
+    "json-22",
+    "json-24",
+    "json-26"
+  ],
+  "concurrency": 1,
+  "sampling": "thinking: temperature1/top_p0.95/top_k20/xhigh; nonthinking: greedy; seed15027",
+  "api": "http://127.0.0.1:18021/v1",
+  "server": {
+    "model": "<WORKSPACE>/models/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen",
+    "config_sha256": "7a6b1bbea65bf844cd3d824845dd54d57fe8c92d46249233e1347e90c9f41b3f",
+    "index_sha256": "04a57f3004384efa2cb0aa67400454a478d3d0bd94cbad0dd27f199ee2b2e7e4",
+    "launcher_sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8",
+    "tokenizer_sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3",
+    "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041",
+    "shards": {
+      "model-00003-of-00006.safetensors": {
+        "size": 3966050624,
+        "mtime_ns": 1790420289637223015
+      },
+      "model-00002-of-00006.safetensors": {
+        "size": 2730582048,
+        "mtime_ns": 1790420535527155687
+      },
+      "model-mtp-bf16.safetensors": {
+        "size": 431364472,
+        "mtime_ns": 1790420539382140680
+      },
+      "model-00005-of-00006.safetensors": {
+        "size": 3993185016,
+        "mtime_ns": 1790420447135512937
+      },
+      "model-00006-of-00006.safetensors": {
+        "size": 248265904,
+        "mtime_ns": 1790420420446626139
+      },
+      "model-00001-of-00006.safetensors": {
+        "size": 1291264336,
+        "mtime_ns": 1790420521965208848
+      },
+      "model_extra_tensors.safetensors": {
+        "size": 212992352,
+        "mtime_ns": 1790420543620124231
+      },
+      "model-00004-of-00006.safetensors": {
+        "size": 3966050704,
+        "mtime_ns": 1790420327914040554
+      }
+    },
+    "mode": "single-user",
+    "int8_activations": false,
+    "speculation": "mtp",
+    "max_model_len": 150000,
+    "max_num_seqs": 8,
+    "kv_cache_dtype": "fp8",
+    "prefix_cache": true,
+    "gpu_memory_utilization": 0.93,
+    "serving_profile": "local-single-user",
+    "vision": true,
+    "draft_tokens": 3,
+    "local_env_sha256": "ce50b2e28ea929a620bd62aa7f94aacb844832f9e3dd5767b6952c2ffa0e74a8",
+    "extra_args": "--limit-mm-per-prompt {\"image\":{\"count\":10}}"
+  }
+}

evaluation/swift15-baseline-pilot/summary.json

@@ -0,0 +1,86 @@
+{
+  "created": "2026-09-27T14:16:13Z",
+  "concurrency": 1,
+  "resumed": false,
+  "new_run_wall_seconds": 3445.778244882007,
+  "new_tasks": 20,
+  "suites": {
+    "gsm8k": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 275,
+      "mean_input_tokens": 77.4,
+      "mean_total_tokens": 352.4,
+      "mean_model_seconds": 2.3928021708037703,
+      "median_model_seconds": 2.220176461036317,
+      "p95_model_seconds": 3.1514280069386587,
+      "summed_request_seconds_per_correct": 2.3928021708037703,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "ifbench": {
+      "attempted": 5,
+      "correct": 3,
+      "accuracy": 0.6,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 1952.6,
+      "mean_input_tokens": 134.6,
+      "mean_total_tokens": 2087.2,
+      "mean_model_seconds": 21.503820910397916,
+      "median_model_seconds": 10.035008723963983,
+      "p95_model_seconds": 68.41221156402025,
+      "summed_request_seconds_per_correct": 35.839701517329864,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "livecodebench": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 49329,
+      "mean_input_tokens": 742.4,
+      "mean_total_tokens": 50071.4,
+      "mean_model_seconds": 660.0320765013806,
+      "median_model_seconds": 480.73357990791555,
+      "p95_model_seconds": 1812.1029692189768,
+      "summed_request_seconds_per_correct": 660.0320765013806,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    },
+    "tools": {
+      "attempted": 5,
+      "correct": 5,
+      "accuracy": 1.0,
+      "truncation_policy": "count_as_wrong",
+      "truncated_counted_as_wrong": 0,
+      "errors": 0,
+      "truncated": 0,
+      "incomplete_token_counts": 0,
+      "mean_output_tokens": 33.6,
+      "mean_input_tokens": 330.6,
+      "mean_total_tokens": 364.2,
+      "mean_model_seconds": 0.6659889166243375,
+      "median_model_seconds": 0.5145661529386416,
+      "p95_model_seconds": 1.1192404899047688,
+      "summed_request_seconds_per_correct": 0.6659889166243375,
+      "note": "Summed request seconds are not GPU compute time when concurrency exceeds one."
+    }
+  },
+  "quality_comparison_ready": true,
+  "truncation_policy": "count_as_wrong",
+  "truncated_task_ids": [],
+  "suite_wall_seconds": 3445.778244882007,
+  "wall_seconds_per_correct": 191.43212471566707
+}