Publish complete real sliding-window ONNX package

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.onnx.data filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,21 @@
+---
+license: apache-2.0
+tags: [onnx, onnxruntime, onnx-genai, inference-metadata, sliding-window-attention]
+---
+# onnx-genai-example-mistral-7b-v0-1-sliding-window
+
+Private real-weight fp16 ONNX package from [`mistralai/Mistral-7B-v0.1`](https://huggingface.co/mistralai/Mistral-7B-v0.1/tree/27d67f1b5f57dc0953326b2601d68371d40ea8da) at immutable revision `27d67f1b5f57dc0953326b2601d68371d40ea8da`. Source license: Apache-2.0.
+
+The graph contains 32 CUDA `com.microsoft::GroupQueryAttention` nodes with `local_window_size=4096`. A real H200 CUDA probe generated 12 finite-logit tokens from positions 4092–4103, crossing the configured boundary. The package includes canonical metadata, tokenizer, policies, provenance, request/output, graph report, and exact timings.
+
+## Download
+```bash
+hf download justinchuby/onnx-genai-example-mistral-7b-v0-1-sliding-window --repo-type model --local-dir ./mistral-7b-v0.1-sliding-window
+```
+## Exact runtime probe
+```bash
+cd mistral-7b-v0.1-sliding-window
+python3 evidence/probe_sliding_window.py
+cat evidence/sliding_window_runtime_probe.json
+```
+Requires CUDA-capable `onnxruntime-gpu`, `onnx`, `transformers`, and `numpy`; exact successful versions are in `output.json`.

SOURCE_LICENSE.md

@@ -0,0 +1,3 @@
+# Source license
+
+[mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1/tree/27d67f1b5f57dc0953326b2601d68371d40ea8da) is Apache-2.0.

evidence/probe_sliding_window.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+import onnx
+import onnxruntime as ort
+import transformers
+from transformers import AutoTokenizer
+
+PACKAGE = Path(__file__).resolve().parents[1]
+WINDOW = 4096
+PROMPT_LENGTH = 4092
+MAX_NEW_TOKENS = 12
+
+
+def main() -> None:
+    evidence = PACKAGE / "evidence"
+    evidence.mkdir(exist_ok=True)
+    tokenizer = AutoTokenizer.from_pretrained(PACKAGE)
+    repeat_id = tokenizer.encode(" window", add_special_tokens=False)[0]
+    input_ids = np.array(
+        [[tokenizer.bos_token_id] + [repeat_id] * (PROMPT_LENGTH - 1)],
+        dtype=np.int64,
+    )
+
+    graph = onnx.load(PACKAGE / "model.onnx", load_external_data=False)
+    local_windows = sorted(
+        {
+            attribute.i
+            for node in graph.graph.node
+            for attribute in node.attribute
+            if attribute.name == "local_window_size"
+        }
+    )
+    started = time.perf_counter()
+    session_started = time.perf_counter()
+    session = ort.InferenceSession(
+        str(PACKAGE / "model.onnx"),
+        providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
+    )
+    load_seconds = time.perf_counter() - session_started
+    feeds: dict[str, np.ndarray] = {
+        "input_ids": input_ids,
+        "attention_mask": np.ones((1, PROMPT_LENGTH), dtype=np.int64),
+    }
+    for value in session.get_inputs():
+        if value.name.startswith("past_key_values."):
+            feeds[value.name] = np.empty(
+                (1, int(value.shape[1]), 0, int(value.shape[3])),
+                dtype=np.float16,
+            )
+
+    output_names = [value.name for value in session.get_outputs()]
+    generated = []
+    step_seconds = []
+    finite = True
+    for step in range(MAX_NEW_TOKENS):
+        step_started = time.perf_counter()
+        outputs = session.run(None, feeds)
+        step_seconds.append(time.perf_counter() - step_started)
+        logits = outputs[0][:, -1, :].astype(np.float32)
+        finite = finite and bool(np.isfinite(logits).all())
+        token = int(np.argmax(logits[0]))
+        generated.append(token)
+        presents = dict(zip(output_names[1:], outputs[1:]))
+        feeds = {
+            "input_ids": np.array([[token]], dtype=np.int64),
+            "attention_mask": np.ones((1, PROMPT_LENGTH + step + 1), dtype=np.int64),
+        }
+        for value in session.get_inputs():
+            if value.name.startswith("past_key_values."):
+                suffix = value.name.removeprefix("past_key_values.")
+                feeds[value.name] = presents[f"present.{suffix}"]
+
+    payload = {
+        "request": {
+            "prompt_construction": {
+                "bos_token_id": tokenizer.bos_token_id,
+                "repeated_token_id": repeat_id,
+                "repeated_token_text": tokenizer.decode([repeat_id]),
+                "prompt_length": PROMPT_LENGTH,
+            },
+            "max_new_tokens": MAX_NEW_TOKENS,
+        },
+        "output": {
+            "token_ids": generated,
+            "text": tokenizer.decode(generated),
+            "start_position": PROMPT_LENGTH,
+            "end_position_exclusive": PROMPT_LENGTH + MAX_NEW_TOKENS,
+        },
+        "graph": {
+            "group_query_attention_nodes": sum(
+                node.op_type == "GroupQueryAttention" for node in graph.graph.node
+            ),
+            "local_window_sizes": local_windows,
+        },
+        "assertions": {
+            "configured_window_is_4096": local_windows == [WINDOW],
+            "generation_crossed_window_boundary": (
+                PROMPT_LENGTH < WINDOW < PROMPT_LENGTH + MAX_NEW_TOKENS
+            ),
+            "finite_logits": finite,
+            "generated_multiple_tokens": len(generated) == MAX_NEW_TOKENS,
+        },
+        "timings": {
+            "session_load_seconds": load_seconds,
+            "prefill_seconds": step_seconds[0],
+            "decode_step_seconds": step_seconds[1:],
+            "total_seconds": time.perf_counter() - started,
+        },
+        "versions": {
+            "onnxruntime": ort.__version__,
+            "onnx": onnx.__version__,
+            "transformers": transformers.__version__,
+        },
+        "providers": session.get_providers(),
+    }
+    (evidence / "sliding_window_runtime_probe.json").write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n"
+    )
+    if not all(payload["assertions"].values()):
+        raise RuntimeError(payload["assertions"])
+
+
+if __name__ == "__main__":
+    main()

evidence/sliding_window_runtime_probe.json

@@ -0,0 +1,69 @@
+{
+  "assertions": {
+    "configured_window_is_4096": true,
+    "finite_logits": true,
+    "generated_multiple_tokens": true,
+    "generation_crossed_window_boundary": true
+  },
+  "graph": {
+    "group_query_attention_nodes": 32,
+    "local_window_sizes": [
+      4096
+    ]
+  },
+  "output": {
+    "end_position_exclusive": 4104,
+    "start_position": 4092,
+    "text": "window window window window window window window window window window window window",
+    "token_ids": [
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924
+    ]
+  },
+  "providers": [
+    "CUDAExecutionProvider",
+    "CPUExecutionProvider"
+  ],
+  "request": {
+    "max_new_tokens": 12,
+    "prompt_construction": {
+      "bos_token_id": 1,
+      "prompt_length": 4092,
+      "repeated_token_id": 2924,
+      "repeated_token_text": "window"
+    }
+  },
+  "timings": {
+    "decode_step_seconds": [
+      0.46859942795708776,
+      0.09483771701343358,
+      0.09751369617879391,
+      0.09462281805463135,
+      0.09493547398597002,
+      0.09594826796092093,
+      0.09737299382686615,
+      0.10137195093557239,
+      0.10905329487286508,
+      0.10262800892814994,
+      0.10246599186211824
+    ],
+    "prefill_seconds": 0.7347528520040214,
+    "session_load_seconds": 10.895026066107675,
+    "total_seconds": 13.0937190069817
+  },
+  "versions": {
+    "onnx": "1.22.0",
+    "onnxruntime": "1.28.0",
+    "transformers": "5.16.0.dev0"
+  }
+}

graph_report.json

@@ -0,0 +1,6 @@
+{
+  "group_query_attention_nodes": 32,
+  "local_window_sizes": [
+    4096
+  ]
+}

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35542d5d7d5de687933865febf6fcc65d4cebe72613351095c318dcfd2665987
+size 250699

model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876319f1f1301dfd0580c1aded6a86a7dce6800a79aa6e507eb264a19298e112
+size 14491910144

output.json

@@ -0,0 +1,36 @@
+{
+  "assertions": {
+    "configured_window_is_4096": true,
+    "finite_logits": true,
+    "generated_multiple_tokens": true,
+    "generation_crossed_window_boundary": true
+  },
+  "output": {
+    "end_position_exclusive": 4104,
+    "start_position": 4092,
+    "text": "window window window window window window window window window window window window",
+    "token_ids": [
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924,
+      2924
+    ]
+  },
+  "providers": [
+    "CUDAExecutionProvider",
+    "CPUExecutionProvider"
+  ],
+  "versions": {
+    "onnx": "1.22.0",
+    "onnxruntime": "1.28.0",
+    "transformers": "5.16.0.dev0"
+  }
+}

performance.json

@@ -0,0 +1,19 @@
+{
+  "decode_step_seconds": [
+    0.46859942795708776,
+    0.09483771701343358,
+    0.09751369617879391,
+    0.09462281805463135,
+    0.09493547398597002,
+    0.09594826796092093,
+    0.09737299382686615,
+    0.10137195093557239,
+    0.10905329487286508,
+    0.10262800892814994,
+    0.10246599186211824
+  ],
+  "hardware": "NVIDIA H200",
+  "prefill_seconds": 0.7347528520040214,
+  "session_load_seconds": 10.895026066107675,
+  "total_seconds": 13.0937190069817
+}

policies/cache_length_update.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430f8578b4fd9c08f2870186a4142d093e821a8d75cfa5fb65b2c021c85f669f
+size 930

policies/decoder_state_initializer.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea0efdda1de5ac393a4cf1e2ca634301561117ad414d511cb222273d9234822d
+size 84889

policies/decoder_step_update.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94edf6a26c7a89da1f1c1d256c1a7fc6bfe0a5da02e9bef75a13beb26d551f2d
+size 2430

policies/generated_length_update.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430f8578b4fd9c08f2870186a4142d093e821a8d75cfa5fb65b2c021c85f669f
+size 930

policies/last_token_logits.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7363179c34e44fd85267dd45f660f9dd0ee631da81916dfb6347a7ac8bc7d47
+size 820

policies/termination.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36f7e946899cdae9fab4b5f589b6c8f2c82c0fd4ced07081f79c4c52afb0e79e
+size 5880

policies/termination_batch_initializer.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c6e2757a6ecee52036a500dda21eb45094d1c8ad387db07149b0c53664765cf
+size 1677

policies/token_sampler.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c77038c17e1a94bebe397d5eda4f84647f25ddff2141a4261fc33d007a20c0
+size 58720

policies/token_state_update.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8775ecb9d843287b9fff9a88e3d8cd94767b44371587211ad214eac85ad07e27
+size 1229

policies/token_to_slot.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47956a6e4658afe8ad44e1574813b0ebc4e430fa6d874c51319fe39c3adfb425
+size 438

provenance.json

@@ -0,0 +1,26 @@
+{
+  "build": {
+    "command": "mobius build --model mistralai/Mistral-7B-v0.1 --revision 27d67f1b5f57dc0953326b2601d68371d40ea8da --dtype f16 --ep cuda --runtime onnx-genai OUTPUT",
+    "mobius_git_sha": "57c149d1845c97781ae386da5c1c93528fde8841"
+  },
+  "files": [
+    {
+      "bytes": 250699,
+      "path": "model.onnx",
+      "sha256": "35542d5d7d5de687933865febf6fcc65d4cebe72613351095c318dcfd2665987"
+    },
+    {
+      "bytes": 14491910144,
+      "path": "model.onnx.data",
+      "sha256": "876319f1f1301dfd0580c1aded6a86a7dce6800a79aa6e507eb264a19298e112"
+    }
+  ],
+  "sources": [
+    {
+      "model": "mistralai/Mistral-7B-v0.1",
+      "revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da",
+      "source_license": "apache-2.0",
+      "source_url": "https://huggingface.co/mistralai/Mistral-7B-v0.1/tree/27d67f1b5f57dc0953326b2601d68371d40ea8da"
+    }
+  ]
+}

request.json

@@ -0,0 +1,9 @@
+{
+  "max_new_tokens": 12,
+  "prompt_construction": {
+    "bos_token_id": 1,
+    "prompt_length": 4092,
+    "repeated_token_id": 2924,
+    "repeated_token_text": "window"
+  }
+}

source.json

@@ -0,0 +1,6 @@
+{
+  "model": "mistralai/Mistral-7B-v0.1",
+  "revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da",
+  "source_license": "apache-2.0",
+  "source_url": "https://huggingface.co/mistralai/Mistral-7B-v0.1/tree/27d67f1b5f57dc0953326b2601d68371d40ea8da"
+}

source_provenance.json

@@ -0,0 +1,14 @@
+{
+  "build": {
+    "command": "mobius build --model mistralai/Mistral-7B-v0.1 --revision 27d67f1b5f57dc0953326b2601d68371d40ea8da --dtype f16 --ep cuda --runtime onnx-genai OUTPUT",
+    "mobius_git_sha": "57c149d1845c97781ae386da5c1c93528fde8841"
+  },
+  "model": {
+    "id": "mistralai/Mistral-7B-v0.1",
+    "license": "Apache-2.0",
+    "license_evidence": "Pinned README front matter",
+    "revision": "27d67f1b5f57dc0953326b2601d68371d40ea8da",
+    "sliding_window": 4096,
+    "source_url": "https://huggingface.co/mistralai/Mistral-7B-v0.1/tree/27d67f1b5f57dc0953326b2601d68371d40ea8da"
+  }
+}

sources/model/README.md

@@ -0,0 +1,48 @@
+---
+library_name: transformers
+language:
+- en
+license: apache-2.0
+tags:
+- pretrained
+- mistral-common
+inference: false
+extra_gated_description: >-
+  If you want to learn more about how we process your personal data, please read
+  our <a href="https://mistral.ai/terms/">Privacy Policy</a>.
+---
+
+# Model Card for Mistral-7B-v0.1
+
+The Mistral-7B-v0.1 Large Language Model (LLM) is a pretrained generative text model with 7 billion parameters. 
+Mistral-7B-v0.1 outperforms Llama 2 13B on all benchmarks we tested.
+
+For full details of this model please read our [paper](https://arxiv.org/abs/2310.06825) and [release blog post](https://mistral.ai/news/announcing-mistral-7b/).
+
+## Model Architecture
+
+Mistral-7B-v0.1 is a transformer model, with the following architecture choices:
+- Grouped-Query Attention
+- Sliding-Window Attention
+- Byte-fallback BPE tokenizer
+
+## Troubleshooting
+
+- If you see the following error:
+```
+KeyError: 'mistral'
+```
+- Or:
+```
+NotImplementedError: Cannot copy out of meta tensor; no data!
+```
+
+Ensure you are utilizing a stable version of Transformers, 4.34.0 or newer.
+
+## Notice
+
+Mistral 7B is a pretrained base model and therefore does not have any moderation mechanisms.
+
+## The Mistral AI Team
+ 
+Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed.

sources/model/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.34.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}