xnnpack 8k: HuggingFaceTB/SmolLM2-1.7B@effd688a1292

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 xnnpack/SmolLM2-1.7B-8da4w-4k.pte filter=lfs diff=lfs merge=lfs -text
 xnnpack/SmolLM2-1.7B-8da4w-2k.pte filter=lfs diff=lfs merge=lfs -text
+xnnpack/SmolLM2-1.7B-8da4w-8k.pte filter=lfs diff=lfs merge=lfs -text

README.md

@@ -25,6 +25,7 @@ Every backend is exported at every context window the runner could build (2k to
 |---|---|---|---|---|---|
 | XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-1.7B-8da4w-2k.pte`](xnnpack/SmolLM2-1.7B-8da4w-2k.pte) | 2,048 tokens | 1.07 GB | passed ("Paris") |
 | XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-1.7B-8da4w-4k.pte`](xnnpack/SmolLM2-1.7B-8da4w-4k.pte) | 4,096 tokens | 1.07 GB | passed ("Paris") |
+| XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-1.7B-8da4w-8k.pte`](xnnpack/SmolLM2-1.7B-8da4w-8k.pte) | 8,192 tokens | 1.07 GB | passed ("Paris") |
 
 Tokenizer: [`tokenizer.json`](tokenizer.json), copied unchanged from the source repo. Each backend folder has a `config.json` listing every window as a variant with the metadata the `.pte` reports, and an `export-report-<window>.json` per file with the full export record.
 
@@ -32,6 +33,7 @@ Tokenizer: [`tokenizer.json`](tokenizer.json), copied unchanged from the source
 
 - XNNPACK (CPU) at 2,048 tokens: the KV cache costs 393,216 bytes per token (fp32), 805,306,368 bytes for the whole window, allocated in full when the model loads.
 - XNNPACK (CPU) at 4,096 tokens: the KV cache costs 393,216 bytes per token (fp32), 1,610,612,736 bytes for the whole window, allocated in full when the model loads.
+- XNNPACK (CPU) at 8,192 tokens: the KV cache costs 393,216 bytes per token (fp32), 3,221,225,472 bytes for the whole window, allocated in full when the model loads.
 
 ## How it was made
 

xnnpack/SmolLM2-1.7B-8da4w-8k.pte

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:907e45a181d3bb77cb9c850ac6246563a02c9eed222d9da30a138c315453a8c8
+size 1068591872

xnnpack/config.json

@@ -50,6 +50,28 @@
         "use_sdpa_with_kv_cache": true,
         "enable_dynamic_shape": true
       }
+    },
+    {
+      "file": "SmolLM2-1.7B-8da4w-8k.pte",
+      "size_bytes": 1068591872,
+      "sha256": "907e45a181d3bb77cb9c850ac6246563a02c9eed222d9da30a138c315453a8c8",
+      "context": 8192,
+      "source_revision": "effd688a12921b4cc83e3312b6feb579f70f9c71",
+      "quantization": "8da4w-g32, int8 embeddings",
+      "fits_phone_budget": true,
+      "methods": {
+        "get_max_context_len": 8192,
+        "get_max_seq_len": 2048,
+        "get_bos_id": 0,
+        "get_eos_ids": [
+          0
+        ],
+        "get_vocab_size": 49152,
+        "get_n_layers": 24,
+        "use_kv_cache": true,
+        "use_sdpa_with_kv_cache": true,
+        "enable_dynamic_shape": true
+      }
     }
   ]
 }

xnnpack/export-report-8k.json

@@ -0,0 +1,183 @@
+{
+  "backend": "xnnpack",
+  "target": null,
+  "tokenizer": "tokenizer.json",
+  "output_repo": "experimentalmachines/SmolLM2-1.7B-ExecuTorch",
+  "source": {
+    "id": "HuggingFaceTB/SmolLM2-1.7B",
+    "sha": "effd688a12921b4cc83e3312b6feb579f70f9c71",
+    "created_at": "2024-10-30T22:50:10+00:00",
+    "total_params": 1711376384,
+    "family": "llama",
+    "variant": "base",
+    "license": {
+      "license": "apache-2.0"
+    },
+    "license_files": []
+  },
+  "toolchain": {
+    "executorch": "1.4.0",
+    "torch": "2.13.0+cpu",
+    "torchao": "0.18.0"
+  },
+  "recipe": {
+    "model_class": "smollm2",
+    "converter": "llama",
+    "params": {
+      "dim": 2048,
+      "ffn_dim_multiplier": 1,
+      "hidden_dim": 8192,
+      "n_heads": 32,
+      "head_dim": 64,
+      "n_kv_heads": 32,
+      "n_layers": 24,
+      "norm_eps": 1e-05,
+      "rope_theta": 130000.0,
+      "vocab_size": 49152,
+      "use_scaled_rope": false,
+      "use_hf_rope": false,
+      "attention_qkv_bias": false
+    },
+    "qmode": "8da4w",
+    "group_size": 32,
+    "embedding_quantize": "8,0",
+    "prefill_chunk": 2048,
+    "kv_cache_dtype": "fp32",
+    "label": "8da4w-g32, int8 embeddings",
+    "description": "ExecuTorch 1.4.0 `export_llm`: 8-bit dynamic activations and 4-bit weights in groups of 32, int8 per-channel embeddings, XNNPACK with extended ops, prefill chunk 2048, fp32 KV cache."
+  },
+  "window": {
+    "context": 8192,
+    "reason": "requested window 8192",
+    "kv_cache_bytes_per_token": 393216,
+    "device_resident_bytes": 4803586590,
+    "phone_budget_bytes": 5000000000,
+    "fits_phone_budget": true,
+    "table": [
+      {
+        "context": 32768,
+        "kv_cache_bytes": 12884901888,
+        "device_resident_bytes": 14492680488,
+        "export_peak_bytes": 48402462976,
+        "fits_device": false,
+        "fits_host": false
+      },
+      {
+        "context": 16384,
+        "kv_cache_bytes": 6442450944,
+        "device_resident_bytes": 8033284556,
+        "export_peak_bytes": 22632659200,
+        "fits_device": false,
+        "fits_host": true
+      },
+      {
+        "context": 8192,
+        "kv_cache_bytes": 3221225472,
+        "device_resident_bytes": 4803586590,
+        "export_peak_bytes": 14579595520,
+        "fits_device": true,
+        "fits_host": true
+      },
+      {
+        "context": 4096,
+        "kv_cache_bytes": 1610612736,
+        "device_resident_bytes": 3188737607,
+        "export_peak_bytes": 11761023232,
+        "fits_device": true,
+        "fits_host": true
+      },
+      {
+        "context": 2048,
+        "kv_cache_bytes": 805306368,
+        "device_resident_bytes": 2381313116,
+        "export_peak_bytes": 10653726976,
+        "fits_device": true,
+        "fits_host": true
+      }
+    ]
+  },
+  "files": [
+    {
+      "path": "xnnpack/SmolLM2-1.7B-8da4w-8k.pte",
+      "bytes": 1068591872,
+      "sha256": "907e45a181d3bb77cb9c850ac6246563a02c9eed222d9da30a138c315453a8c8"
+    }
+  ],
+  "estimates": {
+    "pte_bytes_estimate": 1082361118,
+    "pte_bytes_actual": 1068591872,
+    "export_peak_bytes_estimate": 14579595520
+  },
+  "host": {
+    "nproc": 4,
+    "mem_total_bytes": 16766414848,
+    "swap_total_bytes": 25769799680,
+    "cpu_model": "AMD EPYC 7763 64-Core Processor",
+    "peak_rss_convert_bytes": 4106387456,
+    "peak_rss_export_bytes": 15043227648,
+    "peak_swap_used_bytes": 292925440,
+    "min_mem_available_bytes": 707555328,
+    "export_seconds": 663.6,
+    "total_seconds": 697.9
+  },
+  "conversion": {
+    "tensors": 219,
+    "tied_embeddings": true,
+    "dtypes": [
+      "torch.bfloat16"
+    ]
+  },
+  "metadata": {
+    "enable_dynamic_shape": true,
+    "get_bos_id": 0,
+    "get_eos_ids": [
+      0
+    ],
+    "get_max_context_len": 8192,
+    "get_max_seq_len": 2048,
+    "get_n_layers": 24,
+    "get_vocab_size": 49152,
+    "use_kv_cache": true,
+    "use_sdpa_with_kv_cache": true,
+    "methods": [
+      "enable_dynamic_shape",
+      "forward",
+      "get_bos_id",
+      "get_eos_ids",
+      "get_max_context_len",
+      "get_max_seq_len",
+      "get_n_layers",
+      "get_vocab_size",
+      "use_kv_cache",
+      "use_sdpa_with_kv_cache"
+    ]
+  },
+  "smoke": {
+    "prompt": "The capital of France is",
+    "template_error": null,
+    "reply": " Paris.\n\nThe capital of Italy is Rome.\n\nThe capital of Spain is Madrid.\n\nThe capital of Germany is Berlin.\n\nThe",
+    "tokens": 32,
+    "answered": true,
+    "answer_required": true,
+    "passed": true,
+    "problems": [],
+    "stats": {
+      "prefill_token_per_sec": 22.9358,
+      "decode_token_per_sec": 19.7704,
+      "prompt_tokens": 5,
+      "generated_tokens": 31,
+      "model_load_start_ms": 1789290077503,
+      "model_load_end_ms": 1789290081220,
+      "inference_start_ms": 1789290081220,
+      "inference_end_ms": 1789290083006,
+      "prompt_eval_end_ms": 1789290081438,
+      "first_token_ms": 1789290081438,
+      "aggregate_sampling_time_ms": 2,
+      "SCALING_FACTOR_UNITS_PER_SECOND": 1000
+    }
+  },
+  "run": {
+    "url": "https://github.com/ExperimentalMachines/executorch-model-exporter/actions/runs/34748476338",
+    "id": "34748476338"
+  }
+}