Record verified clean-download P150 serving benchmark

benchmarks/clean-download-speed.json

@@ -0,0 +1,254 @@
+{
+  "protocol": {
+    "temperature": 0,
+    "concurrency": 1,
+    "warmups_per_workload": 1,
+    "requested_completion_tokens": 512,
+    "repeats": 5,
+    "decode_definition": "(completion_tokens - 1) / (last_text_time - first_text_time)"
+  },
+  "rows": [
+    {
+      "workload": "short",
+      "repeat": 0,
+      "completion_tokens": 512,
+      "prompt_tokens": 5,
+      "ttft_ms": 185.55569398449734,
+      "decode_tok_s": 71.49168959645837,
+      "elapsed_s": 7.333292900992092,
+      "completion_sha256": "6636f7ecf5afa486ea804a08f19caee42843c851e99cd4263530c88b69a16dce",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 296.0,
+        "vllm:spec_decode_num_draft_tokens_total": 296.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 215.0
+      }
+    },
+    {
+      "workload": "short",
+      "repeat": 1,
+      "completion_tokens": 512,
+      "prompt_tokens": 5,
+      "ttft_ms": 185.1059080217965,
+      "decode_tok_s": 71.43477092634538,
+      "elapsed_s": 7.338534704002086,
+      "completion_sha256": "6636f7ecf5afa486ea804a08f19caee42843c851e99cd4263530c88b69a16dce",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 296.0,
+        "vllm:spec_decode_num_draft_tokens_total": 296.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 215.0
+      }
+    },
+    {
+      "workload": "short",
+      "repeat": 2,
+      "completion_tokens": 512,
+      "prompt_tokens": 5,
+      "ttft_ms": 188.72601201292127,
+      "decode_tok_s": 71.16728538612892,
+      "elapsed_s": 7.3690437659970485,
+      "completion_sha256": "6636f7ecf5afa486ea804a08f19caee42843c851e99cd4263530c88b69a16dce",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 296.0,
+        "vllm:spec_decode_num_draft_tokens_total": 296.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 215.0
+      }
+    },
+    {
+      "workload": "short",
+      "repeat": 3,
+      "completion_tokens": 512,
+      "prompt_tokens": 5,
+      "ttft_ms": 196.98050303850323,
+      "decode_tok_s": 71.20606162352988,
+      "elapsed_s": 7.373373275040649,
+      "completion_sha256": "6636f7ecf5afa486ea804a08f19caee42843c851e99cd4263530c88b69a16dce",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 296.0,
+        "vllm:spec_decode_num_draft_tokens_total": 296.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 215.0
+      }
+    },
+    {
+      "workload": "short",
+      "repeat": 4,
+      "completion_tokens": 512,
+      "prompt_tokens": 5,
+      "ttft_ms": 196.64987601572648,
+      "decode_tok_s": 71.62100207396676,
+      "elapsed_s": 7.331470295030158,
+      "completion_sha256": "6636f7ecf5afa486ea804a08f19caee42843c851e99cd4263530c88b69a16dce",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 296.0,
+        "vllm:spec_decode_num_draft_tokens_total": 296.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 215.0
+      }
+    },
+    {
+      "workload": "code",
+      "repeat": 0,
+      "completion_tokens": 512,
+      "prompt_tokens": 21,
+      "ttft_ms": 178.8653020048514,
+      "decode_tok_s": 71.31224207255637,
+      "elapsed_s": 7.344575221010018,
+      "completion_sha256": "620e9e5773cb2fc9c01763904af16bdc48c50d9f5fd95f9a44642ff413310a02",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 299.0,
+        "vllm:spec_decode_num_draft_tokens_total": 299.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 213.0
+      }
+    },
+    {
+      "workload": "code",
+      "repeat": 1,
+      "completion_tokens": 512,
+      "prompt_tokens": 21,
+      "ttft_ms": 174.2983150179498,
+      "decode_tok_s": 71.24944268075356,
+      "elapsed_s": 7.346338619012386,
+      "completion_sha256": "620e9e5773cb2fc9c01763904af16bdc48c50d9f5fd95f9a44642ff413310a02",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 299.0,
+        "vllm:spec_decode_num_draft_tokens_total": 299.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 213.0
+      }
+    },
+    {
+      "workload": "code",
+      "repeat": 2,
+      "completion_tokens": 512,
+      "prompt_tokens": 21,
+      "ttft_ms": 178.43145597726107,
+      "decode_tok_s": 71.21706916652498,
+      "elapsed_s": 7.353721235995181,
+      "completion_sha256": "620e9e5773cb2fc9c01763904af16bdc48c50d9f5fd95f9a44642ff413310a02",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 299.0,
+        "vllm:spec_decode_num_draft_tokens_total": 299.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 213.0
+      }
+    },
+    {
+      "workload": "code",
+      "repeat": 3,
+      "completion_tokens": 512,
+      "prompt_tokens": 21,
+      "ttft_ms": 178.5057139932178,
+      "decode_tok_s": 71.2562456686059,
+      "elapsed_s": 7.349860362999607,
+      "completion_sha256": "620e9e5773cb2fc9c01763904af16bdc48c50d9f5fd95f9a44642ff413310a02",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 299.0,
+        "vllm:spec_decode_num_draft_tokens_total": 299.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 213.0
+      }
+    },
+    {
+      "workload": "code",
+      "repeat": 4,
+      "completion_tokens": 512,
+      "prompt_tokens": 21,
+      "ttft_ms": 176.40958999982104,
+      "decode_tok_s": 71.18627502059648,
+      "elapsed_s": 7.354793168022297,
+      "completion_sha256": "620e9e5773cb2fc9c01763904af16bdc48c50d9f5fd95f9a44642ff413310a02",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 299.0,
+        "vllm:spec_decode_num_draft_tokens_total": 299.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 213.0
+      }
+    },
+    {
+      "workload": "long",
+      "repeat": 0,
+      "completion_tokens": 512,
+      "prompt_tokens": 481,
+      "ttft_ms": 913.9241329976358,
+      "decode_tok_s": 69.96217508448485,
+      "elapsed_s": 8.21790869400138,
+      "completion_sha256": "7863f886daaad996ada3ff09672ca0be23738fb37307e10cb36e6c0229041bb8",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 304.0,
+        "vllm:spec_decode_num_draft_tokens_total": 304.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 208.0
+      }
+    },
+    {
+      "workload": "long",
+      "repeat": 1,
+      "completion_tokens": 512,
+      "prompt_tokens": 481,
+      "ttft_ms": 910.2938679861836,
+      "decode_tok_s": 71.15102313260371,
+      "elapsed_s": 8.092238911020104,
+      "completion_sha256": "8ecebaad413a6c696bababe995df95c269417a0b0a66846144e2f8008d1fcfc2",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 298.0,
+        "vllm:spec_decode_num_draft_tokens_total": 298.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 214.0
+      }
+    },
+    {
+      "workload": "long",
+      "repeat": 2,
+      "completion_tokens": 512,
+      "prompt_tokens": 481,
+      "ttft_ms": 910.7411350123584,
+      "decode_tok_s": 71.01843048242138,
+      "elapsed_s": 8.10611480200896,
+      "completion_sha256": "8ecebaad413a6c696bababe995df95c269417a0b0a66846144e2f8008d1fcfc2",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 298.0,
+        "vllm:spec_decode_num_draft_tokens_total": 298.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 214.0
+      }
+    },
+    {
+      "workload": "long",
+      "repeat": 3,
+      "completion_tokens": 512,
+      "prompt_tokens": 481,
+      "ttft_ms": 910.7204360188916,
+      "decode_tok_s": 70.57319635124608,
+      "elapsed_s": 8.151486968039535,
+      "completion_sha256": "8ecebaad413a6c696bababe995df95c269417a0b0a66846144e2f8008d1fcfc2",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 298.0,
+        "vllm:spec_decode_num_draft_tokens_total": 298.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 214.0
+      }
+    },
+    {
+      "workload": "long",
+      "repeat": 4,
+      "completion_tokens": 512,
+      "prompt_tokens": 481,
+      "ttft_ms": 910.0917300092988,
+      "decode_tok_s": 67.91384052761907,
+      "elapsed_s": 8.434374456992373,
+      "completion_sha256": "e8402b6410df4a7c02ee08d3990c89268712eec1ea646ffbccfff03ec94b70e9",
+      "speculation_metrics": {
+        "vllm:spec_decode_num_drafts_total": 312.0,
+        "vllm:spec_decode_num_draft_tokens_total": 312.0,
+        "vllm:spec_decode_num_accepted_tokens_total": 200.0
+      }
+    }
+  ],
+  "summary": {
+    "short": {
+      "median_decode_tok_s": 71.43477092634538,
+      "median_ttft_ms": 188.72601201292127,
+      "distinct_completions": 1
+    },
+    "code": {
+      "median_decode_tok_s": 71.24944268075356,
+      "median_ttft_ms": 178.43145597726107,
+      "distinct_completions": 1
+    },
+    "long": {
+      "median_decode_tok_s": 70.57319635124608,
+      "median_ttft_ms": 910.7204360188916,
+      "distinct_completions": 3
+    }
+  }
+}

benchmarks/verification.json

@@ -0,0 +1,31 @@
+{
+  "tested_repo": "Lottolabs/Qwen3.5-9B-TT-BFP4-P150",
+  "tested_revision": "57a271fb1073166cb8746ba684c3aba5fc81ca3c",
+  "public_download": true,
+  "fresh_quant_download": true,
+  "downloaded_package_files_byte_identical": 527,
+  "quantized_tensor_files": 460,
+  "quantized_tensor_bytes": 9699607184,
+  "post_benchmark_quantized_files_unchanged": 460,
+  "old_tensor_cache_volume_mounted": false,
+  "upstream_dependency": "Pinned original checkpoint copied into isolated test directory and verified against Hub file hashes; not redownloaded in full. No old TT caches copied.",
+  "upstream_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
+  "runtime_rebuilt_from_downloaded_source": true,
+  "runtime_image_id": "sha256:3c76d5443033f2245a3fc2edebdeb6c520ed6f392f3a081991fc9e1a27e1b95d",
+  "health_status": 200,
+  "paired_preupload_completion_matches": 15,
+  "paired_comparisons": 15,
+  "output_repeatability": {
+    "short": 1,
+    "code": 1,
+    "long": 3
+  },
+  "hardware": {
+    "device": "Tenstorrent Blackhole P150",
+    "device_count": 1,
+    "host_architecture": "linux/amd64",
+    "host_driver_tested": "2.11.0",
+    "firmware_bundle_tested": "19.13.2",
+    "installer_python": ">=3.11"
+  }
+}

manifest.json

@@ -128,9 +128,11 @@
     "all_target_layers": 32,
     "mtp_head_loaded": true,
     "generated_cache_files": 0,
-    "clean_download_benchmark": "pending",
+    "clean_download_benchmark": "passed; benchmarks/clean-download-speed.json",
     "runtime_build": "passed: rebuilt ttnn from pinned public base plus source overlay",
-    "runtime_image_id": "sha256:2d9dd4ef50d4916cf876ebf69a5c665fde56fcb2d02a9c0b21858dedd0528cdd"
+    "runtime_image_id": "sha256:2d9dd4ef50d4916cf876ebf69a5c665fde56fcb2d02a9c0b21858dedd0528cdd",
+    "verification": "benchmarks/verification.json",
+    "tested_revision": "57a271fb1073166cb8746ba684c3aba5fc81ca3c"
   },
   "quantized_files": [
     {