benchmarks/rtx3090-128k-validation.json

{
  "status": "passed",
  "reference_gpu": "RTX 3090 24 GB",
  "runtime": "vLLM main 7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6",
  "max_model_len": 131072,
  "near_limit_input_tokens": 130816,
  "generated_tokens": 64,
  "max_num_seqs": 17,
  "max_num_batched_tokens": 8192,
  "kv_cache_memory_bytes": 6444425401,
  "cache_capacity_tokens": 135083,
  "cache_margin_tokens": 4011,
  "cuda_graphs": true,
  "quantization": "compressed-tensors W4A16 G64",
  "limit_mm_per_prompt": {
    "image": 1,
    "video": 0
  },
  "oom": false,
  "allocator_warning": false,
  "wall_seconds": 92.90461822296493
}