{
  "generated_utc": "2026-09-04T23:47:51.958393+00:00",
  "profile": {
    "model": "Q36-v1.3-Q4_K_M.gguf",
    "backend": "llama.cpp",
    "llama_cpp_commit": "427291b5b34cd914a31b3fd3b61a68f6184f4b9f",
    "manifest_sha256": "9490bd427291856b11b8a77efb20f69a0fa24d38831c2b2d8fb9735584754bfd",
    "thinking": false,
    "temperature": 0,
    "seed": 42,
    "max_new_tokens": 4096,
    "context": 16384,
    "wall_budget_minutes": 20.0,
    "per_prompt_seconds": 45,
    "note": "User-requested llama.cpp Q4 evaluation; not directly comparable to the earlier thinking-enabled BF16 run. Selected adapters still require official evaluators.",
    "gpu_inventory": "NVIDIA H200, 570.124.06, 143771 MiB",
    "llama_binary_version": "",
    "cpu_threads": 8
  },
  "available_manifest_prompts": 204,
  "attempted": 204,
  "completed": 186,
  "incomplete_or_error": 18,
  "request_errors": 0,
  "strict_auto_scored": 108,
  "strict_auto_correct": 61,
  "review_or_official_evaluator_pending": 78,
  "elapsed_seconds": 474.9204415604472,
  "by_benchmark": {
    "MMLU-Pro": {
      "attempted": 1,
      "completed": 1,
      "strict_scored": 1,
      "strict_correct": 0,
      "review_pending": 0
    },
    "BBH": {
      "attempted": 23,
      "completed": 20,
      "strict_scored": 20,
      "strict_correct": 3,
      "review_pending": 0
    },
    "ARC-Challenge": {
      "attempted": 10,
      "completed": 8,
      "strict_scored": 8,
      "strict_correct": 8,
      "review_pending": 0
    },
    "GSM8K": {
      "attempted": 10,
      "completed": 9,
      "strict_scored": 9,
      "strict_correct": 5,
      "review_pending": 0
    },
    "MATH-Level-5": {
      "attempted": 10,
      "completed": 8,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 8
    },
    "GPQA-Diamond": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 2,
      "review_pending": 0
    },
    "TruthfulQA": {
      "attempted": 10,
      "completed": 9,
      "strict_scored": 9,
      "strict_correct": 6,
      "review_pending": 0
    },
    "HumanEval-Plus": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 5
    },
    "MBPP-Plus": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 5
    },
    "LiveCodeBench": {
      "attempted": 9,
      "completed": 9,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 9
    },
    "Q36-JSON-Schema": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 0,
      "review_pending": 0
    },
    "Q36-Hermes-Tool-Format": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 5,
      "review_pending": 0
    },
    "Q36-Agent-Function-Calling": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 0,
      "review_pending": 0
    },
    "MMMU": {
      "attempted": 6,
      "completed": 6,
      "strict_scored": 6,
      "strict_correct": 4,
      "review_pending": 0
    },
    "MathVista": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 2,
      "review_pending": 0
    },
    "ChartQA": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 4,
      "review_pending": 0
    },
    "Q36-Benign-Compliance-No-Overrefusal": {
      "attempted": 20,
      "completed": 18,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 18
    },
    "Q36-Contradiction-and-Anti-Sycophancy": {
      "attempted": 10,
      "completed": 8,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 8
    },
    "Q36-Answer-Termination-No-Looping": {
      "attempted": 10,
      "completed": 10,
      "strict_scored": 10,
      "strict_correct": 7,
      "review_pending": 0
    },
    "Q36-CAPTCHA-Detection-and-Handoff": {
      "attempted": 10,
      "completed": 10,
      "strict_scored": 10,
      "strict_correct": 10,
      "review_pending": 0
    },
    "IFEval": {
      "attempted": 10,
      "completed": 6,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 6
    },
    "Q36-Output-Integrity": {
      "attempted": 5,
      "completed": 5,
      "strict_scored": 5,
      "strict_correct": 5,
      "review_pending": 0
    },
    "Q36-Legal-Alternatives-and-Boundaries": {
      "attempted": 10,
      "completed": 10,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 10
    },
    "Q36-Direct-Style-and-Personality": {
      "attempted": 10,
      "completed": 9,
      "strict_scored": 0,
      "strict_correct": 0,
      "review_pending": 9
    }
  },
  "median_tps_including_prefill_for_outputs_ge64_tokens": 156.35807976215648,
  "speed_sample_count": 104,
  "limitations": [
    "Compact mixed diagnostic, not official benchmark leaderboard results.",
    "No meaningful overall accuracy is claimed across heterogeneous tasks and incomplete evaluators.",
    "Strict scores combine required answer content and output format; semantically correct but misformatted outputs can fail.",
    "Coding execution tests, several mathematics tasks and subjective rubrics still require their evaluators/review.",
    "Greedy decoding, thinking disabled. Repetition/token-limit failures remain visible; the Transformers helper is not active in llama.cpp.",
    "Speed includes prefill and is workload dependent, not a pure decode microbenchmark.",
    "Only Q4_K_M on H200 was benchmarked here. This does not establish BF16/other-quant or other-device quality/performance.",
    "No completed original-Huihui comparison; no claim of outperforming the source model.",
    "204 available prompts from the larger proposed matrix; unavailable adapters are documented in the private preparation report."
  ]
}