{
  "suite": "TinyCeNN FastEval v1",
  "official_full_benchmark": false,
  "method": "deterministic sampled zero-shot next-token letter scoring",
  "samples_per_benchmark": 50,
  "seed": 42,
  "datasets": {
    "MMLU-Pro": "TIGER-Lab/MMLU-Pro:test",
    "PIQA": "lighteval/piqa:validation",
    "MMMLU-DE": "openai/MMMLU:DE_DE:test",
    "GPQA-Diamond": "Wanfq/gpqa:gpqa_diamond:train"
  },
  "device": "cuda",
  "torch_version": "2.11.0+cu128",
  "results": [
    {
      "name": "vtava__Qwen3.5-0.8B-CeNN-Integrated-V1-Standalone",
      "benchmarks": [
        {
          "benchmark": "MMLU-Pro",
          "samples": 50,
          "correct": 6,
          "accuracy_pct": 12.0,
          "seconds": 17.82,
          "items_per_second": 2.806,
          "input_tokens": 11865
        },
        {
          "benchmark": "PIQA",
          "samples": 50,
          "correct": 21,
          "accuracy_pct": 42.0,
          "seconds": 14.08,
          "items_per_second": 3.551,
          "input_tokens": 3588
        },
        {
          "benchmark": "MMMLU-DE",
          "samples": 50,
          "correct": 22,
          "accuracy_pct": 44.0,
          "seconds": 10.0,
          "items_per_second": 5.001,
          "input_tokens": 4562
        },
        {
          "benchmark": "GPQA-Diamond",
          "samples": 50,
          "correct": 15,
          "accuracy_pct": 30.0,
          "seconds": 12.9,
          "items_per_second": 3.876,
          "input_tokens": 10910
        }
      ],
      "overall": {
        "samples": 200,
        "correct": 64,
        "accuracy_pct": 32.0,
        "seconds": 54.8,
        "input_tokens": 30925
      },
      "generation": {
        "prompts": 3,
        "generated_tokens": 72,
        "seconds": 12.419,
        "tokens_per_second": 5.798,
        "peak_vram_gib": 1.477,
        "sample_outputs": [
          "\n\n<think>\n\n</think>\n\nVienna is the capital of Austria because it is the largest city in the country, serving as",
          "\n\nTo determine how long the robot can operate, we need to calculate the total number of hours it can run on one",
          "\n\n<think>\n\n</think>\n\nAn **API Gateway** is a central server that acts as a single entry point for all incoming"
        ]
      },
      "peak_vram_gib_observed": 1.477
    },
    {
      "name": "Qwen/Qwen3.5-0.8B",
      "benchmarks": [
        {
          "benchmark": "MMLU-Pro",
          "samples": 50,
          "correct": 6,
          "accuracy_pct": 12.0,
          "seconds": 10.22,
          "items_per_second": 4.892,
          "input_tokens": 11865
        },
        {
          "benchmark": "PIQA",
          "samples": 50,
          "correct": 24,
          "accuracy_pct": 48.0,
          "seconds": 4.66,
          "items_per_second": 10.723,
          "input_tokens": 3588
        },
        {
          "benchmark": "MMMLU-DE",
          "samples": 50,
          "correct": 17,
          "accuracy_pct": 34.0,
          "seconds": 5.41,
          "items_per_second": 9.242,
          "input_tokens": 4562
        },
        {
          "benchmark": "GPQA-Diamond",
          "samples": 50,
          "correct": 13,
          "accuracy_pct": 26.0,
          "seconds": 9.74,
          "items_per_second": 5.133,
          "input_tokens": 10910
        }
      ],
      "overall": {
        "samples": 200,
        "correct": 60,
        "accuracy_pct": 30.0,
        "seconds": 30.04,
        "input_tokens": 30925
      },
      "generation": {
        "prompts": 3,
        "generated_tokens": 72,
        "seconds": 5.384,
        "tokens_per_second": 13.373,
        "peak_vram_gib": 1.473,
        "sample_outputs": [
          "\n\n<think>\n\n</think>\n\nVienna is the capital of Austria because it is the largest city in the country and serves as",
          "\n\nTo determine how long the robot can operate, we need to calculate the total number of hours it can run on one",
          "\n\n<think>\n\n</think>\n\nAn **API Gateway** is a central server that acts as a single entry point for all incoming"
        ]
      },
      "peak_vram_gib_observed": 1.473
    }
  ]
}