{
  "suite": "TinyCeNN FastEval v1",
  "official_full_benchmark": false,
  "method": "deterministic sampled zero-shot next-token letter scoring",
  "samples_per_benchmark": 50,
  "seed": 42,
  "datasets": {
    "MMLU-Pro": "TIGER-Lab/MMLU-Pro:test",
    "PIQA": "lighteval/piqa:validation",
    "MMMLU-DE": "openai/MMMLU:DE_DE:test",
    "GPQA-Diamond": "Wanfq/gpqa:gpqa_diamond:train"
  },
  "device": "cuda",
  "torch_version": "2.11.0+cu128",
  "results": [
    {
      "name": "vtava__Qwen3.5-0.8B-PDelta3-CLVR-Local32-Standalone",
      "benchmarks": [
        {
          "benchmark": "MMLU-Pro",
          "samples": 50,
          "correct": 7,
          "accuracy_pct": 14.0,
          "seconds": 10.22,
          "items_per_second": 4.891,
          "input_tokens": 11865
        },
        {
          "benchmark": "PIQA",
          "samples": 50,
          "correct": 24,
          "accuracy_pct": 48.0,
          "seconds": 4.61,
          "items_per_second": 10.835,
          "input_tokens": 3588
        },
        {
          "benchmark": "MMMLU-DE",
          "samples": 50,
          "correct": 15,
          "accuracy_pct": 30.0,
          "seconds": 5.49,
          "items_per_second": 9.106,
          "input_tokens": 4562
        },
        {
          "benchmark": "GPQA-Diamond",
          "samples": 50,
          "correct": 11,
          "accuracy_pct": 22.0,
          "seconds": 9.47,
          "items_per_second": 5.282,
          "input_tokens": 10910
        }
      ],
      "overall": {
        "samples": 200,
        "correct": 57,
        "accuracy_pct": 28.5,
        "seconds": 29.8,
        "input_tokens": 30925
      },
      "generation": {
        "prompts": 3,
        "generated_tokens": 72,
        "seconds": 5.9,
        "tokens_per_second": 12.204,
        "peak_vram_gib": 1.481,
        "sample_outputs": [
          "\n\n<think>\n\n</think>\n\nVienna is the capital of Austria because it is the seat of the Austrian federal government and serves",
          "\n\nTo determine how long the robot can operate, we need to calculate the total number of hours it can run on one",
          "\n\n<think>\n\n</think>\n\nAn **API Gateway** is a central server that acts as a single entry point for all incoming"
        ]
      },
      "peak_vram_gib_observed": 1.481
    }
  ]
}