File size: 4,135 Bytes
38b3221 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | {
"suite": "TinyCeNN FastEval v1",
"official_full_benchmark": false,
"method": "deterministic sampled zero-shot next-token letter scoring",
"samples_per_benchmark": 50,
"seed": 42,
"datasets": {
"MMLU-Pro": "TIGER-Lab/MMLU-Pro:test",
"PIQA": "lighteval/piqa:validation",
"MMMLU-DE": "openai/MMMLU:DE_DE:test",
"GPQA-Diamond": "Wanfq/gpqa:gpqa_diamond:train"
},
"device": "cuda",
"torch_version": "2.11.0+cu128",
"results": [
{
"name": "vtava__Qwen3.5-0.8B-MemoryFusion-Standalone",
"benchmarks": [
{
"benchmark": "MMLU-Pro",
"samples": 50,
"correct": 5,
"accuracy_pct": 10.0,
"seconds": 16.41,
"items_per_second": 3.046,
"input_tokens": 11865
},
{
"benchmark": "PIQA",
"samples": 50,
"correct": 27,
"accuracy_pct": 54.0,
"seconds": 7.66,
"items_per_second": 6.529,
"input_tokens": 3588
},
{
"benchmark": "MMMLU-DE",
"samples": 50,
"correct": 14,
"accuracy_pct": 28.0,
"seconds": 8.62,
"items_per_second": 5.802,
"input_tokens": 4562
},
{
"benchmark": "GPQA-Diamond",
"samples": 50,
"correct": 15,
"accuracy_pct": 30.0,
"seconds": 15.72,
"items_per_second": 3.18,
"input_tokens": 10910
}
],
"overall": {
"samples": 200,
"correct": 61,
"accuracy_pct": 30.5,
"seconds": 48.41,
"input_tokens": 30925
},
"generation": {
"prompts": 3,
"generated_tokens": 72,
"seconds": 8.094,
"tokens_per_second": 8.895,
"peak_vram_gib": 1.51,
"sample_outputs": [
"\n\n<think>\n\n</think>\n\nVienna is the capital of Austria because it serves as the central political, cultural, and administrative",
"\n\nTo determine how long the robot can operate, we need to calculate the total number of hours it can run on one",
"\n\n<think>\n\n</think>\n\nAn **API gateway** is a central server that acts as a bridge between a web application ("
]
},
"peak_vram_gib_observed": 1.51
},
{
"name": "Qwen/Qwen3.5-0.8B",
"benchmarks": [
{
"benchmark": "MMLU-Pro",
"samples": 50,
"correct": 6,
"accuracy_pct": 12.0,
"seconds": 10.51,
"items_per_second": 4.756,
"input_tokens": 11865
},
{
"benchmark": "PIQA",
"samples": 50,
"correct": 24,
"accuracy_pct": 48.0,
"seconds": 4.68,
"items_per_second": 10.674,
"input_tokens": 3588
},
{
"benchmark": "MMMLU-DE",
"samples": 50,
"correct": 17,
"accuracy_pct": 34.0,
"seconds": 5.45,
"items_per_second": 9.17,
"input_tokens": 4562
},
{
"benchmark": "GPQA-Diamond",
"samples": 50,
"correct": 13,
"accuracy_pct": 26.0,
"seconds": 9.65,
"items_per_second": 5.181,
"input_tokens": 10910
}
],
"overall": {
"samples": 200,
"correct": 60,
"accuracy_pct": 30.0,
"seconds": 30.3,
"input_tokens": 30925
},
"generation": {
"prompts": 3,
"generated_tokens": 72,
"seconds": 5.225,
"tokens_per_second": 13.781,
"peak_vram_gib": 1.473,
"sample_outputs": [
"\n\n<think>\n\n</think>\n\nVienna is the capital of Austria because it is the largest city in the country and serves as",
"\n\nTo determine how long the robot can operate, we need to calculate the total number of hours it can run on one",
"\n\n<think>\n\n</think>\n\nAn **API Gateway** is a central server that acts as a single entry point for all incoming"
]
},
"peak_vram_gib_observed": 1.473
}
]
} |