Hugging Face's logo

RedHatAI
/
Qwen3.6-35B-A3B-NVFP4

Image-Text-to-Text
Transformers
Safetensors
qwen3_5_moe
fp4
qwen
nvfp4
vllm
llm-compressor
compressed-tensors
conversational
8-bit precision
Model card Files
xet
 Community
10
ChibuUkachi commited on
Commit
54d4831
·
1 Parent(s): db94449

add bf16 every eval ever

Browse files
Files changed (7) hide show
  1. bf16_every_eval_ever/aime25.json +127 -0
  2. bf16_every_eval_ever/gpqa_diamond.json +85 -0
  3. bf16_every_eval_ever/gsm8k_platinum_cot_llama.json +121 -0
  4. bf16_every_eval_ever/ifeval.json +215 -0
  5. bf16_every_eval_ever/lcb_codegeneration_v6.json +85 -0
  6. bf16_every_eval_ever/math_500.json +82 -0
  7. bf16_every_eval_ever/mmlu_pro_chat.json +55 -0
bf16_every_eval_ever/aime25.json ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "aime25|0/Qwen/Qwen3.6-35B-A3B/1782386750.923367",
4
+ "evaluation_timestamp": "11302276",
5
+ "retrieved_timestamp": "1782386750.923367",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "6f0f351abfd7c06004bf0dcdaeab14c8f289b86c"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "inference_engine": {
21
+ "name": "vllm"
22
+ },
23
+ "additional_details": {
24
+ "provider": "hosted_vllm",
25
+ "base_url": "http://0.0.0.0:8700/v1",
26
+ "concurrent_requests": "16",
27
+ "verbose": "False",
28
+ "api_max_retry": "8",
29
+ "api_retry_sleep": "1.0",
30
+ "api_retry_multiplier": "2.0",
31
+ "timeout": "1200.0",
32
+ "num_seeds_merged": "8"
33
+ }
34
+ },
35
+ "evaluation_results": [
36
+ {
37
+ "evaluation_name": "aime25",
38
+ "source_data": {
39
+ "dataset_name": "aime25",
40
+ "source_type": "hf_dataset",
41
+ "hf_repo": "yentinglin/aime_2025",
42
+ "hf_split": "train"
43
+ },
44
+ "metric_config": {
45
+ "evaluation_description": "pass@k:k=1&n=1",
46
+ "lower_is_better": false,
47
+ "score_type": "continuous",
48
+ "min_score": 0.0,
49
+ "max_score": 1.0
50
+ },
51
+ "score_details": {
52
+ "score": 0.9291666666666667,
53
+ "details": {
54
+ "seed_scores": "[0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9, 0.9333333333333333, 0.9333333333333333]",
55
+ "evaluation_timestamps": "[11302276, 11303332, 11304489, 11305762, 11307069, 11308407, 11309761, 11311085]",
56
+ "seed_values": "[42, 1234, 4158, 5322, 1356, 9843, 3344, 5678]"
57
+ },
58
+ "uncertainty": {
59
+ "standard_error": {
60
+ "value": 0.004166666666666666,
61
+ "method": "across_seeds"
62
+ },
63
+ "num_samples": 8
64
+ }
65
+ },
66
+ "generation_config": {
67
+ "generation_args": {
68
+ "temperature": 1.0,
69
+ "top_p": 0.95,
70
+ "top_k": 20.0,
71
+ "max_tokens": 64000,
72
+ "max_attempts": 1
73
+ },
74
+ "additional_details": {
75
+ "seed": "42",
76
+ "min_p": "0.0",
77
+ "num_fewshot": "0"
78
+ }
79
+ }
80
+ },
81
+ {
82
+ "evaluation_name": "aime25",
83
+ "source_data": {
84
+ "dataset_name": "aime25",
85
+ "source_type": "hf_dataset",
86
+ "hf_repo": "yentinglin/aime_2025",
87
+ "hf_split": "train"
88
+ },
89
+ "metric_config": {
90
+ "evaluation_description": "avg@n:n=1",
91
+ "lower_is_better": false,
92
+ "score_type": "continuous",
93
+ "min_score": 0.0,
94
+ "max_score": 1.0
95
+ },
96
+ "score_details": {
97
+ "score": 0.9291666666666667,
98
+ "details": {
99
+ "seed_scores": "[0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9333333333333333, 0.9, 0.9333333333333333, 0.9333333333333333]",
100
+ "evaluation_timestamps": "[11302276, 11303332, 11304489, 11305762, 11307069, 11308407, 11309761, 11311085]",
101
+ "seed_values": "[42, 1234, 4158, 5322, 1356, 9843, 3344, 5678]"
102
+ },
103
+ "uncertainty": {
104
+ "standard_error": {
105
+ "value": 0.004166666666666666,
106
+ "method": "across_seeds"
107
+ },
108
+ "num_samples": 8
109
+ }
110
+ },
111
+ "generation_config": {
112
+ "generation_args": {
113
+ "temperature": 1.0,
114
+ "top_p": 0.95,
115
+ "top_k": 20.0,
116
+ "max_tokens": 64000,
117
+ "max_attempts": 1
118
+ },
119
+ "additional_details": {
120
+ "seed": "42",
121
+ "min_p": "0.0",
122
+ "num_fewshot": "0"
123
+ }
124
+ }
125
+ }
126
+ ]
127
+ }
bf16_every_eval_ever/gpqa_diamond.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "gpqa:diamond|0/Qwen/Qwen3.6-35B-A3B/1782386774.185584",
4
+ "evaluation_timestamp": "11312355",
5
+ "retrieved_timestamp": "1782386774.185584",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "6f0f351abfd7c06004bf0dcdaeab14c8f289b86c"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "inference_engine": {
21
+ "name": "vllm"
22
+ },
23
+ "additional_details": {
24
+ "provider": "hosted_vllm",
25
+ "base_url": "http://0.0.0.0:8700/v1",
26
+ "concurrent_requests": "16",
27
+ "verbose": "False",
28
+ "api_max_retry": "8",
29
+ "api_retry_sleep": "1.0",
30
+ "api_retry_multiplier": "2.0",
31
+ "timeout": "1200.0",
32
+ "num_seeds_merged": "3"
33
+ }
34
+ },
35
+ "evaluation_results": [
36
+ {
37
+ "evaluation_name": "gpqa:diamond",
38
+ "source_data": {
39
+ "dataset_name": "gpqa:diamond",
40
+ "source_type": "hf_dataset",
41
+ "hf_repo": "Idavidrein/gpqa",
42
+ "hf_split": "train",
43
+ "additional_details": {
44
+ "hf_subset": "gpqa_diamond"
45
+ }
46
+ },
47
+ "metric_config": {
48
+ "evaluation_description": "gpqa_pass@k:k=1",
49
+ "lower_is_better": false,
50
+ "score_type": "continuous",
51
+ "min_score": 0.0,
52
+ "max_score": 1.0
53
+ },
54
+ "score_details": {
55
+ "score": 0.8451178451178452,
56
+ "details": {
57
+ "seed_scores": "[0.8333333333333334, 0.8434343434343434, 0.8585858585858586]",
58
+ "evaluation_timestamps": "[11312355, 11314987, 11316709]",
59
+ "seed_values": "[42, 1234, 4158]"
60
+ },
61
+ "uncertainty": {
62
+ "standard_error": {
63
+ "value": 0.007338213709664417,
64
+ "method": "across_seeds"
65
+ },
66
+ "num_samples": 3
67
+ }
68
+ },
69
+ "generation_config": {
70
+ "generation_args": {
71
+ "temperature": 1.0,
72
+ "top_p": 0.95,
73
+ "top_k": 20.0,
74
+ "max_tokens": 64000,
75
+ "max_attempts": 1
76
+ },
77
+ "additional_details": {
78
+ "seed": "42",
79
+ "min_p": "0.0",
80
+ "num_fewshot": "0"
81
+ }
82
+ }
83
+ }
84
+ ]
85
+ }
bf16_every_eval_ever/gsm8k_platinum_cot_llama.json ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "gsm8k_platinum_cot_llama/Qwen/Qwen3.6-35B-A3B/1782386692.42944",
4
+ "evaluation_timestamp": "1782269407",
5
+ "retrieved_timestamp": "1782386692.42944",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "additional_details": {
21
+ "model_args": "{'model': 'Qwen/Qwen3.6-35B-A3B', 'max_length': 262144, 'base_url': 'http://0.0.0.0:8700/v1/chat/completions', 'num_concurrent': 64, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 1200}",
22
+ "num_seeds_merged": "3"
23
+ }
24
+ },
25
+ "evaluation_results": [
26
+ {
27
+ "evaluation_name": "gsm8k_platinum_cot_llama/strict-match",
28
+ "source_data": {
29
+ "dataset_name": "gsm8k_platinum_cot_llama",
30
+ "source_type": "hf_dataset",
31
+ "hf_repo": "madrylab/gsm8k-platinum",
32
+ "hf_split": "test"
33
+ },
34
+ "metric_config": {
35
+ "evaluation_description": "exact_match (filter: strict-match)",
36
+ "lower_is_better": false,
37
+ "score_type": "continuous",
38
+ "min_score": 0.0,
39
+ "max_score": 1.0
40
+ },
41
+ "score_details": {
42
+ "score": 0.9572649572649573,
43
+ "details": {
44
+ "seed_scores": "[0.9561621174524401, 0.9594706368899917, 0.9561621174524401]",
45
+ "evaluation_timestamps": "[1782271494, 1782273956, 1782269407]",
46
+ "seed_values": "[1234, 4158, 42]"
47
+ },
48
+ "uncertainty": {
49
+ "standard_error": {
50
+ "value": 0.0011028398125172052,
51
+ "method": "across_seeds"
52
+ },
53
+ "num_samples": 3
54
+ }
55
+ },
56
+ "generation_config": {
57
+ "generation_args": {
58
+ "temperature": 1.0,
59
+ "top_p": 0.95,
60
+ "top_k": 20.0,
61
+ "max_tokens": 64000,
62
+ "max_attempts": 1
63
+ },
64
+ "additional_details": {
65
+ "do_sample": "true",
66
+ "until": "[\"<|eot_id|>\", \"<|start_header_id|>user<|end_header_id|>\", \"Q:\", \"</s>\", \"<|im_end|>\"]",
67
+ "min_p": "0.0",
68
+ "seed": "1234",
69
+ "num_fewshot": "0"
70
+ }
71
+ }
72
+ },
73
+ {
74
+ "evaluation_name": "gsm8k_platinum_cot_llama/flexible-extract",
75
+ "source_data": {
76
+ "dataset_name": "gsm8k_platinum_cot_llama",
77
+ "source_type": "hf_dataset",
78
+ "hf_repo": "madrylab/gsm8k-platinum",
79
+ "hf_split": "test"
80
+ },
81
+ "metric_config": {
82
+ "evaluation_description": "exact_match (filter: flexible-extract)",
83
+ "lower_is_better": false,
84
+ "score_type": "continuous",
85
+ "min_score": 0.0,
86
+ "max_score": 1.0
87
+ },
88
+ "score_details": {
89
+ "score": 0.9586435070306039,
90
+ "details": {
91
+ "seed_scores": "[0.9586435070306039, 0.9602977667493796, 0.956989247311828]",
92
+ "evaluation_timestamps": "[1782271494, 1782273956, 1782269407]",
93
+ "seed_values": "[1234, 4158, 42]"
94
+ },
95
+ "uncertainty": {
96
+ "standard_error": {
97
+ "value": 0.0009550872939447674,
98
+ "method": "across_seeds"
99
+ },
100
+ "num_samples": 3
101
+ }
102
+ },
103
+ "generation_config": {
104
+ "generation_args": {
105
+ "temperature": 1.0,
106
+ "top_p": 0.95,
107
+ "top_k": 20.0,
108
+ "max_tokens": 64000,
109
+ "max_attempts": 1
110
+ },
111
+ "additional_details": {
112
+ "do_sample": "true",
113
+ "until": "[\"<|eot_id|>\", \"<|start_header_id|>user<|end_header_id|>\", \"Q:\", \"</s>\", \"<|im_end|>\"]",
114
+ "min_p": "0.0",
115
+ "seed": "1234",
116
+ "num_fewshot": "0"
117
+ }
118
+ }
119
+ }
120
+ ]
121
+ }
bf16_every_eval_ever/ifeval.json ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "ifeval/Qwen/Qwen3.6-35B-A3B/1782386605.760545",
4
+ "evaluation_timestamp": "1782270017",
5
+ "retrieved_timestamp": "1782386605.760545",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "additional_details": {
21
+ "model_args": "{'model': 'Qwen/Qwen3.6-35B-A3B', 'max_length': 262144, 'base_url': 'http://0.0.0.0:8700/v1/chat/completions', 'num_concurrent': 64, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 1200}",
22
+ "num_seeds_merged": "3"
23
+ }
24
+ },
25
+ "evaluation_results": [
26
+ {
27
+ "evaluation_name": "ifeval",
28
+ "source_data": {
29
+ "dataset_name": "ifeval",
30
+ "source_type": "hf_dataset",
31
+ "hf_repo": "google/IFEval",
32
+ "hf_split": "train"
33
+ },
34
+ "metric_config": {
35
+ "evaluation_description": "prompt_level_strict_acc",
36
+ "lower_is_better": false,
37
+ "score_type": "continuous",
38
+ "min_score": 0.0,
39
+ "max_score": 1.0
40
+ },
41
+ "score_details": {
42
+ "score": 0.9014171287738755,
43
+ "details": {
44
+ "seed_scores": "[0.9020332717190388, 0.9020332717190388, 0.9001848428835489]",
45
+ "evaluation_timestamps": "[1782272527, 1782274610, 1782270017]",
46
+ "seed_values": "[1234, 4158, 42]"
47
+ },
48
+ "uncertainty": {
49
+ "standard_error": {
50
+ "value": 0.0006161429451632857,
51
+ "method": "across_seeds"
52
+ },
53
+ "num_samples": 3
54
+ }
55
+ },
56
+ "generation_config": {
57
+ "generation_args": {
58
+ "temperature": 1.0,
59
+ "top_p": 0.95,
60
+ "top_k": 20.0,
61
+ "max_tokens": 64000,
62
+ "max_attempts": 1
63
+ },
64
+ "additional_details": {
65
+ "until": "[]",
66
+ "do_sample": "true",
67
+ "min_p": "0.0",
68
+ "seed": "1234",
69
+ "num_fewshot": "0"
70
+ }
71
+ }
72
+ },
73
+ {
74
+ "evaluation_name": "ifeval",
75
+ "source_data": {
76
+ "dataset_name": "ifeval",
77
+ "source_type": "hf_dataset",
78
+ "hf_repo": "google/IFEval",
79
+ "hf_split": "train"
80
+ },
81
+ "metric_config": {
82
+ "evaluation_description": "inst_level_strict_acc",
83
+ "lower_is_better": false,
84
+ "score_type": "continuous",
85
+ "min_score": 0.0,
86
+ "max_score": 1.0
87
+ },
88
+ "score_details": {
89
+ "score": 0.9308553157474021,
90
+ "details": {
91
+ "seed_scores": "[0.9316546762589928, 0.9328537170263789, 0.9280575539568345]",
92
+ "evaluation_timestamps": "[1782272527, 1782274610, 1782270017]",
93
+ "seed_values": "[1234, 4158, 42]"
94
+ },
95
+ "uncertainty": {
96
+ "standard_error": {
97
+ "value": 0.0014410676560607384,
98
+ "method": "across_seeds"
99
+ },
100
+ "num_samples": 3
101
+ }
102
+ },
103
+ "generation_config": {
104
+ "generation_args": {
105
+ "temperature": 1.0,
106
+ "top_p": 0.95,
107
+ "top_k": 20.0,
108
+ "max_tokens": 64000,
109
+ "max_attempts": 1
110
+ },
111
+ "additional_details": {
112
+ "until": "[]",
113
+ "do_sample": "true",
114
+ "min_p": "0.0",
115
+ "seed": "1234",
116
+ "num_fewshot": "0"
117
+ }
118
+ }
119
+ },
120
+ {
121
+ "evaluation_name": "ifeval",
122
+ "source_data": {
123
+ "dataset_name": "ifeval",
124
+ "source_type": "hf_dataset",
125
+ "hf_repo": "google/IFEval",
126
+ "hf_split": "train"
127
+ },
128
+ "metric_config": {
129
+ "evaluation_description": "prompt_level_loose_acc",
130
+ "lower_is_better": false,
131
+ "score_type": "continuous",
132
+ "min_score": 0.0,
133
+ "max_score": 1.0
134
+ },
135
+ "score_details": {
136
+ "score": 0.9285274183610598,
137
+ "details": {
138
+ "seed_scores": "[0.9279112754158965, 0.933456561922366, 0.9242144177449169]",
139
+ "evaluation_timestamps": "[1782272527, 1782274610, 1782270017]",
140
+ "seed_values": "[1234, 4158, 42]"
141
+ },
142
+ "uncertainty": {
143
+ "standard_error": {
144
+ "value": 0.002685704832742256,
145
+ "method": "across_seeds"
146
+ },
147
+ "num_samples": 3
148
+ }
149
+ },
150
+ "generation_config": {
151
+ "generation_args": {
152
+ "temperature": 1.0,
153
+ "top_p": 0.95,
154
+ "top_k": 20.0,
155
+ "max_tokens": 64000,
156
+ "max_attempts": 1
157
+ },
158
+ "additional_details": {
159
+ "until": "[]",
160
+ "do_sample": "true",
161
+ "min_p": "0.0",
162
+ "seed": "1234",
163
+ "num_fewshot": "0"
164
+ }
165
+ }
166
+ },
167
+ {
168
+ "evaluation_name": "ifeval",
169
+ "source_data": {
170
+ "dataset_name": "ifeval",
171
+ "source_type": "hf_dataset",
172
+ "hf_repo": "google/IFEval",
173
+ "hf_split": "train"
174
+ },
175
+ "metric_config": {
176
+ "evaluation_description": "inst_level_loose_acc",
177
+ "lower_is_better": false,
178
+ "score_type": "continuous",
179
+ "min_score": 0.0,
180
+ "max_score": 1.0
181
+ },
182
+ "score_details": {
183
+ "score": 0.9484412470023981,
184
+ "details": {
185
+ "seed_scores": "[0.9484412470023981, 0.9532374100719424, 0.9436450839328537]",
186
+ "evaluation_timestamps": "[1782272527, 1782274610, 1782270017]",
187
+ "seed_values": "[1234, 4158, 42]"
188
+ },
189
+ "uncertainty": {
190
+ "standard_error": {
191
+ "value": 0.0027690660392787636,
192
+ "method": "across_seeds"
193
+ },
194
+ "num_samples": 3
195
+ }
196
+ },
197
+ "generation_config": {
198
+ "generation_args": {
199
+ "temperature": 1.0,
200
+ "top_p": 0.95,
201
+ "top_k": 20.0,
202
+ "max_tokens": 64000,
203
+ "max_attempts": 1
204
+ },
205
+ "additional_details": {
206
+ "until": "[]",
207
+ "do_sample": "true",
208
+ "min_p": "0.0",
209
+ "seed": "1234",
210
+ "num_fewshot": "0"
211
+ }
212
+ }
213
+ }
214
+ ]
215
+ }
bf16_every_eval_ever/lcb_codegeneration_v6.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "lcb:codegeneration_v6|0/Qwen/Qwen3.6-35B-A3B/1782386822.083653",
4
+ "evaluation_timestamp": "11343229",
5
+ "retrieved_timestamp": "1782386822.083653",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "6f0f351abfd7c06004bf0dcdaeab14c8f289b86c"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "inference_engine": {
21
+ "name": "vllm"
22
+ },
23
+ "additional_details": {
24
+ "provider": "hosted_vllm",
25
+ "base_url": "http://0.0.0.0:8700/v1",
26
+ "concurrent_requests": "16",
27
+ "verbose": "False",
28
+ "api_max_retry": "8",
29
+ "api_retry_sleep": "1.0",
30
+ "api_retry_multiplier": "2.0",
31
+ "timeout": "1200.0",
32
+ "num_seeds_merged": "3"
33
+ }
34
+ },
35
+ "evaluation_results": [
36
+ {
37
+ "evaluation_name": "lcb:codegeneration_v6",
38
+ "source_data": {
39
+ "dataset_name": "lcb:codegeneration_v6",
40
+ "source_type": "hf_dataset",
41
+ "hf_repo": "lighteval/code_generation_lite",
42
+ "hf_split": "test",
43
+ "additional_details": {
44
+ "hf_subset": "v6"
45
+ }
46
+ },
47
+ "metric_config": {
48
+ "evaluation_description": "codegen_pass@1:16",
49
+ "lower_is_better": false,
50
+ "score_type": "continuous",
51
+ "min_score": 0.0,
52
+ "max_score": 1.0
53
+ },
54
+ "score_details": {
55
+ "score": 0.7733333333333333,
56
+ "details": {
57
+ "seed_scores": "[0.7828571428571428, 0.7542857142857143, 0.7828571428571428]",
58
+ "evaluation_timestamps": "[11343229, 11345199, 11355037]",
59
+ "seed_values": "[42, 1234, 4158]"
60
+ },
61
+ "uncertainty": {
62
+ "standard_error": {
63
+ "value": 0.009523809523809492,
64
+ "method": "across_seeds"
65
+ },
66
+ "num_samples": 3
67
+ }
68
+ },
69
+ "generation_config": {
70
+ "generation_args": {
71
+ "temperature": 1.0,
72
+ "top_p": 0.95,
73
+ "top_k": 20.0,
74
+ "max_tokens": 64000,
75
+ "max_attempts": 1
76
+ },
77
+ "additional_details": {
78
+ "seed": "42",
79
+ "min_p": "0.0",
80
+ "num_fewshot": "0"
81
+ }
82
+ }
83
+ }
84
+ ]
85
+ }
bf16_every_eval_ever/math_500.json ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "math_500|0/Qwen/Qwen3.6-35B-A3B/1782386797.165193",
4
+ "evaluation_timestamp": "11267660",
5
+ "retrieved_timestamp": "1782386797.165193",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "6f0f351abfd7c06004bf0dcdaeab14c8f289b86c"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "inference_engine": {
21
+ "name": "vllm"
22
+ },
23
+ "additional_details": {
24
+ "provider": "hosted_vllm",
25
+ "base_url": "http://0.0.0.0:8700/v1",
26
+ "concurrent_requests": "16",
27
+ "verbose": "False",
28
+ "api_max_retry": "8",
29
+ "api_retry_sleep": "1.0",
30
+ "api_retry_multiplier": "2.0",
31
+ "timeout": "1200.0",
32
+ "num_seeds_merged": "3"
33
+ }
34
+ },
35
+ "evaluation_results": [
36
+ {
37
+ "evaluation_name": "math_500",
38
+ "source_data": {
39
+ "dataset_name": "math_500",
40
+ "source_type": "hf_dataset",
41
+ "hf_repo": "HuggingFaceH4/MATH-500",
42
+ "hf_split": "test"
43
+ },
44
+ "metric_config": {
45
+ "evaluation_description": "pass@k:k=1&n=1",
46
+ "lower_is_better": false,
47
+ "score_type": "continuous",
48
+ "min_score": 0.0,
49
+ "max_score": 1.0
50
+ },
51
+ "score_details": {
52
+ "score": 0.848,
53
+ "details": {
54
+ "seed_scores": "[0.846, 0.85, 0.848]",
55
+ "evaluation_timestamps": "[11267660, 11274842, 11281738]",
56
+ "seed_values": "[42, 1234, 4158]"
57
+ },
58
+ "uncertainty": {
59
+ "standard_error": {
60
+ "value": 0.0011547005383792527,
61
+ "method": "across_seeds"
62
+ },
63
+ "num_samples": 3
64
+ }
65
+ },
66
+ "generation_config": {
67
+ "generation_args": {
68
+ "temperature": 1.0,
69
+ "top_p": 0.95,
70
+ "top_k": 20.0,
71
+ "max_tokens": 64000,
72
+ "max_attempts": 1
73
+ },
74
+ "additional_details": {
75
+ "seed": "42",
76
+ "min_p": "0.0",
77
+ "num_fewshot": "0"
78
+ }
79
+ }
80
+ }
81
+ ]
82
+ }
bf16_every_eval_ever/mmlu_pro_chat.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "mmlu_pro_chat/Qwen/Qwen3.6-35B-A3B/1782386635.276605",
4
+ "evaluation_timestamp": "1782275992",
5
+ "retrieved_timestamp": "1782386635.276605",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "Qwen/Qwen3.6-35B-A3B",
18
+ "id": "Qwen/Qwen3.6-35B-A3B",
19
+ "developer": "Qwen",
20
+ "additional_details": {
21
+ "model_args": "{'model': 'Qwen/Qwen3.6-35B-A3B', 'max_length': 262144, 'base_url': 'http://0.0.0.0:8700/v1/chat/completions', 'num_concurrent': 64, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 1200}",
22
+ "num_seeds_merged": "3"
23
+ }
24
+ },
25
+ "evaluation_results": [
26
+ {
27
+ "evaluation_name": "mmlu_pro_chat/custom-extract",
28
+ "source_data": {
29
+ "dataset_name": "mmlu_pro_chat",
30
+ "source_type": "other"
31
+ },
32
+ "metric_config": {
33
+ "evaluation_description": "exact_match (filter: custom-extract)",
34
+ "lower_is_better": false,
35
+ "score_type": "continuous",
36
+ "min_score": 0.0,
37
+ "max_score": 1.0
38
+ },
39
+ "score_details": {
40
+ "score": 0.8531693262411347,
41
+ "details": {
42
+ "seed_scores": "[0.856216755319149, 0.8533909574468085, 0.8499002659574468]",
43
+ "evaluation_timestamps": "[1782346537, 1782362973, 1782275992]"
44
+ },
45
+ "uncertainty": {
46
+ "standard_error": {
47
+ "value": 0.0018267776509125202,
48
+ "method": "across_seeds"
49
+ },
50
+ "num_samples": 3
51
+ }
52
+ }
53
+ }
54
+ ]
55
+ }