Hugging Face's logo

Spaces:
ai-forever
/
LIBRA-Leaderboard
Running

App Files Community
ai-forever commited on
Commit
6d9a5f4
·
1 Parent(s): c67332f

add updated scores

Browse files
Files changed (26) hide show
  1. app.py +21 -10
  2. datasets_config.json +177 -1
  3. results/01-ai_Yi-34B-200K.json +140 -0
  4. results/01-ai_Yi-9B-200K.json +140 -0
  5. results/ChatGLM2-6B-32K.json +0 -1
  6. results/GLM4-9B-Chat.json +0 -1
  7. results/GPT-4o.json +0 -1
  8. results/LLaMA-2-7B-32k.json +0 -1
  9. results/LLaMA-3-8B-Instruct.json +0 -1
  10. results/LLaMA-3-8B.json +0 -1
  11. results/LLaMA-3.1-8B-Instruct.json +0 -1
  12. results/LLaMA-3.1-8B.json +0 -1
  13. results/LiquidAI_LFM2.5-1.2B-Instruct.json +141 -0
  14. results/LongAlpaca.json +0 -1
  15. results/LongChat-7B-v1.5-32k.json +0 -1
  16. results/Mistral-7B-Instruct-v0.3.json +0 -1
  17. results/Mistral-7B-v0.1.json +0 -1
  18. results/Mistral-7B-v0.3.json +0 -1
  19. results/Mistral-Nemo-Instruct-2407.json +0 -1
  20. results/Phi-3-mini-128k-instruct.json +0 -1
  21. results/Qwen_Qwen3-30B-A3B-Instruct-2507.json +140 -0
  22. results/Saiga-LLaMA-3-8B.json +0 -1
  23. results/T-lite-instruct-0.1.json +0 -1
  24. results/gradientai_Llama-3-8B-Instruct-Gradient-1048k.json +140 -0
  25. results/meta-llama_Llama-3.1-8B-Instruct.json +140 -0
  26. results/microsoft_Phi-3.5-mini-instruct.json +140 -0
app.py CHANGED
@@ -53,9 +53,10 @@ def make_overall_table_by_tasks(files):
53
  results[datasets_params[key]["name"]].append(result_dct[model_name][key])
54
 
55
  table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
56
- cols = table.columns.tolist()
57
- cols = [cols[0]] + [cols[22]] + cols[1:22]
58
- return table[cols]
 
59
 
60
 
61
  def make_overall_table_by_lengths(files):
@@ -93,9 +94,9 @@ def make_overall_table_by_lengths(files):
93
  results[key].append(result_dct[model_name][key])
94
 
95
  table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
96
- cols = table.columns.tolist()
97
- cols = [cols[0]] + [cols[7]] + cols[1:7]
98
- return table[cols]
99
 
100
 
101
  def load_model(files, tab_name):
@@ -106,16 +107,26 @@ def load_model(files, tab_name):
106
  model_name = file.split('/')[-1].split(".json")[0]
107
  results['Model'].append(model_name)
108
  result = json.load(open("results/" + file, "r"))
 
 
 
 
 
 
 
 
109
  for length in LENGTHS:
110
- if length in result[tab_name].keys():
111
  if length == "dataset_total_score":
112
- results["Dataset Total Score"].append(round(result[tab_name][length] * 100, 1))
113
  continue
114
- results[length].append(round(result[tab_name][length] * 100, 1))
115
  else:
116
  results[length].append("-")
117
 
118
- return pd.DataFrame(results).sort_values(['Dataset Total Score'], ascending=False)
 
 
119
 
120
 
121
  def build_leaderboard_tab(files):
 
53
  results[datasets_params[key]["name"]].append(result_dct[model_name][key])
54
 
55
  table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
56
+ front = ["Model", "Total Score"]
57
+ rest = [datasets_params[t]["name"] for t in TASKS if datasets_params[t]["name"] in table.columns]
58
+ rest += [c for c in table.columns if c not in front and c not in rest]
59
+ return table[front + rest]
60
 
61
 
62
  def make_overall_table_by_lengths(files):
 
94
  results[key].append(result_dct[model_name][key])
95
 
96
  table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
97
+ front = ["Model", "Total Score"]
98
+ length_cols = [c for c in LENGTHS if c in table.columns]
99
+ return table[front + length_cols]
100
 
101
 
102
  def load_model(files, tab_name):
 
107
  model_name = file.split('/')[-1].split(".json")[0]
108
  results['Model'].append(model_name)
109
  result = json.load(open("results/" + file, "r"))
110
+ task_block = result.get(tab_name)
111
+ if not task_block:
112
+ results["Dataset Total Score"].append("-")
113
+ for length in LENGTHS:
114
+ if length == "dataset_total_score":
115
+ continue
116
+ results[length].append("-")
117
+ continue
118
  for length in LENGTHS:
119
+ if length in task_block.keys():
120
  if length == "dataset_total_score":
121
+ results["Dataset Total Score"].append(round(task_block[length] * 100, 1))
122
  continue
123
+ results[length].append(round(task_block[length] * 100, 1))
124
  else:
125
  results[length].append("-")
126
 
127
+ df = pd.DataFrame(results)
128
+ df["_sort_key"] = pd.to_numeric(df["Dataset Total Score"], errors="coerce").fillna(-1)
129
+ return df.sort_values("_sort_key", ascending=False).drop(columns=["_sort_key"])
130
 
131
 
132
  def build_leaderboard_tab(files):
datasets_config.json CHANGED
@@ -1 +1,177 @@
1
- {"passkey": {"name": "Passkey", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "matreshka_yes_no": {"name": "MatreshkaYesNo", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "matreshka_names": {"name": "MatreshkaNames", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "passkey_with_librusec": {"name": "PasskeyWithLibrusec", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "librusec_history": {"name": "LibrusecHistory", "lengths": ["8k", "16k", "32k", "64k"]}, "ru_gsm100": {"name": "ruGSM100", "lengths": ["16k"]}, "ru_sci_passage_count": {"name": "ruSciPassageCount", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_2wikimultihopqa": {"name": "ru2WikiMultihopQA", "lengths": ["8k", "16k", "32k"]}, "long_context_multiq": {"name": "LongContextMultiQ", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_sci_abstract_retrieval": {"name": "ruSciAbstractRetrieval", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_trec": {"name": "ruTREC", "lengths": ["4k", "8k", "16k", "32k"]}, "ru_sci_fi": {"name": "ruSciFi", "lengths": ["32k", "64k"]}, "librusec_mhqa": {"name": "LibrusecMHQA", "lengths": ["8k"]}, "ru_babilong_qa1": {"name": "ruBABILongQA1", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa2": {"name": "ruBABILongQA2", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa3": {"name": "ruBABILongQA3", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa4": {"name": "ruBABILongQA4", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa5": {"name": "ruBABILongQA5", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_quality": {"name": "ruQuALITY", "lengths": ["8k", "16k"]}, "ru_tpo": {"name": "ruTPO", "lengths": ["8k"]}, "ru_qasper": {"name": "ruQasper", "lengths": ["8k", "16k", "32k"]}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "passkey": {
3
+ "name": "Passkey",
4
+ "lengths": [
5
+ "4k",
6
+ "8k",
7
+ "16k",
8
+ "32k",
9
+ "64k",
10
+ "128k"
11
+ ]
12
+ },
13
+ "matreshka_yes_no": {
14
+ "name": "MatreshkaYesNo",
15
+ "lengths": [
16
+ "4k",
17
+ "8k",
18
+ "16k",
19
+ "32k",
20
+ "64k",
21
+ "128k"
22
+ ]
23
+ },
24
+ "matreshka_names": {
25
+ "name": "MatreshkaNames",
26
+ "lengths": [
27
+ "4k",
28
+ "8k",
29
+ "16k",
30
+ "32k",
31
+ "64k",
32
+ "128k"
33
+ ]
34
+ },
35
+ "passkey_with_librusec": {
36
+ "name": "PasskeyWithLibrusec",
37
+ "lengths": [
38
+ "4k",
39
+ "8k",
40
+ "16k",
41
+ "32k",
42
+ "64k",
43
+ "128k"
44
+ ]
45
+ },
46
+ "librusec_history": {
47
+ "name": "LibrusecHistory",
48
+ "lengths": [
49
+ "8k",
50
+ "16k",
51
+ "32k",
52
+ "64k"
53
+ ]
54
+ },
55
+ "ru_sci_passage_count": {
56
+ "name": "ruSciPassageCount",
57
+ "lengths": [
58
+ "4k",
59
+ "8k",
60
+ "16k",
61
+ "32k",
62
+ "64k",
63
+ "128k"
64
+ ]
65
+ },
66
+ "ru_2wikimultihopqa": {
67
+ "name": "ru2WikiMultihopQA",
68
+ "lengths": [
69
+ "8k",
70
+ "16k",
71
+ "32k"
72
+ ]
73
+ },
74
+ "long_context_multiq": {
75
+ "name": "LongContextMultiQ",
76
+ "lengths": [
77
+ "4k",
78
+ "8k",
79
+ "16k",
80
+ "32k",
81
+ "64k",
82
+ "128k"
83
+ ]
84
+ },
85
+ "ru_sci_abstract_retrieval": {
86
+ "name": "ruSciAbstractRetrieval",
87
+ "lengths": [
88
+ "4k",
89
+ "8k",
90
+ "16k",
91
+ "32k",
92
+ "64k",
93
+ "128k"
94
+ ]
95
+ },
96
+ "ru_sci_fi": {
97
+ "name": "ruSciFi",
98
+ "lengths": [
99
+ "32k",
100
+ "64k"
101
+ ]
102
+ },
103
+ "librusec_mhqa": {
104
+ "name": "LibrusecMHQA",
105
+ "lengths": [
106
+ "8k"
107
+ ]
108
+ },
109
+ "ru_babilong_qa1": {
110
+ "name": "ruBABILongQA1",
111
+ "lengths": [
112
+ "4k",
113
+ "8k",
114
+ "16k",
115
+ "32k",
116
+ "64k",
117
+ "128k"
118
+ ]
119
+ },
120
+ "ru_babilong_qa2": {
121
+ "name": "ruBABILongQA2",
122
+ "lengths": [
123
+ "4k",
124
+ "8k",
125
+ "16k",
126
+ "32k",
127
+ "64k",
128
+ "128k"
129
+ ]
130
+ },
131
+ "ru_babilong_qa3": {
132
+ "name": "ruBABILongQA3",
133
+ "lengths": [
134
+ "4k",
135
+ "8k",
136
+ "16k",
137
+ "32k",
138
+ "64k",
139
+ "128k"
140
+ ]
141
+ },
142
+ "ru_babilong_qa4": {
143
+ "name": "ruBABILongQA4",
144
+ "lengths": [
145
+ "4k",
146
+ "8k",
147
+ "16k",
148
+ "32k",
149
+ "64k",
150
+ "128k"
151
+ ]
152
+ },
153
+ "ru_babilong_qa5": {
154
+ "name": "ruBABILongQA5",
155
+ "lengths": [
156
+ "4k",
157
+ "8k",
158
+ "16k",
159
+ "32k",
160
+ "64k",
161
+ "128k"
162
+ ]
163
+ },
164
+ "ru_quality": {
165
+ "name": "ruQuALITY",
166
+ "lengths": [
167
+ "8k",
168
+ "16k"
169
+ ]
170
+ },
171
+ "ru_tpo": {
172
+ "name": "ruTPO",
173
+ "lengths": [
174
+ "8k"
175
+ ]
176
+ }
177
+ }
results/01-ai_Yi-34B-200K.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.47628148148148153,
3
+ "passkey": {
4
+ "dataset_total_score": 1.0,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 1.0,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.6670000000000001,
14
+ "4k": 0.747,
15
+ "8k": 0.66,
16
+ "16k": 0.693,
17
+ "32k": 0.66,
18
+ "64k": 0.628,
19
+ "128k": 0.614
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.362,
23
+ "4k": 0.373,
24
+ "8k": 0.42,
25
+ "16k": 0.352,
26
+ "32k": 0.287,
27
+ "64k": 0.378
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 0.9983333333333334,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 1.0,
34
+ "32k": 1.0,
35
+ "64k": 0.995,
36
+ "128k": 0.995
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.531,
40
+ "8k": 0.5,
41
+ "16k": 0.562,
42
+ "32k": 0.531,
43
+ "64k": 0.531
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.04950000000000001,
47
+ "4k": 0.144,
48
+ "8k": 0.061,
49
+ "16k": 0.07,
50
+ "32k": 0.011,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.4126666666666667,
56
+ "8k": 0.442,
57
+ "16k": 0.422,
58
+ "32k": 0.374
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.5314,
62
+ "4k": 0.544,
63
+ "8k": 0.653,
64
+ "16k": 0.59,
65
+ "32k": 0.431,
66
+ "64k": 0.439
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.5266666666666667,
70
+ "4k": 0.904,
71
+ "8k": 0.724,
72
+ "16k": 0.59,
73
+ "32k": 0.466,
74
+ "64k": 0.276,
75
+ "128k": 0.2
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.259,
79
+ "32k": 0.25,
80
+ "64k": 0.268
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.339,
84
+ "8k": 0.339
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.30449999999999994,
88
+ "4k": 0.394,
89
+ "8k": 0.384,
90
+ "16k": 0.283,
91
+ "32k": 0.319,
92
+ "64k": 0.275,
93
+ "128k": 0.172
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.12516666666666668,
97
+ "4k": 0.271,
98
+ "8k": 0.197,
99
+ "16k": 0.104,
100
+ "32k": 0.123,
101
+ "64k": 0.046,
102
+ "128k": 0.01
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.09566666666666668,
106
+ "4k": 0.167,
107
+ "8k": 0.088,
108
+ "16k": 0.145,
109
+ "32k": 0.123,
110
+ "64k": 0.031,
111
+ "128k": 0.02
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.25716666666666665,
115
+ "4k": 0.359,
116
+ "8k": 0.376,
117
+ "16k": 0.277,
118
+ "32k": 0.23,
119
+ "64k": 0.2,
120
+ "128k": 0.101
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.7374999999999999,
124
+ "4k": 0.788,
125
+ "8k": 0.727,
126
+ "16k": 0.765,
127
+ "32k": 0.729,
128
+ "64k": 0.719,
129
+ "128k": 0.697
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.7385,
133
+ "8k": 0.667,
134
+ "16k": 0.81
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.638,
138
+ "8k": 0.638
139
+ }
140
+ }
results/01-ai_Yi-9B-200K.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.49187592592592594,
3
+ "passkey": {
4
+ "dataset_total_score": 1.0,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 1.0,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.592,
14
+ "4k": 0.563,
15
+ "8k": 0.547,
16
+ "16k": 0.683,
17
+ "32k": 0.59,
18
+ "64k": 0.583,
19
+ "128k": 0.586
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.22999999999999998,
23
+ "4k": 0.4,
24
+ "8k": 0.34,
25
+ "16k": 0.234,
26
+ "32k": 0.087,
27
+ "64k": 0.089
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 0.9858333333333332,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 1.0,
34
+ "32k": 0.995,
35
+ "64k": 0.98,
36
+ "128k": 0.94
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.6875,
40
+ "8k": 0.75,
41
+ "16k": 0.656,
42
+ "32k": 0.656,
43
+ "64k": 0.688
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.06433333333333334,
47
+ "4k": 0.163,
48
+ "8k": 0.141,
49
+ "16k": 0.06,
50
+ "32k": 0.011,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.4146666666666667,
56
+ "8k": 0.395,
57
+ "16k": 0.424,
58
+ "32k": 0.425
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.28759999999999997,
62
+ "4k": 0.31,
63
+ "8k": 0.438,
64
+ "16k": 0.253,
65
+ "32k": 0.266,
66
+ "64k": 0.171
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.4388333333333332,
70
+ "4k": 0.818,
71
+ "8k": 0.595,
72
+ "16k": 0.586,
73
+ "32k": 0.369,
74
+ "64k": 0.195,
75
+ "128k": 0.07
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.514,
79
+ "32k": 0.361,
80
+ "64k": 0.667
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.341,
84
+ "8k": 0.341
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.41866666666666674,
88
+ "4k": 0.626,
89
+ "8k": 0.515,
90
+ "16k": 0.434,
91
+ "32k": 0.468,
92
+ "64k": 0.297,
93
+ "128k": 0.172
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.11533333333333333,
97
+ "4k": 0.176,
98
+ "8k": 0.184,
99
+ "16k": 0.143,
100
+ "32k": 0.082,
101
+ "64k": 0.077,
102
+ "128k": 0.03
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.24600000000000002,
106
+ "4k": 0.3,
107
+ "8k": 0.338,
108
+ "16k": 0.348,
109
+ "32k": 0.185,
110
+ "64k": 0.215,
111
+ "128k": 0.09
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.3058333333333333,
115
+ "4k": 0.487,
116
+ "8k": 0.471,
117
+ "16k": 0.386,
118
+ "32k": 0.23,
119
+ "64k": 0.16,
120
+ "128k": 0.101
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.6471666666666666,
124
+ "4k": 0.758,
125
+ "8k": 0.687,
126
+ "16k": 0.663,
127
+ "32k": 0.583,
128
+ "64k": 0.667,
129
+ "128k": 0.525
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.9590000000000001,
133
+ "8k": 1.0,
134
+ "16k": 0.918
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.606,
138
+ "8k": 0.606
139
+ }
140
+ }
results/ChatGLM2-6B-32K.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 0.82, "64k": 0, "128k": 0, "dataset_total_score": 0.6366666666666666}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.04666666666666667, "8k": 0.02666666666666667, "16k": 0.006666666666666667, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.013333333333333334}, "passkey_with_librusec": {"4k": 0.99, "8k": 0.995, "16k": 0.985, "32k": 0.93, "64k": 0, "128k": 0, "dataset_total_score": 0.65}, "librusec_history": {"8k": 0.21875, "16k": 0.09375, "32k": 0.03125, "64k": 0, "dataset_total_score": 0.0859375}, "ru_gsm100": {"16k": 0.05, "dataset_total_score": 0.05}, "ru_sci_passage_count": {"4k": 0.09, "8k": 0.06, "16k": 0.07, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.03666666666666667}, "ru_2wikimultihopqa": {"8k": 0.1836734693877551, "16k": 0.21875, "32k": 0.12195121951219512, "dataset_total_score": 0.17479156296665008}, "long_context_multiq": {"8k": 0.05, "16k": 0.015, "4k": 0.005, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.011666666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.4166666666666667, "8k": 0.21887180280037422, "16k": 0.1810637996020305, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.13610037817817858}, "ru_trec": {"4k": 0.05405405405405406, "8k": 0.04, "16k": 0.04395604395604396, "32k": 0.040983606557377046, "dataset_total_score": 0.04474842614186877}, "ru_sci_fi": {"32k": 0.0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.06770833333333333, "dataset_total_score": 0.06770833333333333}, "ru_babilong_qa1": {"4k": 0.27, "8k": 0.23, "16k": 0.23, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.12166666666666666}, "ru_babilong_qa2": {"4k": 0.05, "8k": 0.04, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.015}, "ru_babilong_qa3": {"4k": 0.07, "8k": 0.03, "16k": 0.04, "32k": 0.01, "64k": 0, "128k": 0, "dataset_total_score": 0.025000000000000005}, "ru_babilong_qa4": {"4k": 0.02, "8k": 0.018000000000000002, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.006333333333333334}, "ru_babilong_qa5": {"4k": 0.2, "8k": 0.18, "16k": 0.15, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.08833333333333333}, "ru_quality": {"16k": 0.43892339544513453, "8k": 0.5447154471544716, "dataset_total_score": 0.49181942129980305}, "ru_tpo": {"8k": 0.28950863213811423, "dataset_total_score": 0.28950863213811423}, "ru_qasper": {"16k": 0.03254246612062729, "8k": 0.03544011573751087, "32k": 0.009411737687136327, "dataset_total_score": 0.025798106515091495}, "total_score": 0.15736624130349933}
 
 
results/GLM4-9B-Chat.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.7926421404682275, "8k": 0.75, "16k": 0.7133333333333334, "32k": 0.67, "64k": 0.5966666666666667, "128k": 0.56, "dataset_total_score": 0.6804403567447045}, "matreshka_names": {"4k": 0.6466666666666666, "8k": 0.5066666666666667, "16k": 0.52, "32k": 0.47333333333333333, "64k": 0.37333333333333335, "128k": 0.32, "dataset_total_score": 0.47333333333333333}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 0.84375, "16k": 0.84375, "32k": 0.84375, "64k": 0.75, "dataset_total_score": 0.8203125}, "ru_gsm100": {"16k": 0.08, "dataset_total_score": 0.08}, "ru_sci_passage_count": {"4k": 0.27, "8k": 0.08, "16k": 0.09, "32k": 0.0, "64k": 0.01, "128k": 0.0, "dataset_total_score": 0.07500000000000001}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0.5546875, "32k": 0.35772357723577236, "dataset_total_score": 0.4878104951330125}, "long_context_multiq": {"8k": 0.265, "16k": 0.035, "4k": 0.055, "64k": 0.005, "32k": 0.005, "128k": 0.1, "dataset_total_score": 0.0775}, "ru_sci_abstract_retrieval": {"4k": 0.9819047619047618, "8k": 0.923411865911866, "16k": 0.9122101461259002, "32k": 0.8189266620312142, "64k": 0.6411071734029656, "128k": 0.3908439729202464, "dataset_total_score": 0.7780674303828258}, "ru_trec": {"4k": 0.5675675675675675, "8k": 0.7, "16k": 0.7582417582417582, "32k": 0.7704918032786885, "dataset_total_score": 0.6990752822720037}, "ru_sci_fi": {"32k": 0.3888888888888889, "64k": 0.42857142857142855, "dataset_total_score": 0.4087301587301587}, "librusec_mhqa": {"8k": 0.4453125, "dataset_total_score": 0.4453125}, "ru_babilong_qa1": {"4k": 0.699375, "8k": 0.59, "16k": 0.6, "32k": 0.508125, "64k": 0.429375, "128k": 0.42, "dataset_total_score": 0.5411458333333333}, "ru_babilong_qa2": {"4k": 0.389375, "8k": 0.33, "16k": 0.299375, "32k": 0.2693333333333333, "64k": 0.2675, "128k": 0.23491666666666666, "dataset_total_score": 0.29841666666666666}, "ru_babilong_qa3": {"4k": 0.24598809523809526, "8k": 0.2792380952380953, "16k": 0.21408333333333335, "32k": 0.2264761904761905, "64k": 0.18666666666666668, "128k": 0.18545833333333334, "dataset_total_score": 0.22298511904761908}, "ru_babilong_qa4": {"4k": 0.6207142857142857, "8k": 0.5964285714285714, "16k": 0.5657142857142857, "32k": 0.58, "64k": 0.43, "128k": 0.37714285714285717, "dataset_total_score": 0.5283333333333334}, "ru_babilong_qa5": {"4k": 0.7300000000000001, "8k": 0.7350000000000001, "16k": 0.7200000000000002, "32k": 0.6683333333333334, "64k": 0.6966666666666668, "128k": 0.6700000000000002, "dataset_total_score": 0.7033333333333335}, "ru_quality": {"16k": 0.6521739130434783, "8k": 0.8292682926829268, "dataset_total_score": 0.7407211028632026}, "ru_tpo": {"8k": 0.8685258964143426, "dataset_total_score": 0.8685258964143426}, "ru_qasper": {"16k": 0.05927748156784547, "8k": 0.06532155413695329, "32k": 0.025813608477215297, "dataset_total_score": 0.050137548060671354}, "total_score": 0.5228181376023114}
 
 
results/GPT-4o.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.8, "8k": 0.6, "16k": 1.0, "32k": 0.8, "64k": 0.7, "128k": 0.9, "dataset_total_score": 0.8000000000000002}, "matreshka_names": {"4k": 0.6, "8k": 0.6, "16k": 0.5, "32k": 0.4, "64k": 0.5, "128k": 0.5, "dataset_total_score": 0.5166666666666667}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0.9, "dataset_total_score": 0.975}, "ru_gsm100": {"16k": 1.0, "dataset_total_score": 1.0}, "ru_sci_passage_count": {"4k": 1.0, "8k": 0.5, "16k": 0.3, "32k": 0.0, "64k": 0.2, "128k": 0.1, "dataset_total_score": 0.35000000000000003}, "ru_2wikimultihopqa": {"8k": 0.8, "16k": 0.8, "32k": 0.7, "dataset_total_score": 0.7666666666666666}, "long_context_multiq": {"4k": 0.3, "8k": 1.0, "16k": 0.7, "32k": 0.0, "64k": 0.1, "128k": 0.1, "dataset_total_score": 0.3666666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.99, "8k": 0.9541666666666668, "16k": 0.9254479578392623, "32k": 0.9562564463343153, "64k": 0.590978869808793, "128k": 0.19764315322255238, "dataset_total_score": 0.7690821823119315}, "ru_trec": {"4k": 0.6, "8k": 0.8, "16k": 0.9, "32k": 0.7, "dataset_total_score": 0.75}, "ru_sci_fi": {"32k": 0.6, "64k": 0.9, "dataset_total_score": 0.75}, "librusec_mhqa": {"8k": 0.5, "dataset_total_score": 0.5}, "ru_babilong_qa1": {"4k": 0.9, "8k": 0.8, "16k": 0.7, "32k": 0.9, "64k": 0.8, "128k": 0.6, "dataset_total_score": 0.7833333333333333}, "ru_babilong_qa2": {"4k": 0.4, "8k": 0.3, "16k": 0.4, "32k": 0.4, "64k": 0.5, "128k": 0.2, "dataset_total_score": 0.3666666666666667}, "ru_babilong_qa3": {"4k": 0.2, "8k": 0.3, "16k": 0.1, "32k": 0.2, "64k": 0.2, "128k": 0.2866666666666667, "dataset_total_score": 0.21444444444444444}, "ru_babilong_qa4": {"4k": 0.8800000000000001, "8k": 0.8, "16k": 0.8, "32k": 0.5714285714285715, "64k": 0.8857142857142858, "128k": 0.8, "dataset_total_score": 0.7895238095238096}, "ru_babilong_qa5": {"4k": 0.8666666666666666, "8k": 0.8666666666666666, "16k": 0.9333333333333333, "32k": 0.9666666666666666, "64k": 0.8666666666666668, "128k": 0.9, "dataset_total_score": 0.9}, "ru_quality": {"8k": 0.8, "16k": 0.8666666666666668, "dataset_total_score": 0.8333333333333335}, "ru_tpo": {"8k": 1.0, "dataset_total_score": 1.0}, "ru_qasper": {"8k": 0.2865100250626566, "16k": 0.3184757236227824, "32k": 0.3465384615384616, "dataset_total_score": 0.3171747367413002}, "total_score": 0.70231230982642}
 
 
results/LLaMA-2-7B-32k.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.08, "8k": 0.06666666666666667, "16k": 0.02, "32k": 0.04, "64k": 0, "128k": 0, "dataset_total_score": 0.034444444444444444}, "passkey_with_librusec": {"4k": 1.0, "8k": 0.975, "16k": 0.985, "32k": 0.97, "64k": 0, "128k": 0, "dataset_total_score": 0.6549999999999999}, "librusec_history": {"8k": 0.6875, "16k": 0.5, "32k": 0.4375, "64k": 0, "dataset_total_score": 0.40625}, "ru_gsm100": {"16k": 0.07, "dataset_total_score": 0.07}, "ru_sci_passage_count": {"4k": 0.18, "8k": 0.05, "16k": 0.05, "32k": 0.005, "64k": 0, "128k": 0, "dataset_total_score": 0.047499999999999994}, "ru_2wikimultihopqa": {"8k": 0.4489795918367347, "16k": 0.3984375, "32k": 0.2682926829268293, "dataset_total_score": 0.37190325825452136}, "long_context_multiq": {"8k": 0.33, "16k": 0.1, "4k": 0.045, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.07916666666666666}, "ru_sci_abstract_retrieval": {"4k": 0.8519047619047619, "8k": 0.7612582259010829, "16k": 0.4675868475624726, "32k": 0.2665782908189807, "64k": 0, "128k": 0, "dataset_total_score": 0.39122135436454974}, "ru_trec": {"4k": 0.24324324324324326, "8k": 0.18, "16k": 0.24175824175824176, "32k": 0.28688524590163933, "dataset_total_score": 0.2379716827257811}, "ru_sci_fi": {"32k": 0.1111111111111111, "64k": 0, "dataset_total_score": 0.05555555555555555}, "librusec_mhqa": {"8k": 0.2760416666666667, "dataset_total_score": 0.2760416666666667}, "ru_babilong_qa1": {"4k": 0.6, "8k": 0.66, "16k": 0.66, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.4033333333333333}, "ru_babilong_qa2": {"4k": 0.25, "8k": 0.3, "16k": 0.25875, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.16645833333333335}, "ru_babilong_qa3": {"4k": 0.22933333333333333, "8k": 0.28933333333333333, "16k": 0.26, "32k": 0.2, "64k": 0, "128k": 0, "dataset_total_score": 0.1631111111111111}, "ru_babilong_qa4": {"4k": 0.31, "8k": 0.34, "16k": 0.23, "32k": 0.12, "64k": 0, "128k": 0, "dataset_total_score": 0.16666666666666666}, "ru_babilong_qa5": {"4k": 0.59, "8k": 0.66, "16k": 0.64, "32k": 0.69, "64k": 0, "128k": 0, "dataset_total_score": 0.43}, "ru_quality": {"16k": 0.13871635610766048, "8k": 0.17073170731707318, "dataset_total_score": 0.15472403171236682}, "ru_tpo": {"8k": 0.5431606905710492, "dataset_total_score": 0.5431606905710492}, "ru_qasper": {"16k": 0.05999038960490889, "8k": 0.0580459343880765, "32k": 0.022401950361811175, "dataset_total_score": 0.04681275811826552}, "total_score": 0.2714095362059408}
 
 
results/LLaMA-3-8B-Instruct.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.8394648829431438, "8k": 0.8, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.27324414715719064}, "matreshka_names": {"4k": 0.5333333333333333, "8k": 0.46, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.16555555555555557}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "librusec_history": {"8k": 0.90625, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2265625}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.31, "8k": 0.08, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.065}, "ru_2wikimultihopqa": {"8k": 0.5306122448979592, "16k": 0, "32k": 0, "dataset_total_score": 0.17687074829931973}, "long_context_multiq": {"8k": 0.245, "4k": 0.05, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.049166666666666664}, "ru_sci_abstract_retrieval": {"4k": 0.9663095238095238, "8k": 0.9151886869744013, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.31358303513065416}, "ru_trec": {"4k": 0.5945945945945946, "8k": 0.5, "16k": 0, "32k": 0, "dataset_total_score": 0.2736486486486487}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4609375, "dataset_total_score": 0.4609375}, "ru_babilong_qa1": {"4k": 0.6862083333333334, "8k": 0.7335416666666665, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.236625}, "ru_babilong_qa2": {"4k": 0.14, "8k": 0.10866666666666668, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04144444444444445}, "ru_babilong_qa3": {"4k": 0.09, "8k": 0.1798095238095238, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04496825396825397}, "ru_babilong_qa4": {"4k": 0.5725714285714285, "8k": 0.6034285714285714, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.19599999999999998}, "ru_babilong_qa5": {"4k": 0.7666666666666667, "8k": 0.7516666666666667, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2530555555555556}, "ru_quality": {"8k": 0.6910569105691058, "16k": 0, "dataset_total_score": 0.3455284552845529}, "ru_tpo": {"8k": 0.7808764940239044, "dataset_total_score": 0.7808764940239044}, "ru_qasper": {"8k": 0.06525418503550595, "16k": 0, "32k": 0, "dataset_total_score": 0.021751395011835317}, "total_score": 0.21864214601967852}
 
 
results/LLaMA-3-8B.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.6220735785953178, "8k": 0.59, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2020122630992196}, "matreshka_names": {"4k": 0.4, "8k": 0.2, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.10000000000000002}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "librusec_history": {"8k": 0.90625, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2265625}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.15, "8k": 0.05, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.03333333333333333}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0, "32k": 0, "dataset_total_score": 0.18367346938775508}, "long_context_multiq": {"8k": 0.325, "4k": 0.095, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.07}, "ru_sci_abstract_retrieval": {"4k": 0.9711111111111111, "8k": 0.8806912531912532, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3086337273837274}, "ru_trec": {"4k": 0.3783783783783784, "8k": 0.38, "16k": 0, "32k": 0, "dataset_total_score": 0.1895945945945946}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4140625, "dataset_total_score": 0.4140625}, "ru_babilong_qa1": {"4k": 0.68, "8k": 0.57, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.20833333333333334}, "ru_babilong_qa2": {"4k": 0.27, "8k": 0.19, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.07666666666666667}, "ru_babilong_qa3": {"4k": 0.28470833333333334, "8k": 0.25866666666666666, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.09056249999999999}, "ru_babilong_qa4": {"4k": 0.5838571428571429, "8k": 0.5642857142857143, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.19135714285714286}, "ru_babilong_qa5": {"4k": 0.6716666666666667, "8k": 0.6866666666666668, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2263888888888889}, "ru_quality": {"8k": 0.17073170731707318, "16k": 0, "dataset_total_score": 0.08536585365853659}, "ru_tpo": {"8k": 0.5816733067729084, "dataset_total_score": 0.5816733067729084}, "ru_qasper": {"8k": 0.06526998039125843, "16k": 0, "32k": 0, "dataset_total_score": 0.021756660130419478}, "total_score": 0.18460206698919968}
 
 
results/LLaMA-3.1-8B-Instruct.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.8361204013377926, "8k": 0.7366666666666667, "16k": 0.7566666666666667, "32k": 0.66, "64k": 0.63, "128k": 0.55, "dataset_total_score": 0.6949089557785211}, "matreshka_names": {"4k": 0.56, "8k": 0.52, "16k": 0.38, "32k": 0.38666666666666666, "64k": 0.30666666666666664, "128k": 0.23333333333333334, "dataset_total_score": 0.3977777777777778}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 0.65625, "16k": 0.65625, "32k": 0.625, "64k": 0.65625, "dataset_total_score": 0.6484375}, "ru_gsm100": {"16k": 0.23, "dataset_total_score": 0.23}, "ru_sci_passage_count": {"4k": 0.185, "8k": 0.07, "16k": 0.05, "32k": 0.02, "64k": 0.0, "128k": 0.01, "dataset_total_score": 0.05583333333333334}, "ru_2wikimultihopqa": {"8k": 0.2857142857142857, "16k": 0.328125, "32k": 0.21951219512195122, "dataset_total_score": 0.2777838269454123}, "long_context_multiq": {"8k": 0.27, "16k": 0.035, "4k": 0.065, "64k": 0.005, "32k": 0.0, "128k": 0.1, "dataset_total_score": 0.07916666666666666}, "ru_sci_abstract_retrieval": {"4k": 0.9832142857142858, "8k": 0.9310686932115504, "16k": 0.9261840800423767, "32k": 0.8304544464405875, "64k": 0.6458243484988054, "128k": 0.3264052378026478, "dataset_total_score": 0.7738585152850423}, "ru_trec": {"4k": 0.5675675675675675, "8k": 0.46, "16k": 0.6263736263736264, "32k": 0.5983606557377049, "dataset_total_score": 0.5630754624197247}, "ru_sci_fi": {"32k": 0.2222222222222222, "64k": 0.32142857142857145, "dataset_total_score": 0.27182539682539686}, "librusec_mhqa": {"8k": 0.3177083333333333, "dataset_total_score": 0.3177083333333333}, "ru_babilong_qa1": {"4k": 0.6580833333333334, "8k": 0.635, "16k": 0.593125, "32k": 0.53625, "64k": 0.4474166666666667, "128k": 0.47875, "dataset_total_score": 0.5581041666666667}, "ru_babilong_qa2": {"4k": 0.396875, "8k": 0.246875, "16k": 0.24491666666666667, "32k": 0.2780416666666667, "64k": 0.14616666666666667, "128k": 0.13, "dataset_total_score": 0.24047916666666666}, "ru_babilong_qa3": {"4k": 0.28979166666666667, "8k": 0.3107857142857143, "16k": 0.24823809523809526, "32k": 0.2512261904761905, "64k": 0.16870833333333335, "128k": 0.1666190476190476, "dataset_total_score": 0.23922817460317458}, "ru_babilong_qa4": {"4k": 0.115, "8k": 0.14571428571428574, "16k": 0.14285714285714288, "32k": 0.15642857142857144, "64k": 0.16714285714285715, "128k": 0.11, "dataset_total_score": 0.13952380952380952}, "ru_babilong_qa5": {"4k": 0.6483333333333334, "8k": 0.5883333333333334, "16k": 0.5466666666666667, "32k": 0.5633333333333334, "64k": 0.6166666666666667, "128k": 0.59, "dataset_total_score": 0.5922222222222223}, "ru_quality": {"16k": 0.3954451345755694, "8k": 0.4471544715447155, "dataset_total_score": 0.4212998030601425}, "ru_tpo": {"8k": 0.8154050464807437, "dataset_total_score": 0.8154050464807437}, "ru_qasper": {"16k": 0.07217324214881427, "8k": 0.08332129217427023, "32k": 0.04010014013685399, "dataset_total_score": 0.0651982248199795}, "total_score": 0.4467541134480291}
 
 
results/LLaMA-3.1-8B.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.4080267558528428, "8k": 0.42, "16k": 0.44, "32k": 0.44666666666666666, "64k": 0.4, "128k": 0.28, "dataset_total_score": 0.39911557041991824}, "matreshka_names": {"4k": 0.46, "8k": 0.34, "16k": 0.23333333333333334, "32k": 0.16666666666666666, "64k": 0.08, "128k": 0.06666666666666667, "dataset_total_score": 0.22444444444444447}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 0.96875, "16k": 0.96875, "32k": 0.9375, "64k": 0.9375, "dataset_total_score": 0.953125}, "ru_gsm100": {"16k": 0.2, "dataset_total_score": 0.2}, "ru_sci_passage_count": {"4k": 0.13, "8k": 0.04, "16k": 0.04, "32k": 0.02, "64k": 0.0, "128k": 0.013333333333333332, "dataset_total_score": 0.04055555555555556}, "ru_2wikimultihopqa": {"8k": 0.3673469387755102, "16k": 0.3828125, "32k": 0.25203252032520324, "dataset_total_score": 0.33406398636690443}, "long_context_multiq": {"8k": 0.245, "16k": 0.03, "4k": 0.045, "64k": 0.005, "32k": 0.0, "128k": 0.035, "dataset_total_score": 0.06}, "ru_sci_abstract_retrieval": {"4k": 0.971547619047619, "8k": 0.9226294340580055, "16k": 0.8926773863981885, "32k": 0.737346388706189, "64k": 0.6493593352514232, "128k": 0.2430637216014266, "dataset_total_score": 0.7361039808438087}, "ru_trec": {"4k": 0.5405405405405406, "8k": 0.58, "16k": 0.6263736263736264, "32k": 0.6475409836065574, "dataset_total_score": 0.598613787630181}, "ru_sci_fi": {"32k": 0.027777777777777776, "64k": 0.03571428571428571, "dataset_total_score": 0.031746031746031744}, "librusec_mhqa": {"8k": 0.453125, "dataset_total_score": 0.453125}, "ru_babilong_qa1": {"4k": 0.6274583333333333, "8k": 0.6125, "16k": 0.618125, "32k": 0.514375, "64k": 0.57625, "128k": 0.2874583333333333, "dataset_total_score": 0.5393611111111111}, "ru_babilong_qa2": {"4k": 0.4447916666666667, "8k": 0.3418333333333334, "16k": 0.181125, "32k": 0.21991666666666668, "64k": 0.2025, "128k": 0.134375, "dataset_total_score": 0.2540902777777778}, "ru_babilong_qa3": {"4k": 0.3370833333333333, "8k": 0.34199999999999997, "16k": 0.34662499999999996, "32k": 0.26304166666666673, "64k": 0.2912916666666667, "128k": 0.19883333333333336, "dataset_total_score": 0.2964791666666667}, "ru_babilong_qa4": {"4k": 0.6778571428571429, "8k": 0.6585714285714286, "16k": 0.6308571428571429, "32k": 0.5071428571428572, "64k": 0.43071428571428577, "128k": 0.21857142857142858, "dataset_total_score": 0.5206190476190476}, "ru_babilong_qa5": {"4k": 0.695, "8k": 0.7083333333333335, "16k": 0.6783333333333332, "32k": 0.6916666666666668, "64k": 0.6916666666666668, "128k": 0.6100000000000001, "dataset_total_score": 0.6791666666666667}, "ru_quality": {"16k": 0.028985507246376812, "8k": 0.21138211382113825, "dataset_total_score": 0.12018381053375753}, "ru_tpo": {"8k": 0.4980079681274901, "dataset_total_score": 0.4980079681274901}, "ru_qasper": {"16k": 0.05029792951580615, "8k": 0.05657815571812267, "32k": 0.022286589880109883, "dataset_total_score": 0.04305422503801291}, "total_score": 0.4277074109784464}
 
 
results/LiquidAI_LFM2.5-1.2B-Instruct.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.2860481481481481,
3
+ "passkey": {
4
+ "dataset_total_score": 0.6666666666666666,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 0.0,
10
+ "128k": 0.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.38833333333333336,
14
+ "4k": 0.457,
15
+ "8k": 0.45,
16
+ "16k": 0.49,
17
+ "32k": 0.303,
18
+ "64k": 0.559,
19
+ "128k": 0.071
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.06720000000000001,
23
+ "4k": 0.187,
24
+ "8k": 0.087,
25
+ "16k": 0.062,
26
+ "32k": 0.0,
27
+ "64k": 0.0
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 0.6608333333333333,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 0.995,
34
+ "32k": 0.97,
35
+ "64k": 0.0,
36
+ "128k": 0.0
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.297,
40
+ "8k": 0.5,
41
+ "16k": 0.344,
42
+ "32k": 0.219,
43
+ "64k": 0.125
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.029333333333333336,
47
+ "4k": 0.048,
48
+ "8k": 0.051,
49
+ "16k": 0.02,
50
+ "32k": 0.046,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.36199999999999993,
56
+ "8k": 0.408,
57
+ "16k": 0.391,
58
+ "32k": 0.287
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.17666666666666667,
62
+ "4k": 0.418,
63
+ "8k": 0.364,
64
+ "16k": 0.241,
65
+ "32k": 0.037,
66
+ "64k": 0.0,
67
+ "128k": 0.0
68
+ },
69
+ "ru_sci_abstract_retrieval": {
70
+ "dataset_total_score": 0.2318333333333333,
71
+ "4k": 0.603,
72
+ "8k": 0.433,
73
+ "16k": 0.238,
74
+ "32k": 0.107,
75
+ "64k": 0.005,
76
+ "128k": 0.005
77
+ },
78
+ "ru_sci_fi": {
79
+ "dataset_total_score": 0.1095,
80
+ "32k": 0.088,
81
+ "64k": 0.131
82
+ },
83
+ "librusec_mhqa": {
84
+ "dataset_total_score": 0.227,
85
+ "8k": 0.227
86
+ },
87
+ "ru_babilong_qa1": {
88
+ "dataset_total_score": 0.0695,
89
+ "4k": 0.162,
90
+ "8k": 0.152,
91
+ "16k": 0.071,
92
+ "32k": 0.032,
93
+ "64k": 0.0,
94
+ "128k": 0.0
95
+ },
96
+ "ru_babilong_qa2": {
97
+ "dataset_total_score": 0.024999999999999998,
98
+ "4k": 0.071,
99
+ "8k": 0.026,
100
+ "16k": 0.026,
101
+ "32k": 0.027,
102
+ "64k": 0.0,
103
+ "128k": 0.0
104
+ },
105
+ "ru_babilong_qa3": {
106
+ "dataset_total_score": 0.047999999999999994,
107
+ "4k": 0.067,
108
+ "8k": 0.088,
109
+ "16k": 0.087,
110
+ "32k": 0.046,
111
+ "64k": 0.0,
112
+ "128k": 0.0
113
+ },
114
+ "ru_babilong_qa4": {
115
+ "dataset_total_score": 0.07516666666666667,
116
+ "4k": 0.333,
117
+ "8k": 0.082,
118
+ "16k": 0.036,
119
+ "32k": 0.0,
120
+ "64k": 0.0,
121
+ "128k": 0.0
122
+ },
123
+ "ru_babilong_qa5": {
124
+ "dataset_total_score": 0.29683333333333334,
125
+ "4k": 0.596,
126
+ "8k": 0.485,
127
+ "16k": 0.398,
128
+ "32k": 0.302,
129
+ "64k": 0.0,
130
+ "128k": 0.0
131
+ },
132
+ "ru_quality": {
133
+ "dataset_total_score": 0.714,
134
+ "8k": 0.667,
135
+ "16k": 0.761
136
+ },
137
+ "ru_tpo": {
138
+ "dataset_total_score": 0.704,
139
+ "8k": 0.704
140
+ }
141
+ }
results/LongAlpaca.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 0.775, "8k": 0.825, "16k": 0.575, "32k": 0.37, "64k": 0, "128k": 0, "dataset_total_score": 0.42416666666666664}, "matreshka_yes_no": {"4k": 0.4782608695652174, "8k": 0.3933333333333333, "16k": 0.48, "32k": 0.4766666666666667, "64k": 0, "128k": 0, "dataset_total_score": 0.30471014492753623}, "matreshka_names": {"4k": 0.013333333333333334, "8k": 0.006666666666666667, "16k": 0.006666666666666667, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0044444444444444444}, "passkey_with_librusec": {"4k": 0.71, "8k": 0.7, "16k": 0.56, "32k": 0.465, "64k": 0, "128k": 0, "dataset_total_score": 0.4058333333333333}, "librusec_history": {"8k": 0.1875, "16k": 0.15625, "32k": 0.1875, "64k": 0, "dataset_total_score": 0.1328125}, "ru_gsm100": {"16k": 0.02, "dataset_total_score": 0.02}, "ru_sci_passage_count": {"4k": 0.13, "8k": 0.05, "16k": 0.02, "32k": 0.03, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333333}, "ru_2wikimultihopqa": {"8k": 0.40816326530612246, "16k": 0.2890625, "32k": 0.21138211382113822, "dataset_total_score": 0.3028692930424202}, "long_context_multiq": {"8k": 0.015, "16k": 0.0, "4k": 0.03, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0075}, "ru_sci_abstract_retrieval": {"4k": 0.6496428571428572, "8k": 0.44704965669251384, "16k": 0.2043098561917367, "32k": 0.11199263409419845, "64k": 0, "128k": 0, "dataset_total_score": 0.23549916735355103}, "ru_trec": {"4k": 0.0, "8k": 0.02, "16k": 0.0, "32k": 0.0, "dataset_total_score": 0.005}, "ru_sci_fi": {"32k": 0.027777777777777776, "64k": 0, "dataset_total_score": 0.013888888888888888}, "librusec_mhqa": {"8k": 0.078125, "dataset_total_score": 0.078125}, "ru_babilong_qa1": {"4k": 0.09, "8k": 0.06, "16k": 0.06, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333333}, "ru_babilong_qa2": {"4k": 0.01, "8k": 0.01, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0033333333333333335}, "ru_babilong_qa3": {"4k": 0.05, "8k": 0.09, "16k": 0.04, "32k": 0.029333333333333336, "64k": 0, "128k": 0, "dataset_total_score": 0.03488888888888889}, "ru_babilong_qa4": {"4k": 0.0, "8k": 0.01, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0016666666666666668}, "ru_babilong_qa5": {"4k": 0.4416666666666667, "8k": 0.44, "16k": 0.475, "32k": 0.40666666666666673, "64k": 0, "128k": 0, "dataset_total_score": 0.2938888888888889}, "ru_quality": {"16k": 0.4824016563146999, "8k": 0.39837398373983746, "dataset_total_score": 0.4403878200272687}, "ru_tpo": {"8k": 0.06772908366533864, "dataset_total_score": 0.06772908366533864}, "ru_qasper": {"16k": 0.02179956634280568, "8k": 0.02260857442083522, "32k": 0.015666306859159988, "dataset_total_score": 0.020024815874266962}, "total_score": 0.13683026679372187}
 
 
results/LongChat-7B-v1.5-32k.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 0.995, "8k": 1.0, "16k": 1.0, "32k": 0.995, "64k": 0, "128k": 0, "dataset_total_score": 0.665}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.17333333333333334, "8k": 0.06666666666666667, "16k": 0.08, "32k": 0.03333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.058888888888888886}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0.985, "32k": 0.975, "64k": 0, "128k": 0, "dataset_total_score": 0.66}, "librusec_history": {"8k": 0.5625, "16k": 0.34375, "32k": 0.15625, "64k": 0, "dataset_total_score": 0.265625}, "ru_gsm100": {"16k": 0.05, "dataset_total_score": 0.05}, "ru_sci_passage_count": {"4k": 0.18, "8k": 0.08, "16k": 0.01, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333334}, "ru_2wikimultihopqa": {"8k": 0.42857142857142855, "16k": 0.3984375, "32k": 0.22764227642276422, "dataset_total_score": 0.3515504016647309}, "long_context_multiq": {"8k": 0.14, "16k": 0.025, "4k": 0.025, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.03166666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.8742063492063491, "8k": 0.7618964368964368, "16k": 0.6055101098106439, "32k": 0.22205033016504383, "64k": 0, "128k": 0, "dataset_total_score": 0.4106105376797456}, "ru_trec": {"4k": 0.05405405405405406, "8k": 0.1, "16k": 0.07692307692307693, "32k": 0.06557377049180328, "dataset_total_score": 0.07413772536723356}, "ru_sci_fi": {"32k": 0.05555555555555555, "64k": 0, "dataset_total_score": 0.027777777777777776}, "librusec_mhqa": {"8k": 0.24739583333333334, "dataset_total_score": 0.24739583333333334}, "ru_babilong_qa1": {"4k": 0.26, "8k": 0.29, "16k": 0.31, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.17500000000000002}, "ru_babilong_qa2": {"4k": 0.11, "8k": 0.08, "16k": 0.16, "32k": 0.08, "64k": 0, "128k": 0, "dataset_total_score": 0.07166666666666667}, "ru_babilong_qa3": {"4k": 0.09, "8k": 0.05, "16k": 0.04, "32k": 0.06, "64k": 0, "128k": 0, "dataset_total_score": 0.04}, "ru_babilong_qa4": {"4k": 0.25214285714285717, "8k": 0.2921428571428571, "16k": 0.15642857142857142, "32k": 0.05928571428571429, "64k": 0, "128k": 0, "dataset_total_score": 0.12666666666666665}, "ru_babilong_qa5": {"4k": 0.5133333333333333, "8k": 0.5, "16k": 0.48333333333333334, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.3327777777777778}, "ru_quality": {"16k": 0.1780538302277433, "8k": 0.28455284552845533, "dataset_total_score": 0.2313033378780993}, "ru_tpo": {"8k": 0.3957503320053121, "dataset_total_score": 0.3957503320053121}, "ru_qasper": {"16k": 0.06475047107877817, "8k": 0.060803436943138944, "32k": 0.023538832049622788, "dataset_total_score": 0.04969758002384664}, "total_score": 0.22130764599351707}
 
 
results/Mistral-7B-Instruct-v0.3.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.5652173913043478, "8k": 0.5066666666666667, "16k": 0.5466666666666666, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.35309178743961356}, "matreshka_names": {"4k": 0.38, "8k": 0.32, "16k": 0.16666666666666666, "32k": 0.11333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.1633333333333333}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 0.995, "64k": 0, "128k": 0, "dataset_total_score": 0.6658333333333334}, "librusec_history": {"8k": 0.71875, "16k": 0.625, "32k": 0.6875, "64k": 0, "dataset_total_score": 0.5078125}, "ru_gsm100": {"16k": 0.11, "dataset_total_score": 0.11}, "ru_sci_passage_count": {"4k": 0.26, "8k": 0.14, "16k": 0.07, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.08166666666666668}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0.46875, "32k": 0.2764227642276423, "dataset_total_score": 0.4320643907969692}, "long_context_multiq": {"8k": 0.22, "16k": 0.035, "4k": 0.035, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333334}, "ru_sci_abstract_retrieval": {"4k": 0.9824603174603175, "8k": 0.8690578865578865, "16k": 0.7111051699917106, "32k": 0.05148203921970241, "64k": 0, "128k": 0, "dataset_total_score": 0.43568423553826957}, "ru_trec": {"4k": 0.5675675675675675, "8k": 0.38, "16k": 0.4065934065934066, "32k": 0.3442622950819672, "dataset_total_score": 0.4246058173107353}, "ru_sci_fi": {"32k": 0.3055555555555556, "64k": 0, "dataset_total_score": 0.1527777777777778}, "librusec_mhqa": {"8k": 0.3359375, "dataset_total_score": 0.3359375}, "ru_babilong_qa1": {"4k": 0.25, "8k": 0.15, "16k": 0.22, "32k": 0.24, "64k": 0, "128k": 0, "dataset_total_score": 0.14333333333333334}, "ru_babilong_qa2": {"4k": 0.08, "8k": 0.05, "16k": 0.02, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.028333333333333332}, "ru_babilong_qa3": {"4k": 0.1, "8k": 0.08, "16k": 0.1, "32k": 0.08, "64k": 0, "128k": 0, "dataset_total_score": 0.060000000000000005}, "ru_babilong_qa4": {"4k": 0.5178571428571428, "8k": 0.4428571428571429, "16k": 0.39285714285714285, "32k": 0.3028571428571429, "64k": 0, "128k": 0, "dataset_total_score": 0.2760714285714286}, "ru_babilong_qa5": {"4k": 0.5466666666666667, "8k": 0.62, "16k": 0.5533333333333333, "32k": 0.5333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.3755555555555556}, "ru_quality": {"16k": 0.22981366459627334, "8k": 0.3821138211382114, "dataset_total_score": 0.30596374286724237}, "ru_tpo": {"8k": 0.6640106241699868, "dataset_total_score": 0.6640106241699868}, "ru_qasper": {"16k": 0.06576022201401649, "8k": 0.06619134922698901, "32k": 0.02936483912182809, "dataset_total_score": 0.05377213678761119}, "total_score": 0.2992784522292948}
 
 
results/Mistral-7B-v0.1.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 0.975, "16k": 0.125, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.35000000000000003}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.0033333333333333335, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.1675009290226682}, "matreshka_names": {"4k": 0.32666666666666666, "8k": 0.16, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.08111111111111112}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0.3, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.3833333333333333}, "librusec_history": {"8k": 0.78125, "16k": 0.15625, "32k": 0.0, "64k": 0, "dataset_total_score": 0.234375}, "ru_gsm100": {"16k": 0.13, "dataset_total_score": 0.13}, "ru_sci_passage_count": {"4k": 0.04, "8k": 0.04, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.013333333333333334}, "ru_2wikimultihopqa": {"8k": 0.42857142857142855, "16k": 0.1796875, "32k": 0.08130081300813008, "dataset_total_score": 0.22985324719318623}, "long_context_multiq": {"8k": 0.22, "16k": 0.005, "4k": 0.04, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.04416666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.9484126984126985, "8k": 0.7607802118516404, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.28486548504405645}, "ru_trec": {"4k": 0.02702702702702703, "8k": 0.1, "16k": 0.03296703296703297, "32k": 0.0, "dataset_total_score": 0.039998514998515}, "ru_sci_fi": {"32k": 0.027777777777777776, "64k": 0, "dataset_total_score": 0.013888888888888888}, "librusec_mhqa": {"8k": 0.3411458333333333, "dataset_total_score": 0.3411458333333333}, "ru_babilong_qa1": {"4k": 0.63, "8k": 0.63, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.21}, "ru_babilong_qa2": {"4k": 0.21, "8k": 0.25, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.07666666666666666}, "ru_babilong_qa3": {"4k": 0.29, "8k": 0.25, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.09000000000000001}, "ru_babilong_qa4": {"4k": 0.4292857142857143, "8k": 0.3157142857142857, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.12416666666666669}, "ru_babilong_qa5": {"4k": 0.7, "8k": 0.6933333333333335, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.23222222222222225}, "ru_quality": {"16k": 0.11801242236024845, "8k": 0.22764227642276424, "dataset_total_score": 0.17282734939150635}, "ru_tpo": {"8k": 0.3957503320053121, "dataset_total_score": 0.3957503320053121}, "ru_qasper": {"16k": 0.011042882576489372, "8k": 0.0625419691096683, "32k": 0.0008322260797508323, "dataset_total_score": 0.024805692588636172}, "total_score": 0.17333387011743345}
 
 
results/Mistral-7B-v0.3.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.44816053511705684, "8k": 0.47, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.31969342251950944}, "matreshka_names": {"4k": 0.2866666666666667, "8k": 0.16, "16k": 0.10666666666666667, "32k": 0.04666666666666667, "64k": 0, "128k": 0, "dataset_total_score": 0.09999999999999999}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "librusec_history": {"8k": 0.9375, "16k": 0.9375, "32k": 0.84375, "64k": 0, "dataset_total_score": 0.6796875}, "ru_gsm100": {"16k": 0.09, "dataset_total_score": 0.09}, "ru_sci_passage_count": {"4k": 0.0, "8k": 0.0, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0}, "ru_2wikimultihopqa": {"8k": 0.46938775510204084, "16k": 0.4921875, "32k": 0.2682926829268293, "dataset_total_score": 0.40995597934295674}, "long_context_multiq": {"8k": 0.24, "16k": 0.035, "4k": 0.04, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.05249999999999999}, "ru_sci_abstract_retrieval": {"4k": 0.8735714285714286, "8k": 0.5658155733155733, "16k": 0.3690303371278, "32k": 0.01873920061760887, "64k": 0, "128k": 0, "dataset_total_score": 0.30452608993873514}, "ru_trec": {"4k": 0.0, "8k": 0.08, "16k": 0.04395604395604396, "32k": 0.09016393442622951, "dataset_total_score": 0.05352999459556837}, "ru_sci_fi": {"32k": 0.0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.390625, "dataset_total_score": 0.390625}, "ru_babilong_qa1": {"4k": 0.6, "8k": 0.63, "16k": 0.58, "32k": 0.43, "64k": 0, "128k": 0, "dataset_total_score": 0.37333333333333335}, "ru_babilong_qa2": {"4k": 0.35, "8k": 0.23, "16k": 0.18, "32k": 0.24, "64k": 0, "128k": 0, "dataset_total_score": 0.16666666666666666}, "ru_babilong_qa3": {"4k": 0.29, "8k": 0.23, "16k": 0.23, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.15666666666666665}, "ru_babilong_qa4": {"4k": 0.4628571428571429, "8k": 0.3442857142857143, "16k": 0.36214285714285716, "32k": 0.24857142857142858, "64k": 0, "128k": 0, "dataset_total_score": 0.23630952380952383}, "ru_babilong_qa5": {"4k": 0.7033333333333335, "8k": 0.6866666666666668, "16k": 0.7533333333333334, "32k": 0.6833333333333335, "64k": 0, "128k": 0, "dataset_total_score": 0.4711111111111112}, "ru_quality": {"16k": 0.06832298136645963, "8k": 0.23577235772357727, "dataset_total_score": 0.15204766954501844}, "ru_tpo": {"8k": 0.3970783532536521, "dataset_total_score": 0.3970783532536521}, "ru_qasper": {"16k": 0.0648885639017482, "8k": 0.08898968989798027, "32k": 0.01870864419884028, "dataset_total_score": 0.05752896599952292}, "total_score": 0.27355207667217124}
 
 
results/Mistral-Nemo-Instruct-2407.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 0.87, "dataset_total_score": 0.9783333333333334}, "matreshka_yes_no": {"4k": 0.6555183946488294, "8k": 0.5366666666666666, "16k": 0.5, "32k": 0.5, "64k": 0.5, "128k": 0.5, "dataset_total_score": 0.5320308435525827}, "matreshka_names": {"4k": 0.6133333333333333, "8k": 0.5, "16k": 0.43333333333333335, "32k": 0.29333333333333333, "64k": 0.07333333333333333, "128k": 0.02, "dataset_total_score": 0.3222222222222222}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 0.995, "64k": 1.0, "128k": 0.97, "dataset_total_score": 0.9941666666666666}, "librusec_history": {"8k": 0.59375, "16k": 0.5625, "32k": 0.5, "64k": 0.46875, "dataset_total_score": 0.53125}, "ru_gsm100": {"16k": 0.0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.57, "8k": 0.1, "16k": 0.05, "32k": 0.03, "64k": 0.0, "128k": 0.02, "dataset_total_score": 0.12833333333333333}, "ru_2wikimultihopqa": {"8k": 0.3469387755102041, "16k": 0.3125, "32k": 0.17886178861788618, "dataset_total_score": 0.27943352137603006}, "long_context_multiq": {"8k": 0.23, "16k": 0.025, "4k": 0.05, "64k": 0.005, "32k": 0.0, "128k": 0.0, "dataset_total_score": 0.05166666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.9843253968253969, "8k": 0.9340110286538859, "16k": 0.882861713382518, "32k": 0.7548755510867896, "64k": 0.28580245595851705, "128k": 0.07901305389621614, "dataset_total_score": 0.6534815333005539}, "ru_trec": {"4k": 0.4864864864864865, "8k": 0.66, "16k": 0.7142857142857143, "32k": 0.47540983606557374, "dataset_total_score": 0.5840455092094436}, "ru_sci_fi": {"32k": 0.2222222222222222, "64k": 0.25, "dataset_total_score": 0.2361111111111111}, "librusec_mhqa": {"8k": 0.2994791666666667, "dataset_total_score": 0.2994791666666667}, "ru_babilong_qa1": {"4k": 0.71, "8k": 0.73, "16k": 0.59, "32k": 0.6, "64k": 0.46, "128k": 0.19, "dataset_total_score": 0.5466666666666666}, "ru_babilong_qa2": {"4k": 0.28, "8k": 0.26, "16k": 0.19, "32k": 0.16, "64k": 0.11, "128k": 0.04, "dataset_total_score": 0.17333333333333334}, "ru_babilong_qa3": {"4k": 0.22, "8k": 0.28, "16k": 0.14, "32k": 0.14, "64k": 0.11, "128k": 0.07, "dataset_total_score": 0.16}, "ru_babilong_qa4": {"4k": 0.16928571428571432, "8k": 0.18714285714285717, "16k": 0.20142857142857143, "32k": 0.1635714285714286, "64k": 0.02, "128k": 0.0, "dataset_total_score": 0.1235714285714286}, "ru_babilong_qa5": {"4k": 0.48, "8k": 0.5, "16k": 0.49, "32k": 0.4566666666666667, "64k": 0.43, "128k": 0.3966666666666667, "dataset_total_score": 0.45888888888888896}, "ru_quality": {"16k": 0.6811594202898551, "8k": 0.6585365853658537, "dataset_total_score": 0.6698480028278544}, "ru_tpo": {"8k": 0.7675962815405047, "dataset_total_score": 0.7675962815405047}, "ru_qasper": {"16k": 0.28639727595549347, "8k": 0.3018572041274937, "32k": 0.14594086192630237, "dataset_total_score": 0.2447317806697632}, "total_score": 0.41596144237795474}
 
 
results/Phi-3-mini-128k-instruct.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 0.1, "8k": 0.995, "16k": 1.0, "32k": 1.0, "64k": 0.99, "128k": 0.995, "dataset_total_score": 0.8466666666666667}, "matreshka_yes_no": {"4k": 0.8595317725752508, "8k": 0.6833333333333333, "16k": 0.74, "32k": 0.68, "64k": 0.6566666666666666, "128k": 0.62, "dataset_total_score": 0.7065886287625419}, "matreshka_names": {"4k": 0.30666666666666664, "8k": 0.21333333333333335, "16k": 0.19333333333333333, "32k": 0.18, "64k": 0.16, "128k": 0.07333333333333333, "dataset_total_score": 0.18777777777777774}, "passkey_with_librusec": {"4k": 0.235, "8k": 0.995, "16k": 1.0, "32k": 1.0, "64k": 0.97, "128k": 0.93, "dataset_total_score": 0.855}, "librusec_history": {"8k": 0.46875, "16k": 0.4375, "32k": 0.375, "64k": 0.375, "dataset_total_score": 0.4140625}, "ru_gsm100": {"16k": 0.24, "dataset_total_score": 0.24}, "ru_sci_passage_count": {"4k": 0.21, "8k": 0.08, "16k": 0.05, "32k": 0.02, "64k": 0.0, "128k": 0.01, "dataset_total_score": 0.06166666666666667}, "ru_2wikimultihopqa": {"8k": 0.1836734693877551, "16k": 0.203125, "32k": 0.17886178861788618, "dataset_total_score": 0.18855341933521375}, "long_context_multiq": {"8k": 0.225, "16k": 0.035, "4k": 0.035, "64k": 0.005, "32k": 0.0, "128k": 0.01, "dataset_total_score": 0.05166666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.16666666666666666, "8k": 0.45602797995655137, "16k": 0.406087639083534, "32k": 0.4008502386409311, "64k": 0.23707393032062402, "128k": 0.0892775698342537, "dataset_total_score": 0.29266400408376014}, "ru_trec": {"4k": 0.08108108108108109, "8k": 0.06, "16k": 0.24175824175824176, "32k": 0.30327868852459017, "dataset_total_score": 0.17152950284097826}, "ru_sci_fi": {"32k": 0.0, "64k": 0.0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.13802083333333334, "dataset_total_score": 0.13802083333333334}, "ru_babilong_qa1": {"4k": 0.14, "8k": 0.44, "16k": 0.42, "32k": 0.35, "64k": 0.33, "128k": 0.15, "dataset_total_score": 0.305}, "ru_babilong_qa2": {"4k": 0.01, "8k": 0.12, "16k": 0.18, "32k": 0.08, "64k": 0.08, "128k": 0.06, "dataset_total_score": 0.08833333333333333}, "ru_babilong_qa3": {"4k": 0.05, "8k": 0.14, "16k": 0.07, "32k": 0.13, "64k": 0.1, "128k": 0.05, "dataset_total_score": 0.09000000000000001}, "ru_babilong_qa4": {"4k": 0.0, "8k": 0.05, "16k": 0.01, "32k": 0.0, "64k": 0.0, "128k": 0.0, "dataset_total_score": 0.01}, "ru_babilong_qa5": {"4k": 0.10666666666666666, "8k": 0.4866666666666667, "16k": 0.5000000000000001, "32k": 0.55, "64k": 0.5566666666666666, "128k": 0.44666666666666666, "dataset_total_score": 0.4411111111111112}, "ru_quality": {"16k": 0.33747412008281574, "8k": 0.43902439024390244, "dataset_total_score": 0.38824925516335906}, "ru_tpo": {"8k": 0.40903054448871184, "dataset_total_score": 0.40903054448871184}, "ru_qasper": {"16k": 0.043349249250495984, "8k": 0.04746257838212318, "32k": 0.01566630824976973, "dataset_total_score": 0.03549271196079629}, "total_score": 0.2819720772471865}
 
 
results/Qwen_Qwen3-30B-A3B-Instruct-2507.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.7126490740740741,
3
+ "passkey": {
4
+ "dataset_total_score": 1.0,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 1.0,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.551,
14
+ "4k": 0.557,
15
+ "8k": 0.523,
16
+ "16k": 0.55,
17
+ "32k": 0.543,
18
+ "64k": 0.562,
19
+ "128k": 0.571
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.8119999999999999,
23
+ "4k": 0.827,
24
+ "8k": 0.833,
25
+ "16k": 0.793,
26
+ "32k": 0.807,
27
+ "64k": 0.8
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 1.0,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 1.0,
34
+ "32k": 1.0,
35
+ "64k": 1.0,
36
+ "128k": 1.0
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.9612499999999999,
40
+ "8k": 0.969,
41
+ "16k": 0.969,
42
+ "32k": 0.969,
43
+ "64k": 0.938
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.25766666666666665,
47
+ "4k": 0.692,
48
+ "8k": 0.434,
49
+ "16k": 0.23,
50
+ "32k": 0.069,
51
+ "64k": 0.088,
52
+ "128k": 0.033
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.6663333333333333,
56
+ "8k": 0.68,
57
+ "16k": 0.669,
58
+ "32k": 0.65
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.4596,
62
+ "4k": 0.519,
63
+ "8k": 0.496,
64
+ "16k": 0.458,
65
+ "32k": 0.459,
66
+ "64k": 0.366
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.815,
70
+ "4k": 0.99,
71
+ "8k": 0.957,
72
+ "16k": 0.919,
73
+ "32k": 0.83,
74
+ "64k": 0.719,
75
+ "128k": 0.475
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.606,
79
+ "32k": 0.62,
80
+ "64k": 0.592
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.51,
84
+ "8k": 0.51
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.805,
88
+ "4k": 0.889,
89
+ "8k": 0.869,
90
+ "16k": 0.818,
91
+ "32k": 0.755,
92
+ "64k": 0.802,
93
+ "128k": 0.697
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.6471666666666667,
97
+ "4k": 0.682,
98
+ "8k": 0.763,
99
+ "16k": 0.753,
100
+ "32k": 0.726,
101
+ "64k": 0.585,
102
+ "128k": 0.374
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.3838333333333334,
106
+ "4k": 0.45,
107
+ "8k": 0.412,
108
+ "16k": 0.406,
109
+ "32k": 0.477,
110
+ "64k": 0.338,
111
+ "128k": 0.22
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.7098333333333332,
115
+ "4k": 0.782,
116
+ "8k": 0.718,
117
+ "16k": 0.759,
118
+ "32k": 0.724,
119
+ "64k": 0.72,
120
+ "128k": 0.556
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.8295,
124
+ "4k": 0.879,
125
+ "8k": 0.838,
126
+ "16k": 0.837,
127
+ "32k": 0.833,
128
+ "64k": 0.812,
129
+ "128k": 0.778
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.8535,
133
+ "8k": 0.778,
134
+ "16k": 0.929
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.96,
138
+ "8k": 0.96
139
+ }
140
+ }
results/Saiga-LLaMA-3-8B.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.8729096989966555, "8k": 0.81, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.28048494983277594}, "matreshka_names": {"4k": 0.5333333333333333, "8k": 0.4, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.15555555555555556}, "passkey_with_librusec": {"4k": 1.0, "8k": 0.995, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3325}, "librusec_history": {"8k": 0.96875, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2421875}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.195, "8k": 0.035, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333334}, "ru_2wikimultihopqa": {"8k": 0.5306122448979592, "16k": 0, "32k": 0, "dataset_total_score": 0.17687074829931973}, "long_context_multiq": {"8k": 0.235, "4k": 0.055, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333333}, "ru_sci_abstract_retrieval": {"4k": 0.9772222222222221, "8k": 0.9258564054992626, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.31717977128691416}, "ru_trec": {"4k": 0.5135135135135135, "8k": 0.54, "16k": 0, "32k": 0, "dataset_total_score": 0.2633783783783784}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4505208333333333, "dataset_total_score": 0.4505208333333333}, "ru_babilong_qa1": {"4k": 0.7629583333333333, "8k": 0.758125, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.25351388888888887}, "ru_babilong_qa2": {"4k": 0.195625, "8k": 0.06875, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.044062500000000004}, "ru_babilong_qa3": {"4k": 0.14733333333333334, "8k": 0.21585714285714286, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.06053174603174603}, "ru_babilong_qa4": {"4k": 0.6347142857142858, "8k": 0.5821428571428572, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.20280952380952386}, "ru_babilong_qa5": {"4k": 0.7466666666666667, "8k": 0.7633333333333334, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2516666666666667}, "ru_quality": {"8k": 0.35772357723577236, "16k": 0, "dataset_total_score": 0.17886178861788618}, "ru_tpo": {"8k": 0.7569721115537849, "dataset_total_score": 0.7569721115537849}, "ru_qasper": {"8k": 0.07413702213069599, "16k": 0, "32k": 0, "dataset_total_score": 0.024712340710231998}, "total_score": 0.21008610966500027}
 
 
results/T-lite-instruct-0.1.json DELETED
@@ -1 +0,0 @@
1
- {"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.8294314381270903, "8k": 0.7133333333333334, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2571274619100706}, "matreshka_names": {"4k": 0.49333333333333335, "8k": 0.3466666666666667, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.14}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "librusec_history": {"8k": 0.90625, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2265625}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.24, "8k": 0.065, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.050833333333333335}, "ru_2wikimultihopqa": {"8k": 0.3877551020408163, "16k": 0, "32k": 0, "dataset_total_score": 0.1292517006802721}, "long_context_multiq": {"8k": 0.245, "4k": 0.065, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.051666666666666666}, "ru_sci_abstract_retrieval": {"4k": 0.9474376417233561, "8k": 0.9261452832881405, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.31226382083524945}, "ru_trec": {"4k": 0.6486486486486487, "8k": 0.58, "16k": 0, "32k": 0, "dataset_total_score": 0.3071621621621622}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.484375, "dataset_total_score": 0.484375}, "ru_babilong_qa1": {"4k": 0.68, "8k": 0.62, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.21666666666666667}, "ru_babilong_qa2": {"4k": 0.52, "8k": 0.35, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.145}, "ru_babilong_qa3": {"4k": 0.22620833333333334, "8k": 0.26740476190476187, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.0822688492063492}, "ru_babilong_qa4": {"4k": 0.6242857142857143, "8k": 0.7128571428571427, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.22285714285714286}, "ru_babilong_qa5": {"4k": 0.7433333333333334, "8k": 0.7233333333333334, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.24444444444444446}, "ru_quality": {"8k": 0.21951219512195122, "16k": 0, "dataset_total_score": 0.10975609756097561}, "ru_tpo": {"8k": 0.7503320053120849, "dataset_total_score": 0.7503320053120849}, "ru_qasper": {"8k": 0.08197389867894705, "16k": 0, "32k": 0, "dataset_total_score": 0.027324632892982353}, "total_score": 0.21069329291405084}
 
 
results/gradientai_Llama-3-8B-Instruct-Gradient-1048k.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.45613703703703706,
3
+ "passkey": {
4
+ "dataset_total_score": 1.0,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 1.0,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.5441666666666667,
14
+ "4k": 0.493,
15
+ "8k": 0.487,
16
+ "16k": 0.567,
17
+ "32k": 0.553,
18
+ "64k": 0.576,
19
+ "128k": 0.589
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.14140000000000003,
23
+ "4k": 0.26,
24
+ "8k": 0.193,
25
+ "16k": 0.103,
26
+ "32k": 0.107,
27
+ "64k": 0.044
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 0.9966666666666667,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 1.0,
34
+ "32k": 0.995,
35
+ "64k": 0.995,
36
+ "128k": 0.99
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.6015,
40
+ "8k": 0.688,
41
+ "16k": 0.625,
42
+ "32k": 0.562,
43
+ "64k": 0.531
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.056,
47
+ "4k": 0.183,
48
+ "8k": 0.051,
49
+ "16k": 0.08,
50
+ "32k": 0.011,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.408,
56
+ "8k": 0.497,
57
+ "16k": 0.391,
58
+ "32k": 0.336
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.3794,
62
+ "4k": 0.5,
63
+ "8k": 0.455,
64
+ "16k": 0.313,
65
+ "32k": 0.312,
66
+ "64k": 0.317
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.7530000000000001,
70
+ "4k": 0.957,
71
+ "8k": 0.867,
72
+ "16k": 0.838,
73
+ "32k": 0.786,
74
+ "64k": 0.67,
75
+ "128k": 0.4
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.331,
79
+ "32k": 0.333,
80
+ "64k": 0.329
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.214,
84
+ "8k": 0.214
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.28950000000000004,
88
+ "4k": 0.444,
89
+ "8k": 0.394,
90
+ "16k": 0.303,
91
+ "32k": 0.266,
92
+ "64k": 0.209,
93
+ "128k": 0.121
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.10600000000000002,
97
+ "4k": 0.224,
98
+ "8k": 0.145,
99
+ "16k": 0.156,
100
+ "32k": 0.055,
101
+ "64k": 0.046,
102
+ "128k": 0.01
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.07216666666666666,
106
+ "4k": 0.117,
107
+ "8k": 0.118,
108
+ "16k": 0.101,
109
+ "32k": 0.046,
110
+ "64k": 0.031,
111
+ "128k": 0.02
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.4036666666666667,
115
+ "4k": 0.603,
116
+ "8k": 0.612,
117
+ "16k": 0.434,
118
+ "32k": 0.368,
119
+ "64k": 0.253,
120
+ "128k": 0.152
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.49550000000000005,
124
+ "4k": 0.505,
125
+ "8k": 0.525,
126
+ "16k": 0.541,
127
+ "32k": 0.479,
128
+ "64k": 0.479,
129
+ "128k": 0.444
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.6585000000000001,
133
+ "8k": 0.556,
134
+ "16k": 0.761
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.76,
138
+ "8k": 0.76
139
+ }
140
+ }
results/meta-llama_Llama-3.1-8B-Instruct.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.6474074074074074,
3
+ "passkey": {
4
+ "dataset_total_score": 1.0,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 1.0,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.75,
14
+ "4k": 0.81,
15
+ "8k": 0.777,
16
+ "16k": 0.73,
17
+ "32k": 0.767,
18
+ "64k": 0.752,
19
+ "128k": 0.664
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.6801999999999999,
23
+ "4k": 0.747,
24
+ "8k": 0.727,
25
+ "16k": 0.724,
26
+ "32k": 0.647,
27
+ "64k": 0.556
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 1.0,
31
+ "4k": 1.0,
32
+ "8k": 1.0,
33
+ "16k": 1.0,
34
+ "32k": 1.0,
35
+ "64k": 1.0,
36
+ "128k": 1.0
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.9299999999999999,
40
+ "8k": 0.969,
41
+ "16k": 0.938,
42
+ "32k": 0.938,
43
+ "64k": 0.875
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.13216666666666668,
47
+ "4k": 0.49,
48
+ "8k": 0.232,
49
+ "16k": 0.06,
50
+ "32k": 0.0,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.6476666666666667,
56
+ "8k": 0.762,
57
+ "16k": 0.604,
58
+ "32k": 0.577
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.40679999999999994,
62
+ "4k": 0.481,
63
+ "8k": 0.512,
64
+ "16k": 0.434,
65
+ "32k": 0.339,
66
+ "64k": 0.268
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.7991666666666667,
70
+ "4k": 0.99,
71
+ "8k": 0.962,
72
+ "16k": 0.952,
73
+ "32k": 0.806,
74
+ "64k": 0.73,
75
+ "128k": 0.355
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.4385,
79
+ "32k": 0.412,
80
+ "64k": 0.465
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.487,
84
+ "8k": 0.487
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.608,
88
+ "4k": 0.606,
89
+ "8k": 0.626,
90
+ "16k": 0.687,
91
+ "32k": 0.66,
92
+ "64k": 0.604,
93
+ "128k": 0.465
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.3126666666666667,
97
+ "4k": 0.553,
98
+ "8k": 0.421,
99
+ "16k": 0.364,
100
+ "32k": 0.329,
101
+ "64k": 0.169,
102
+ "128k": 0.04
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.2785,
106
+ "4k": 0.333,
107
+ "8k": 0.324,
108
+ "16k": 0.29,
109
+ "32k": 0.262,
110
+ "64k": 0.262,
111
+ "128k": 0.2
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.6616666666666666,
115
+ "4k": 0.731,
116
+ "8k": 0.729,
117
+ "16k": 0.723,
118
+ "32k": 0.69,
119
+ "64k": 0.693,
120
+ "128k": 0.404
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.753,
124
+ "4k": 0.768,
125
+ "8k": 0.727,
126
+ "16k": 0.745,
127
+ "32k": 0.771,
128
+ "64k": 0.719,
129
+ "128k": 0.788
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.829,
133
+ "8k": 0.778,
134
+ "16k": 0.88
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.939,
138
+ "8k": 0.939
139
+ }
140
+ }
results/microsoft_Phi-3.5-mini-instruct.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_score": 0.4884,
3
+ "passkey": {
4
+ "dataset_total_score": 0.9983333333333334,
5
+ "4k": 1.0,
6
+ "8k": 1.0,
7
+ "16k": 1.0,
8
+ "32k": 1.0,
9
+ "64k": 0.99,
10
+ "128k": 1.0
11
+ },
12
+ "matreshka_yes_no": {
13
+ "dataset_total_score": 0.7341666666666667,
14
+ "4k": 0.773,
15
+ "8k": 0.72,
16
+ "16k": 0.783,
17
+ "32k": 0.733,
18
+ "64k": 0.714,
19
+ "128k": 0.682
20
+ },
21
+ "matreshka_names": {
22
+ "dataset_total_score": 0.374,
23
+ "4k": 0.527,
24
+ "8k": 0.46,
25
+ "16k": 0.359,
26
+ "32k": 0.28,
27
+ "64k": 0.244
28
+ },
29
+ "passkey_with_librusec": {
30
+ "dataset_total_score": 0.9125,
31
+ "4k": 0.915,
32
+ "8k": 0.865,
33
+ "16k": 0.84,
34
+ "32k": 0.92,
35
+ "64k": 0.98,
36
+ "128k": 0.955
37
+ },
38
+ "librusec_history": {
39
+ "dataset_total_score": 0.7265,
40
+ "8k": 0.812,
41
+ "16k": 0.812,
42
+ "32k": 0.688,
43
+ "64k": 0.594
44
+ },
45
+ "ru_sci_passage_count": {
46
+ "dataset_total_score": 0.083,
47
+ "4k": 0.24,
48
+ "8k": 0.141,
49
+ "16k": 0.06,
50
+ "32k": 0.046,
51
+ "64k": 0.011,
52
+ "128k": 0.0
53
+ },
54
+ "ru_2wikimultihopqa": {
55
+ "dataset_total_score": 0.305,
56
+ "8k": 0.333,
57
+ "16k": 0.284,
58
+ "32k": 0.298
59
+ },
60
+ "long_context_multiq": {
61
+ "dataset_total_score": 0.34919999999999995,
62
+ "4k": 0.367,
63
+ "8k": 0.446,
64
+ "16k": 0.337,
65
+ "32k": 0.303,
66
+ "64k": 0.293
67
+ },
68
+ "ru_sci_abstract_retrieval": {
69
+ "dataset_total_score": 0.28800000000000003,
70
+ "4k": 0.536,
71
+ "8k": 0.333,
72
+ "16k": 0.248,
73
+ "32k": 0.262,
74
+ "64k": 0.249,
75
+ "128k": 0.1
76
+ },
77
+ "ru_sci_fi": {
78
+ "dataset_total_score": 0.1335,
79
+ "32k": 0.093,
80
+ "64k": 0.174
81
+ },
82
+ "librusec_mhqa": {
83
+ "dataset_total_score": 0.307,
84
+ "8k": 0.307
85
+ },
86
+ "ru_babilong_qa1": {
87
+ "dataset_total_score": 0.43016666666666664,
88
+ "4k": 0.626,
89
+ "8k": 0.626,
90
+ "16k": 0.505,
91
+ "32k": 0.468,
92
+ "64k": 0.275,
93
+ "128k": 0.081
94
+ },
95
+ "ru_babilong_qa2": {
96
+ "dataset_total_score": 0.17416666666666666,
97
+ "4k": 0.259,
98
+ "8k": 0.25,
99
+ "16k": 0.221,
100
+ "32k": 0.11,
101
+ "64k": 0.154,
102
+ "128k": 0.051
103
+ },
104
+ "ru_babilong_qa3": {
105
+ "dataset_total_score": 0.2775,
106
+ "4k": 0.483,
107
+ "8k": 0.294,
108
+ "16k": 0.29,
109
+ "32k": 0.262,
110
+ "64k": 0.246,
111
+ "128k": 0.09
112
+ },
113
+ "ru_babilong_qa4": {
114
+ "dataset_total_score": 0.426,
115
+ "4k": 0.654,
116
+ "8k": 0.553,
117
+ "16k": 0.53,
118
+ "32k": 0.425,
119
+ "64k": 0.293,
120
+ "128k": 0.101
121
+ },
122
+ "ru_babilong_qa5": {
123
+ "dataset_total_score": 0.7121666666666666,
124
+ "4k": 0.707,
125
+ "8k": 0.768,
126
+ "16k": 0.816,
127
+ "32k": 0.688,
128
+ "64k": 0.698,
129
+ "128k": 0.596
130
+ },
131
+ "ru_quality": {
132
+ "dataset_total_score": 0.813,
133
+ "8k": 0.833,
134
+ "16k": 0.793
135
+ },
136
+ "ru_tpo": {
137
+ "dataset_total_score": 0.747,
138
+ "8k": 0.747
139
+ }
140
+ }