Hugging Face's logo

bmarti44
/
coconut-curriculum-checkpoints

English
latent-reasoning
chain-of-thought
continuous-thought
coconut
prosqa
graph-traversal
curriculum-learning
pause-tokens
gpt2
Model card Files
xet
 Community
bmarti44 commited on
Commit
6bf0e17
·
verified ·
1 Parent(s): adcc88b

Upload folder using huggingface_hub

Browse files
Files changed (49) hide show
  1. .gitattributes +2 -0
  2. experiments/causal_sanity/exp0_sanity_result.json +15 -0
  3. experiments/causal_sanity/m1_causal.json +346 -0
  4. experiments/corruption/m3_corruption.json +36 -0
  5. experiments/corruption/m5_corruption.json +36 -0
  6. experiments/corruption/results.json +80 -0
  7. experiments/entropy_diagnostic.json +26 -0
  8. experiments/fig3_corruption_curves_fixed.png +3 -0
  9. experiments/m3_test_eval.json +1 -0
  10. experiments/m5_test_eval.json +9 -0
  11. experiments/m6/accuracy.json +27 -0
  12. experiments/m6/corruption.json +26 -0
  13. experiments/m6/m6_hidden_states.npz +3 -0
  14. experiments/m6/m6_labels.json +1 -0
  15. experiments/m6/m6_linear_perm.json +302 -0
  16. experiments/m6/m6_transplant.json +7 -0
  17. experiments/m6/mcnemar.json +205 -0
  18. experiments/m6/per_sample_correctness.json +4512 -0
  19. experiments/m6/summary.json +17 -0
  20. experiments/m6_epoch39/accuracy.json +27 -0
  21. experiments/m6_epoch39/per_sample_correctness.json +4512 -0
  22. experiments/m6_epoch39/summary.json +37 -0
  23. experiments/mlp_probe_grid_search.json +3378 -0
  24. experiments/ood/detailed_outputs.json +3 -0
  25. experiments/ood/log.txt +375 -0
  26. experiments/ood/results.json +32 -0
  27. experiments/per_sample_correctness.json +1 -0
  28. experiments/per_sample_logprobs.json +0 -0
  29. experiments/per_sample_logprobs_v3.json +1 -0
  30. experiments/per_sample_species_logprobs_m3_vs_m5.json +0 -0
  31. experiments/per_sample_species_logprobs_m3_vs_m6.json +0 -0
  32. experiments/per_sample_species_logprobs_m5_vs_m6.json +0 -0
  33. experiments/probing/log.txt +184 -0
  34. experiments/probing/m3_probing.json +773 -0
  35. experiments/probing/m5_probing.json +773 -0
  36. experiments/probing/results.json +1556 -0
  37. experiments/probing_corrected/m3_hidden_states.npz +3 -0
  38. experiments/probing_corrected/m3_labels.json +1 -0
  39. experiments/probing_corrected/m3_linear_perm.json +334 -0
  40. experiments/probing_corrected/m5_hidden_states.npz +3 -0
  41. experiments/probing_corrected/m5_labels.json +1 -0
  42. experiments/probing_corrected/m5_linear_perm.json +334 -0
  43. experiments/probing_corrected/results_linear_perm.json +336 -0
  44. experiments/wilcoxon_diagnostics.json +508 -0
  45. experiments/wilcoxon_sensitivity.json +88 -0
  46. experiments/wilcoxon_sensitivity_v3.json +102 -0
  47. experiments/wilcoxon_teacher_forced_m3_vs_m5.json +109 -0
  48. experiments/wilcoxon_teacher_forced_m3_vs_m6.json +109 -0
  49. experiments/wilcoxon_teacher_forced_m5_vs_m6.json +109 -0
.gitattributes CHANGED
@@ -84,3 +84,5 @@ cot-baseline/checkpoint_7 filter=lfs diff=lfs merge=lfs -text
84
  cot-baseline/checkpoint_8 filter=lfs diff=lfs merge=lfs -text
85
  cot-baseline/checkpoint_9 filter=lfs diff=lfs merge=lfs -text
86
  cot-baseline/checkpoint_best filter=lfs diff=lfs merge=lfs -text
 
 
 
84
  cot-baseline/checkpoint_8 filter=lfs diff=lfs merge=lfs -text
85
  cot-baseline/checkpoint_9 filter=lfs diff=lfs merge=lfs -text
86
  cot-baseline/checkpoint_best filter=lfs diff=lfs merge=lfs -text
87
+ experiments/fig3_corruption_curves_fixed.png filter=lfs diff=lfs merge=lfs -text
88
+ experiments/ood/detailed_outputs.json filter=lfs diff=lfs merge=lfs -text
experiments/causal_sanity/exp0_sanity_result.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "passed": true,
3
+ "peak_ce": 1.2572,
4
+ "n_positions_above_03": 1,
5
+ "ce_above_03_fraction": 0.05,
6
+ "clean_gt_corrupt_fraction": 0.119,
7
+ "elapsed_seconds": 75.1,
8
+ "n_traced": 42,
9
+ "criteria": {
10
+ "peak_ce_threshold": 1.0,
11
+ "min_positions_above_03": 1,
12
+ "clean_gt_corrupt_threshold": 0.5,
13
+ "max_time_seconds": 600
14
+ }
15
+ }
experiments/causal_sanity/m1_causal.json ADDED
@@ -0,0 +1,346 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mean_ce": [
3
+ [
4
+ -0.16012496646810076,
5
+ 0.01997633009751217,
6
+ -0.011189369222812154,
7
+ 0.05165428229819717,
8
+ 0.026143581603684,
9
+ -0.010497100037811432,
10
+ 0.009958913356476577,
11
+ -0.018316142171473548,
12
+ 0.06625397039822456,
13
+ 0.10633759634878241,
14
+ 0.017236376167957854,
15
+ -0.001228730528668957,
16
+ 0.004490682802466753,
17
+ 0.009115884122688647,
18
+ 0.1228624699481922,
19
+ 0.02976465405199852,
20
+ -0.09837585612330024,
21
+ -0.008551122557819051,
22
+ 0.1854778259820659,
23
+ 0.9951394702229243
24
+ ],
25
+ [
26
+ -0.17159805287663468,
27
+ 0.007685010217634519,
28
+ 0.0006605351113609268,
29
+ 0.013615752213937127,
30
+ 0.033158736575204996,
31
+ -0.007738768229894277,
32
+ -0.012798348415291887,
33
+ -0.01288241849592576,
34
+ 0.06581448278991608,
35
+ 0.11579181893779712,
36
+ 0.03359278946096855,
37
+ 0.026086086705694497,
38
+ 0.019036320275735447,
39
+ 0.03223905314931078,
40
+ 0.1407119985999539,
41
+ 0.007009530155604939,
42
+ -0.09916664032400921,
43
+ -0.0695058984933531,
44
+ 0.1758610978407338,
45
+ 1.0567818513280467
46
+ ],
47
+ [
48
+ -0.1457318086445327,
49
+ 0.0027790961609346822,
50
+ -0.012740898956283022,
51
+ 0.006555862303907191,
52
+ 0.03543356162229797,
53
+ -0.016784650095375882,
54
+ -0.02049173130208186,
55
+ -0.020024662038580117,
56
+ 0.05211714524618024,
57
+ 0.1347041275325647,
58
+ 0.008352765434555038,
59
+ 0.018698487496012977,
60
+ -0.016699565700085406,
61
+ 0.0039234971464250255,
62
+ 0.13663534893829338,
63
+ 0.007532493155215162,
64
+ -0.07642640272112404,
65
+ -0.15215203710849834,
66
+ 0.16862078167393285,
67
+ 1.144441695779006
68
+ ],
69
+ [
70
+ -0.13583462169619112,
71
+ -0.002189373609434716,
72
+ -0.031163806892287278,
73
+ 0.002228885022784129,
74
+ 0.03401181620859263,
75
+ -0.02540451575682411,
76
+ -0.02116537422602536,
77
+ -0.011652327807535645,
78
+ 0.05560385526252098,
79
+ 0.11850349796128856,
80
+ -0.010255390330579608,
81
+ 0.036560032195425175,
82
+ -0.0038072789352764777,
83
+ -0.0016032696913398138,
84
+ 0.11238487093027788,
85
+ 0.003980585505784385,
86
+ 0.00028638209826507236,
87
+ -0.18926579035779661,
88
+ 0.17811277656521604,
89
+ 1.2271104212241741
90
+ ],
91
+ [
92
+ -0.10256900524603799,
93
+ 0.009046757855468792,
94
+ -0.020032503020177908,
95
+ 0.007028433640565124,
96
+ 0.033155589118097094,
97
+ -0.024342103960125858,
98
+ -0.026545082887463996,
99
+ -0.015123059722365632,
100
+ 0.024000466633056265,
101
+ 0.09715496002353771,
102
+ -0.0009935001136818467,
103
+ -0.033055432573131927,
104
+ 0.009794845925488224,
105
+ -0.037163374374817267,
106
+ 0.10245862660030391,
107
+ 0.0015636854831897512,
108
+ -0.0057752024422063584,
109
+ -0.14995563946909524,
110
+ 0.12329672931815856,
111
+ 1.2572442685747525
112
+ ],
113
+ [
114
+ -0.10402076839485895,
115
+ -0.01112889452515091,
116
+ 0.001499013091168055,
117
+ -0.035931888086749436,
118
+ 0.027548237114466623,
119
+ -0.014551105772537891,
120
+ -0.006461585387131483,
121
+ -0.01492659537808287,
122
+ -0.0024647052233938444,
123
+ 0.07775404562515804,
124
+ -0.00968840047905958,
125
+ -0.008661533455197866,
126
+ 0.05971897676747538,
127
+ -0.03349698621190297,
128
+ 0.06799424662432256,
129
+ 0.09059268098711241,
130
+ 0.028424475352146105,
131
+ -0.1339872886062575,
132
+ 0.12795648593153028,
133
+ 1.0631186546336266
134
+ ],
135
+ [
136
+ -0.10622710011845318,
137
+ -0.024367744898749457,
138
+ 0.015253084845857742,
139
+ -0.03820152341179304,
140
+ 0.016814680341524897,
141
+ 0.017374284215020804,
142
+ -0.005308444661568347,
143
+ -0.007854317002273145,
144
+ 0.03554385486866505,
145
+ 0.042380960206095,
146
+ -0.0145791393872236,
147
+ 0.015432657060219058,
148
+ 0.07899446844518879,
149
+ -0.021237873661498183,
150
+ 0.019411867158635344,
151
+ 0.06332064579369008,
152
+ -0.006206352854597376,
153
+ -0.0754203125101134,
154
+ 0.11778468834207446,
155
+ 1.042689042518725
156
+ ],
157
+ [
158
+ -0.09461074370432791,
159
+ -0.03148577376498973,
160
+ 0.011496640997795265,
161
+ -0.017348220450395264,
162
+ 0.008549453559750989,
163
+ 0.04862679291636806,
164
+ 0.003845963854566902,
165
+ 0.004584312866103154,
166
+ 0.02225590437773916,
167
+ 0.015679669155626838,
168
+ -0.026049791151196454,
169
+ 0.009035379450443798,
170
+ 0.06826206795503513,
171
+ -0.02801724588558464,
172
+ 0.023387921321429146,
173
+ -0.016792270685099538,
174
+ -0.016493520030110353,
175
+ -0.09829706195219609,
176
+ 0.10253502351621258,
177
+ 0.8800627097424186
178
+ ],
179
+ [
180
+ -0.06949755274649144,
181
+ -0.0315574537420657,
182
+ 0.00036721194857005653,
183
+ 0.007863775529979217,
184
+ 0.012103646503104807,
185
+ 0.050852338901659945,
186
+ -0.006301735095131872,
187
+ 0.013479748107914257,
188
+ 0.008547124739096901,
189
+ 0.02433988433116142,
190
+ -0.009654499466186364,
191
+ 0.006648385080915018,
192
+ 0.07679026956366092,
193
+ -0.04257442867727117,
194
+ 0.01076183609685665,
195
+ -0.025288782878731014,
196
+ -0.0003301632876580231,
197
+ -0.134925731720201,
198
+ 0.03940764422459925,
199
+ 0.8687106859131203
200
+ ],
201
+ [
202
+ -0.04174408026289439,
203
+ -0.008390397711789158,
204
+ -0.014724367915640246,
205
+ -0.002083920895710072,
206
+ 0.005715352034655784,
207
+ 0.0154564246865361,
208
+ -0.0017750481768138875,
209
+ 0.012530756502705262,
210
+ -0.00034629503380442366,
211
+ 0.021036759225673156,
212
+ -0.009998390810880251,
213
+ -0.006638459501006368,
214
+ 0.013367257617347663,
215
+ -0.012166844134111631,
216
+ -0.004194776999836225,
217
+ -0.017144376382928237,
218
+ 0.017586309091998834,
219
+ -0.14362598379306826,
220
+ -0.022941314679155344,
221
+ 0.8185773576316141
222
+ ],
223
+ [
224
+ -0.04138583224381289,
225
+ -0.002460333155831041,
226
+ -0.005796531639856134,
227
+ -0.0067684177087288565,
228
+ 0.0063566108383195175,
229
+ 0.01752717588252675,
230
+ 0.0022973516487158935,
231
+ -0.0021872078085959238,
232
+ 0.0021518070091218386,
233
+ -0.013225320262068614,
234
+ -0.003772691917298187,
235
+ -0.01579807245684832,
236
+ 0.008278018918414384,
237
+ 0.0021464419319260513,
238
+ 0.0009595526999910713,
239
+ -0.01074112937107895,
240
+ -0.009181296214727535,
241
+ -0.00493052675290442,
242
+ -0.026089446420961653,
243
+ 0.4933978360917525
244
+ ],
245
+ [
246
+ 0.009149292142113156,
247
+ -0.003609511548978105,
248
+ 0.0049170833562806405,
249
+ -0.005325324432707347,
250
+ 0.009721030865243915,
251
+ 0.012419284375750144,
252
+ 0.0065431187864623515,
253
+ 0.00041741887542662927,
254
+ 0.007716099241021581,
255
+ -0.0041172923042538286,
256
+ 0.0007289908864467926,
257
+ 0.002635974822088256,
258
+ 0.008905466582432164,
259
+ -0.0018257428528887995,
260
+ 0.007311456956998676,
261
+ -0.011122054198559134,
262
+ -0.006934612698912083,
263
+ 0.010700592904259534,
264
+ 0.00921156669120079,
265
+ 0.22763108309758218
266
+ ],
267
+ [
268
+ 0.0,
269
+ 0.0,
270
+ 0.0,
271
+ 0.0,
272
+ 0.0,
273
+ 0.0,
274
+ 0.0,
275
+ 0.0,
276
+ 0.0,
277
+ 0.0,
278
+ 0.0,
279
+ 0.0,
280
+ 0.0,
281
+ 0.0,
282
+ 0.0,
283
+ 0.0,
284
+ 0.0,
285
+ 0.0,
286
+ 0.0,
287
+ 0.0
288
+ ]
289
+ ],
290
+ "n_traced": 42,
291
+ "n_skipped": 8,
292
+ "n_wrong_answer": 8,
293
+ "max_positions": 20,
294
+ "position_counts": [
295
+ 20,
296
+ 20,
297
+ 20,
298
+ 20,
299
+ 20,
300
+ 20,
301
+ 20,
302
+ 20,
303
+ 20,
304
+ 20,
305
+ 20,
306
+ 20,
307
+ 20,
308
+ 20,
309
+ 20,
310
+ 20,
311
+ 20,
312
+ 20,
313
+ 20,
314
+ 20,
315
+ 20,
316
+ 20,
317
+ 20,
318
+ 20,
319
+ 20,
320
+ 20,
321
+ 20,
322
+ 20,
323
+ 20,
324
+ 20,
325
+ 20,
326
+ 20,
327
+ 20,
328
+ 20,
329
+ 20,
330
+ 20,
331
+ 20,
332
+ 20,
333
+ 20,
334
+ 20,
335
+ 20,
336
+ 20
337
+ ],
338
+ "peak_layer": 4,
339
+ "peak_position": 19,
340
+ "peak_ce": 1.2572,
341
+ "ce_above_03_fraction": 0.05,
342
+ "clean_gt_corrupt_fraction": 0.119,
343
+ "ce_in_range_fraction": 0.5615,
344
+ "epsilon_sensitivity_pct": 0.0,
345
+ "noise_std": 0.1173
346
+ }
experiments/corruption/m3_corruption.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "clean_accuracy": 0.97,
3
+ "forward_corruption": [
4
+ 0.968,
5
+ 0.968,
6
+ 0.968,
7
+ 0.574,
8
+ 0.156,
9
+ 0.024
10
+ ],
11
+ "reverse_corruption": [
12
+ 0.97,
13
+ 0.968,
14
+ 0.968,
15
+ 0.574,
16
+ 0.156,
17
+ 0.024
18
+ ],
19
+ "single_position": [
20
+ 0.968,
21
+ 0.968,
22
+ 0.968,
23
+ 0.576,
24
+ 0.156,
25
+ 0.024
26
+ ],
27
+ "total_samples": 500,
28
+ "sensitivity": "high_uniform",
29
+ "cross_transplant_accuracy": 0.97,
30
+ "cross_transplant_pairs": 200,
31
+ "permutation_flip_rate": 0.0,
32
+ "mean_permutation_flip_rate": 0.0,
33
+ "permutation_total_samples": 500,
34
+ "partial_permutation_flip_rate": 0.0,
35
+ "partial_permutation_total_samples": 500
36
+ }
experiments/corruption/m5_corruption.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "clean_accuracy": 0.966,
3
+ "forward_corruption": [
4
+ 0.964,
5
+ 0.962,
6
+ 0.958,
7
+ 0.572,
8
+ 0.156,
9
+ 0.022
10
+ ],
11
+ "reverse_corruption": [
12
+ 0.966,
13
+ 0.96,
14
+ 0.96,
15
+ 0.572,
16
+ 0.154,
17
+ 0.022
18
+ ],
19
+ "single_position": [
20
+ 0.964,
21
+ 0.962,
22
+ 0.962,
23
+ 0.578,
24
+ 0.158,
25
+ 0.022
26
+ ],
27
+ "total_samples": 500,
28
+ "sensitivity": "high_uniform",
29
+ "cross_transplant_accuracy": 0.965,
30
+ "cross_transplant_pairs": 200,
31
+ "permutation_flip_rate": 0.0,
32
+ "mean_permutation_flip_rate": 0.0,
33
+ "permutation_total_samples": 500,
34
+ "partial_permutation_flip_rate": 0.0,
35
+ "partial_permutation_total_samples": 500
36
+ }
experiments/corruption/results.json ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "m3": {
3
+ "clean_accuracy": 0.97,
4
+ "forward_corruption": [
5
+ 0.968,
6
+ 0.968,
7
+ 0.968,
8
+ 0.574,
9
+ 0.156,
10
+ 0.024
11
+ ],
12
+ "reverse_corruption": [
13
+ 0.97,
14
+ 0.968,
15
+ 0.968,
16
+ 0.574,
17
+ 0.156,
18
+ 0.024
19
+ ],
20
+ "single_position": [
21
+ 0.968,
22
+ 0.968,
23
+ 0.968,
24
+ 0.576,
25
+ 0.156,
26
+ 0.024
27
+ ],
28
+ "total_samples": 500,
29
+ "sensitivity": "high_uniform",
30
+ "cross_transplant_accuracy": 0.97,
31
+ "cross_transplant_pairs": 200,
32
+ "permutation_flip_rate": 0.0,
33
+ "mean_permutation_flip_rate": 0.0,
34
+ "permutation_total_samples": 500,
35
+ "partial_permutation_flip_rate": 0.0,
36
+ "partial_permutation_total_samples": 500
37
+ },
38
+ "m5": {
39
+ "clean_accuracy": 0.966,
40
+ "forward_corruption": [
41
+ 0.964,
42
+ 0.962,
43
+ 0.958,
44
+ 0.572,
45
+ 0.156,
46
+ 0.022
47
+ ],
48
+ "reverse_corruption": [
49
+ 0.966,
50
+ 0.96,
51
+ 0.96,
52
+ 0.572,
53
+ 0.154,
54
+ 0.022
55
+ ],
56
+ "single_position": [
57
+ 0.964,
58
+ 0.962,
59
+ 0.962,
60
+ 0.578,
61
+ 0.158,
62
+ 0.022
63
+ ],
64
+ "total_samples": 500,
65
+ "sensitivity": "high_uniform",
66
+ "cross_transplant_accuracy": 0.965,
67
+ "cross_transplant_pairs": 200,
68
+ "permutation_flip_rate": 0.0,
69
+ "mean_permutation_flip_rate": 0.0,
70
+ "permutation_total_samples": 500,
71
+ "partial_permutation_flip_rate": 0.0,
72
+ "partial_permutation_total_samples": 500
73
+ },
74
+ "verification": {
75
+ "replacement_l2_distance": 202.65,
76
+ "m3_test_id_accuracy": 0.97,
77
+ "m3_test_id_accuracy_delta": 0.0,
78
+ "position_check_passed": true
79
+ }
80
+ }
experiments/entropy_diagnostic.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prosqa_id": {
3
+ "n": 500,
4
+ "m3_mean_entropy": 8.245684373986754e-08,
5
+ "m5_mean_entropy": 7.790865615004972e-07,
6
+ "m3_std_entropy": 5.380577548978713e-07,
7
+ "m5_std_entropy": 2.5222992223835106e-06,
8
+ "m3_median_entropy": 3.2982319098096013e-09,
9
+ "m5_median_entropy": 9.583473570273782e-08,
10
+ "entropy_diff_mean": -6.966297177606296e-07,
11
+ "entropy_diff_p_value": 3.2357157290647014e-09,
12
+ "direction": "M5 > M3"
13
+ },
14
+ "dag": {
15
+ "n": 1000,
16
+ "m3_mean_entropy": 5.54877669094885e-08,
17
+ "m5_mean_entropy": 9.562996519757449e-07,
18
+ "m3_std_entropy": 1.9521541945357018e-07,
19
+ "m5_std_entropy": 4.271332058753898e-06,
20
+ "m3_median_entropy": 8.63850679877487e-09,
21
+ "m5_median_entropy": 2.2466630866802006e-07,
22
+ "entropy_diff_mean": -9.008118850662564e-07,
23
+ "entropy_diff_p_value": 3.5962255165486594e-11,
24
+ "direction": "M5 > M3"
25
+ }
26
+ }
experiments/fig3_corruption_curves_fixed.png ADDED

Git LFS Details

  • SHA256: 74b99537cb4a38912086e3a5a28e6783e7f766289e72438f385a814fa92b9430
  • Pointer size: 131 Bytes
  • Size of remote file: 162 kB
experiments/m3_test_eval.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"model": "m3", "checkpoint": "checkpoint_best_epoch49", "dataset": "prosqa_test", "n_samples": 500, "correct": 490, "accuracy": 0.98, "time_s": 35.4}
experiments/m5_test_eval.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "m5",
3
+ "checkpoint": "checkpoint_best_epoch43",
4
+ "dataset": "prosqa_test",
5
+ "n_samples": 500,
6
+ "correct": 478,
7
+ "accuracy": 0.956,
8
+ "time_s": 25.1
9
+ }
experiments/m6/accuracy.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prosqa_test": {
3
+ "accuracy": 0.948,
4
+ "correct": 474,
5
+ "total": 500
6
+ },
7
+ "ood_7hop": {
8
+ "accuracy": 0.769,
9
+ "correct": 769,
10
+ "total": 1000
11
+ },
12
+ "ood_8hop": {
13
+ "accuracy": 0.752,
14
+ "correct": 752,
15
+ "total": 1000
16
+ },
17
+ "ood_dag": {
18
+ "accuracy": 0.598,
19
+ "correct": 598,
20
+ "total": 1000
21
+ },
22
+ "ood_dense": {
23
+ "accuracy": 0.648,
24
+ "correct": 648,
25
+ "total": 1000
26
+ }
27
+ }
experiments/m6/corruption.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "forward": {
3
+ "0": 0.024,
4
+ "1": 0.024,
5
+ "2": 0.024,
6
+ "3": 0.024,
7
+ "4": 0.022,
8
+ "5": 0.022
9
+ },
10
+ "reverse": {
11
+ "0": 0.024,
12
+ "1": 0.024,
13
+ "2": 0.024,
14
+ "3": 0.022,
15
+ "4": 0.024,
16
+ "5": 0.024
17
+ },
18
+ "single_position": {
19
+ "0": 0.024,
20
+ "1": 0.024,
21
+ "2": 0.024,
22
+ "3": 0.024,
23
+ "4": 0.024,
24
+ "5": 0.024
25
+ }
26
+ }
experiments/m6/m6_hidden_states.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67e4a03ed62ee8c5c314dd896f02b7b64e295e4ded9c8abdf11eb6dd1bf6f155
3
+ size 75520402
experiments/m6/m6_labels.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"0": ["scrompus", "storpus", "sterpus", "vumpus", "jelpus", "bompus", "boompus", "brimpus", "sterpus", "bompus", "chorpus", "fompus", "rempus", "rompus", "sterpus", "rorpus", "numpus", "dumpus", "storpus", "brimpus", "gerpus", "rompus", "jompus", "numpus", "rompus", "terpus", "sterpus", "fompus", "storpus", "dumpus", "scrompus", "quimpus", "yerpus", "jelpus", "gorpus", "tumpus", "shumpus", "chorpus", "yerpus", "vumpus", "impus", "scrompus", "zumpus", "rorpus", "rempus", "fompus", "yumpus", "rorpus", "bompus", "dumpus", "gwompus", "felpus", "chorpus", "wumpus", "jompus", "felpus", "grimpus", "wumpus", "fompus", "wumpus", "gwompus", "gwompus", "bompus", "vumpus", "lorpus", "worpus", "yumpus", "fompus", "gorpus", "scrompus", "jompus", "brimpus", "yumpus", "worpus", "rempus", "jelpus", "fompus", "scrompus", "jompus", "rompus", "grimpus", "rompus", "terpus", "gerpus", "yumpus", "storpus", "gwompus", "lorpus", "lorpus", "rompus", "timpus", "grimpus", "tumpus", "rorpus", "gwompus", "gorpus", "numpus", "boompus", "jelpus", "grimpus", "dumpus", "lorpus", "rempus", "jompus", "bompus", "sterpus", "scrompus", "lempus", "grimpus", "quimpus", "hilpus", "jelpus", "worpus", "shumpus", "jompus", "zumpus", "quimpus", "gerpus", "grimpus", "yimpus", "zhorpus", "lempus", "zumpus", "zhorpus", "zumpus", "zumpus", "wumpus", "rompus", "shumpus", "quimpus", "chorpus", "grimpus", "sterpus", "sterpus", "lempus", "yimpus", "gerpus", "zumpus", "sterpus", "worpus", "rorpus", "yimpus", "jelpus", "storpus", "dumpus", "fompus", "brimpus", "terpus", "storpus", "gerpus", "timpus", "vumpus", "sterpus", "rorpus", "gerpus", "vumpus", "fompus", "bompus", "yerpus", "yerpus", "jelpus", "sterpus", "yumpus", "shumpus", "tumpus", "rompus", "dumpus", "storpus", "lempus", "rempus", "lempus", "gorpus", "lorpus", "lorpus", "bompus", "numpus", "felpus", "jompus", "rempus", "boompus", "brimpus", "sterpus", "dumpus", "wumpus", "vumpus", "vumpus", "felpus", "timpus", "boompus", "dumpus", "bompus", "boompus", "brimpus", "shumpus", "hilpus", "lempus", "impus", "rompus", "boompus", "terpus", "gorpus", "tumpus", "numpus", "kerpus", "gerpus", "scrompus", "grimpus", "lempus", "grimpus", "timpus", "chorpus", "hilpus", "vumpus", "dumpus", "gorpus", "gerpus", "lempus", "yimpus", "fompus", "yumpus", "shumpus", "lorpus", "fompus", "kerpus", "jompus", "storpus", "timpus", "kerpus", "sterpus", "rompus", "yimpus", "brimpus", "storpus", "boompus", "grimpus", "terpus", "kerpus", "rempus", "brimpus", "grimpus", "gwompus", "jelpus", "jelpus", "jelpus", "scrompus", "yumpus", "numpus", "grimpus", "yerpus", "jompus", "yumpus", "wumpus", "chorpus", "jelpus", "bompus", "shumpus", "brimpus", "hilpus", "jelpus", "zumpus", "sterpus", "fompus", "jompus", "dumpus", "zhorpus", "sterpus", "lorpus", "scrompus", "zhorpus", "yimpus", "jelpus", "scrompus", "rompus", "wumpus", "quimpus", "kerpus", "storpus", "bompus", "wumpus", "terpus", "quimpus", "impus", "quimpus", "boompus", "rorpus", "fompus", "storpus", "chorpus", "timpus", "zumpus", "wumpus", "grimpus", "kerpus", "lempus", "fompus", "quimpus", "zumpus", "grimpus", "yerpus", "yimpus", "numpus", "jompus", "jompus", "gorpus", "yimpus", "jelpus", "gorpus", "sterpus", "fompus", "scrompus", "zumpus", "tumpus", "rempus", "hilpus", "storpus", "rompus", "shumpus", "yumpus", "lempus", "jompus", "zumpus", "tumpus", "gerpus", "yerpus", "quimpus", "impus", "worpus", "zumpus", "yimpus", "terpus", "felpus", "tumpus", "wumpus", "bompus", "felpus", "vumpus", "quimpus", "yerpus", "scrompus", "worpus", "zumpus", "chorpus", "shumpus", "yimpus", "gwompus", "brimpus", "rorpus", "tumpus", "impus", "felpus", "boompus", "kerpus", "fompus", "gwompus", "rorpus", "vumpus", "yerpus", "impus", "quimpus", "yimpus", "tumpus", "quimpus", "quimpus", "bompus", "hilpus", "numpus", "numpus", "vumpus", "zumpus", "numpus", "scrompus", "tumpus", "fompus", "quimpus", "yerpus", "impus", "gerpus", "rempus", "jelpus", "chorpus", "felpus", "felpus", "numpus", "terpus", "sterpus", "yerpus", "terpus", "gorpus", "gorpus", "timpus", "zumpus", "shumpus", "hilpus", "scrompus", "vumpus", "jompus", "terpus", "jelpus", "hilpus", "bompus", "sterpus", "boompus", "lempus", "bompus", "scrompus", "jelpus", "rempus", "zumpus", "sterpus", "kerpus", "zumpus", "zhorpus", "vumpus", "dumpus", "storpus", "yimpus", "yimpus", "timpus", "storpus", "fompus", "dumpus", "gerpus", "bompus", "jelpus", "felpus", "rompus", "numpus", "zhorpus", "quimpus", "quimpus", "vumpus", "impus", "kerpus", "quimpus", "boompus", "yumpus", "hilpus", "gerpus", "terpus", "fompus", "shumpus", "kerpus", "lempus", "jelpus", "sterpus", "rempus", "boompus", "gorpus", "gorpus", "jompus", "grimpus", "sterpus", "scrompus", "zumpus", "rorpus", "rompus", "rompus", "scrompus", "dumpus", "zumpus", "grimpus", "jompus", "shumpus", "gorpus", "shumpus", "vumpus", "rempus", "yumpus", "zhorpus", "rempus", "bompus", "dumpus", "storpus", "scrompus", "boompus", "wumpus", "boompus", "grimpus", "storpus", "scrompus", "rempus", "timpus", "scrompus", "worpus", "kerpus", "gwompus", "zumpus", "yumpus", "jompus", "zhorpus", "timpus", "storpus", "fompus", "scrompus", "timpus", "yimpus", "gorpus", "wumpus", "storpus", "sterpus"], "1": ["rempus", "grimpus", "worpus", "rempus", "tumpus", "storpus", "rorpus", "storpus", "boompus", "rompus", "terpus", "gerpus", "lorpus", "storpus", "yerpus", "lorpus", "kerpus", "storpus", "gerpus", "numpus", "tumpus", "gerpus", "numpus", "jompus", "hilpus", "hilpus", "grimpus", "vumpus", "fompus", "jompus", "vumpus", "boompus", "storpus", "jompus", "grimpus", "impus", "lempus", "gerpus", "grimpus", "zhorpus", "fompus", "sterpus", "tumpus", "lempus", "tumpus", "wumpus", "storpus", "bompus", "gorpus", "sterpus", "numpus", "zhorpus", "gwompus", "scrompus", "boompus", "grimpus", "rorpus", "storpus", "zumpus", "tumpus", "rempus", "vumpus", "gorpus", "zumpus", "chorpus", "tumpus", "worpus", "chorpus", "storpus", "vumpus", "rorpus", "gwompus", "zhorpus", "yerpus", "shumpus", "hilpus", "gorpus", "rompus", "grimpus", "worpus", "gwompus", "brimpus", "worpus", "fompus", "storpus", "kerpus", "kerpus", "yimpus", "wumpus", "vumpus", "grimpus", "shumpus", "gorpus", "lorpus", "numpus", "tumpus", "yimpus", "wumpus", "rorpus", "yerpus", "zhorpus", "rorpus", "terpus", "lempus", "worpus", "rompus", "yimpus", "rempus", "brimpus", "storpus", "gorpus", "gwompus", "kerpus", "gerpus", "terpus", "impus", "jompus", "felpus", "impus", "dumpus", "vumpus", "jelpus", "gorpus", "lorpus", "numpus", "storpus", "hilpus", "storpus", "quimpus", "hilpus", "storpus", "gerpus", "chorpus", "fompus", "yerpus", "zumpus", "jelpus", "jompus", "terpus", "rorpus", "shumpus", "shumpus", "tumpus", "gorpus", "wumpus", "rempus", "chorpus", "rempus", "scrompus", "rempus", "jompus", "timpus", "bompus", "kerpus", "brimpus", "gwompus", "numpus", "jompus", "impus", "boompus", "bompus", "boompus", "tumpus", "numpus", "yerpus", "wumpus", "rompus", "terpus", "quimpus", "gerpus", "felpus", "quimpus", "quimpus", "numpus", "yumpus", "felpus", "brimpus", "quimpus", "sterpus", "lorpus", "jompus", "rorpus", "brimpus", "storpus", "worpus", "scrompus", "grimpus", "felpus", "worpus", "scrompus", "felpus", "gerpus", "yerpus", "zhorpus", "kerpus", "yumpus", "gerpus", "sterpus", "dumpus", "quimpus", "tumpus", "timpus", "quimpus", "boompus", "chorpus", "gerpus", "wumpus", "gwompus", "yerpus", "boompus", "hilpus", "felpus", "scrompus", "boompus", "rompus", "fompus", "quimpus", "felpus", "zumpus", "wumpus", "felpus", "chorpus", "yimpus", "worpus", "rempus", "jelpus", "gerpus", "yerpus", "gwompus", "jompus", "rorpus", "sterpus", "gwompus", "lorpus", "shumpus", "jompus", "fompus", "storpus", "rempus", "lorpus", "gorpus", "brimpus", "grimpus", "yimpus", "yerpus", "zumpus", "tumpus", "kerpus", "brimpus", "sterpus", "bompus", "numpus", "zumpus", "zhorpus", "tumpus", "numpus", "timpus", "rompus", "worpus", "rompus", "chorpus", "numpus", "hilpus", "timpus", "sterpus", "grimpus", "jompus", "terpus", "shumpus", "tumpus", "gwompus", "jelpus", "wumpus", "dumpus", "sterpus", "gorpus", "jompus", "dumpus", "zumpus", "brimpus", "jompus", "bompus", "sterpus", "rompus", "tumpus", "timpus", "gerpus", "hilpus", "wumpus", "rorpus", "rorpus", "quimpus", "brimpus", "zhorpus", "wumpus", "sterpus", "grimpus", "yimpus", "tumpus", "bompus", "jompus", "kerpus", "impus", "yerpus", "boompus", "yerpus", "bompus", "vumpus", "hilpus", "jelpus", "hilpus", "jelpus", "quimpus", "bompus", "yimpus", "wumpus", "boompus", "gwompus", "gorpus", "storpus", "rempus", "impus", "zhorpus", "vumpus", "numpus", "yumpus", "quimpus", "boompus", "gwompus", "timpus", "jelpus", "kerpus", "yimpus", "jelpus", "bompus", "scrompus", "rorpus", "zumpus", "yumpus", "dumpus", "zhorpus", "wumpus", "fompus", "jelpus", "rorpus", "numpus", "dumpus", "boompus", "worpus", "quimpus", "gerpus", "bompus", "brimpus", "yerpus", "scrompus", "quimpus", "yimpus", "sterpus", "tumpus", "rempus", "hilpus", "zhorpus", "bompus", "tumpus", "storpus", "quimpus", "terpus", "wumpus", "impus", "quimpus", "sterpus", "gorpus", "yerpus", "zhorpus", "worpus", "chorpus", "chorpus", "shumpus", "wumpus", "terpus", "yerpus", "tumpus", "shumpus", "tumpus", "zumpus", "hilpus", "wumpus", "rompus", "terpus", "fompus", "gerpus", "scrompus", "terpus", "chorpus", "lempus", "brimpus", "wumpus", "rompus", "rompus", "grimpus", "impus", "brimpus", "sterpus", "lempus", "gwompus", "kerpus", "felpus", "lempus", "hilpus", "zumpus", "rorpus", "quimpus", "boompus", "timpus", "boompus", "worpus", "shumpus", "yimpus", "yerpus", "worpus", "bompus", "zumpus", "tumpus", "impus", "numpus", "zumpus", "yimpus", "boompus", "rempus", "jelpus", "bompus", "rompus", "brimpus", "terpus", "dumpus", "felpus", "chorpus", "vumpus", "rorpus", "boompus", "yimpus", "bompus", "gwompus", "impus", "jompus", "dumpus", "tumpus", "fompus", "terpus", "worpus", "chorpus", "gwompus", "yerpus", "tumpus", "bompus", "tumpus", "yumpus", "yumpus", "yimpus", "gorpus", "fompus", "dumpus", "gerpus", "hilpus", "vumpus", "zhorpus", "tumpus", "brimpus", "jelpus", "yimpus", "scrompus", "sterpus", "dumpus", "brimpus", "rempus", "grimpus", "rompus", "tumpus", "sterpus", "yumpus", "scrompus", "impus", "jelpus", "storpus", "rompus", "rompus", "fompus", "timpus", "worpus", "yumpus", "zumpus", "wumpus", "felpus", "rompus", "terpus", "gorpus", "yerpus", "jompus", "bompus", "impus"], "2": ["sterpus", "jelpus", "boompus", "timpus", "zumpus", "wumpus", "gorpus", "dumpus", "lorpus", "yerpus", "felpus", "shumpus", "tumpus", "gorpus", "chorpus", "gwompus", "grimpus", "yumpus", "fompus", "boompus", "storpus", "felpus", "hilpus", "gwompus", "chorpus", "gerpus", "boompus", "yerpus", "gorpus", "brimpus", "worpus", "wumpus", "timpus", "gorpus", "sterpus", "rompus", "wumpus", "shumpus", "tumpus", "terpus", "yimpus", "fompus", "bompus", "boompus", "scrompus", "quimpus", "kerpus", "fompus", "wumpus", "gwompus", "tumpus", "kerpus", "tumpus", "rorpus", "chorpus", "rorpus", "bompus", "worpus", "terpus", "bompus", "vumpus", "boompus", "scrompus", "yumpus", "rorpus", "zhorpus", "dumpus", "numpus", "zhorpus", "zumpus", "felpus", "dumpus", "wumpus", "quimpus", "jompus", "storpus", "numpus", "worpus", "boompus", "fompus", "lempus", "boompus", "felpus", "wumpus", "yerpus", "quimpus", "gerpus", "rempus", "storpus", "bompus", "kerpus", "quimpus", "rempus", "timpus", "dumpus", "sterpus", "zumpus", "chorpus", "jompus", "impus", "rorpus", "numpus", "bompus", "felpus", "gerpus", "boompus", "brimpus", "storpus", "rompus", "yumpus", "felpus", "quimpus", "rempus", "fompus", "boompus", "gerpus", "numpus", "gwompus", "storpus", "chorpus", "timpus", "brimpus", "sterpus", "numpus", "timpus", "gerpus", "quimpus", "quimpus", "yerpus", "rempus", "fompus", "dumpus", "scrompus", "vumpus", "impus", "gwompus", "shumpus", "terpus", "impus", "gwompus", "dumpus", "tumpus", "shumpus", "lempus", "jompus", "yerpus", "jelpus", "wumpus", "tumpus", "yerpus", "lorpus", "gorpus", "fompus", "dumpus", "grimpus", "timpus", "timpus", "gorpus", "timpus", "jelpus", "scrompus", "bompus", "shumpus", "scrompus", "impus", "lempus", "jompus", "jelpus", "shumpus", "gorpus", "wumpus", "tumpus", "boompus", "tumpus", "vumpus", "rempus", "quimpus", "yerpus", "vumpus", "brimpus", "chorpus", "worpus", "gorpus", "zhorpus", "timpus", "impus", "impus", "yumpus", "lempus", "impus", "scrompus", "numpus", "wumpus", "wumpus", "yimpus", "impus", "numpus", "gorpus", "brimpus", "tumpus", "impus", "rompus", "terpus", "yumpus", "tumpus", "fompus", "gwompus", "yumpus", "quimpus", "storpus", "wumpus", "gwompus", "shumpus", "rompus", "boompus", "impus", "timpus", "wumpus", "yimpus", "hilpus", "gorpus", "zumpus", "grimpus", "scrompus", "gorpus", "boompus", "sterpus", "brimpus", "vumpus", "zumpus", "jelpus", "gorpus", "yimpus", "yumpus", "numpus", "jelpus", "bompus", "rompus", "felpus", "boompus", "gerpus", "rompus", "lorpus", "yumpus", "worpus", "boompus", "fompus", "rempus", "dumpus", "zhorpus", "brimpus", "terpus", "gwompus", "fompus", "rorpus", "chorpus", "rompus", "wumpus", "fompus", "rorpus", "zhorpus", "rompus", "zhorpus", "gwompus", "brimpus", "gerpus", "dumpus", "rempus", "wumpus", "storpus", "boompus", "hilpus", "kerpus", "yimpus", "rempus", "storpus", "jelpus", "storpus", "bompus", "fompus", "tumpus", "grimpus", "tumpus", "gorpus", "scrompus", "shumpus", "gorpus", "bompus", "numpus", "tumpus", "boompus", "yimpus", "felpus", "lorpus", "felpus", "zumpus", "shumpus", "hilpus", "yumpus", "storpus", "lempus", "yerpus", "lorpus", "dumpus", "brimpus", "kerpus", "fompus", "terpus", "vumpus", "shumpus", "dumpus", "lorpus", "rompus", "worpus", "tumpus", "gorpus", "fompus", "shumpus", "gwompus", "terpus", "fompus", "jompus", "rorpus", "tumpus", "rompus", "scrompus", "gerpus", "jelpus", "rempus", "fompus", "boompus", "timpus", "zhorpus", "lempus", "chorpus", "jelpus", "vumpus", "timpus", "chorpus", "brimpus", "hilpus", "shumpus", "gorpus", "jompus", "hilpus", "impus", "felpus", "zumpus", "yumpus", "gorpus", "brimpus", "hilpus", "tumpus", "numpus", "lempus", "lorpus", "quimpus", "bompus", "brimpus", "quimpus", "timpus", "worpus", "worpus", "dumpus", "brimpus", "impus", "jompus", "impus", "yimpus", "kerpus", "kerpus", "wumpus", "gerpus", "rempus", "gorpus", "felpus", "lorpus", "boompus", "lorpus", "yimpus", "yimpus", "quimpus", "gerpus", "vumpus", "grimpus", "quimpus", "quimpus", "dumpus", "bompus", "lempus", "timpus", "yumpus", "tumpus", "terpus", "numpus", "jelpus", "bompus", "chorpus", "lorpus", "felpus", "gwompus", "rompus", "jompus", "timpus", "lorpus", "dumpus", "sterpus", "yimpus", "fompus", "numpus", "kerpus", "sterpus", "shumpus", "boompus", "gerpus", "grimpus", "jompus", "zhorpus", "vumpus", "chorpus", "quimpus", "quimpus", "hilpus", "timpus", "scrompus", "terpus", "lorpus", "yerpus", "felpus", "grimpus", "yimpus", "worpus", "gerpus", "jelpus", "zumpus", "brimpus", "worpus", "terpus", "rompus", "rompus", "jompus", "lorpus", "shumpus", "grimpus", "hilpus", "yimpus", "rorpus", "boompus", "rorpus", "yumpus", "fompus", "terpus", "terpus", "numpus", "chorpus", "quimpus", "storpus", "rempus", "bompus", "rompus", "bompus", "terpus", "terpus", "sterpus", "rompus", "jompus", "impus", "terpus", "tumpus", "yumpus", "worpus", "rorpus", "fompus", "impus", "numpus", "dumpus", "zumpus", "yerpus", "shumpus", "vumpus", "lorpus", "rempus", "dumpus", "yumpus", "zumpus", "yerpus", "dumpus", "brimpus", "impus", "rorpus", "terpus", "bompus", "boompus", "rempus", "gorpus", "storpus", "dumpus", "gerpus", "jompus", "brimpus"], "3": ["shumpus", "tumpus", "rompus", "vumpus", "lempus", "quimpus", "rorpus", "vumpus", "felpus", "brimpus", "shumpus", "yerpus", "sterpus", "boompus", "kerpus", "shumpus", "gorpus", "chorpus", "gwompus", "sterpus", "fompus", "terpus", "terpus", "dumpus", "numpus", "yerpus", "vumpus", "shumpus", "scrompus", "grimpus", "brimpus", "gwompus", "lempus", "lorpus", "jompus", "jompus", "gwompus", "felpus", "chorpus", "yerpus", "hilpus", "kerpus", "rorpus", "yerpus", "felpus", "yerpus", "bompus", "scrompus", "lorpus", "yumpus", "zumpus", "scrompus", "gwompus", "gwompus", "fompus", "sterpus", "scrompus", "numpus", "yerpus", "bompus", "storpus", "timpus", "jompus", "wumpus", "bompus", "impus", "rorpus", "zumpus", "hilpus", "jompus", "gerpus", "gwompus", "storpus", "timpus", "chorpus", "numpus", "fompus", "chorpus", "vumpus", "yimpus", "numpus", "yumpus", "gorpus", "rompus", "quimpus", "zhorpus", "grimpus", "bompus", "zhorpus", "kerpus", "gerpus", "grimpus", "shumpus", "storpus", "yumpus", "worpus", "yumpus", "shumpus", "yumpus", "yumpus", "yerpus", "yimpus", "lempus", "yimpus", "fompus", "kerpus", "yimpus", "rorpus", "boompus", "gwompus", "hilpus", "yimpus", "dumpus", "zhorpus", "rompus", "lempus", "shumpus", "yimpus", "grimpus", "storpus", "impus", "boompus", "tumpus", "felpus", "shumpus", "felpus", "shumpus", "quimpus", "vumpus", "terpus", "gorpus", "boompus", "worpus", "scrompus", "sterpus", "lorpus", "lorpus", "felpus", "quimpus", "boompus", "shumpus", "rompus", "felpus", "rompus", "zhorpus", "rompus", "dumpus", "lempus", "yerpus", "zhorpus", "dumpus", "storpus", "chorpus", "dumpus", "dumpus", "chorpus", "timpus", "impus", "lempus", "rempus", "worpus", "chorpus", "numpus", "quimpus", "shumpus", "wumpus", "lorpus", "vumpus", "yerpus", "fompus", "timpus", "jompus", "gwompus", "impus", "lempus", "storpus", "quimpus", "zumpus", "terpus", "kerpus", "rompus", "hilpus", "zumpus", "chorpus", "rempus", "jompus", "numpus", "terpus", "rorpus", "felpus", "gorpus", "lorpus", "rompus", "felpus", "terpus", "tumpus", "quimpus", "wumpus", "chorpus", "vumpus", "jompus", "grimpus", "yerpus", "chorpus", "sterpus", "timpus", "rompus", "felpus", "kerpus", "lorpus", "worpus", "tumpus", "scrompus", "zhorpus", "terpus", "sterpus", "hilpus", "quimpus", "zumpus", "worpus", "tumpus", "gwompus", "jompus", "hilpus", "rempus", "rempus", "lorpus", "storpus", "zhorpus", "yerpus", "brimpus", "rompus", "quimpus", "storpus", "storpus", "impus", "fompus", "brimpus", "lempus", "chorpus", "impus", "zhorpus", "zhorpus", "gorpus", "bompus", "gorpus", "jelpus", "chorpus", "fompus", "storpus", "impus", "dumpus", "sterpus", "wumpus", "numpus", "rempus", "terpus", "numpus", "timpus", "rempus", "lorpus", "wumpus", "terpus", "zhorpus", "hilpus", "yumpus", "scrompus", "vumpus", "kerpus", "scrompus", "yimpus", "yimpus", "sterpus", "rempus", "yumpus", "grimpus", "rompus", "terpus", "rompus", "jompus", "brimpus", "gerpus", "numpus", "chorpus", "bompus", "boompus", "zhorpus", "yimpus", "gerpus", "terpus", "timpus", "felpus", "gorpus", "terpus", "terpus", "jompus", "lorpus", "jelpus"], "4": ["worpus", "jompus", "boompus", "hilpus", "yumpus", "terpus", "zhorpus", "wumpus", "sterpus", "brimpus", "hilpus", "tumpus", "rompus", "bompus", "yimpus", "yumpus", "bompus", "rempus", "gerpus", "vumpus", "felpus", "jompus", "quimpus", "scrompus", "brimpus", "wumpus", "tumpus", "lorpus", "yimpus", "brimpus", "storpus", "chorpus", "terpus", "jompus", "terpus", "jelpus", "brimpus", "chorpus", "scrompus", "lempus", "jelpus", "rempus", "wumpus", "boompus", "zumpus", "jompus", "rempus", "gwompus", "timpus", "yimpus", "bompus", "lorpus", "impus", "hilpus", "yimpus", "timpus", "scrompus", "vumpus", "gerpus", "dumpus", "terpus", "worpus", "lempus", "yerpus", "hilpus", "yerpus", "yumpus", "shumpus", "tumpus", "rempus", "worpus", "jompus", "storpus", "tumpus", "storpus", "gerpus", "yerpus", "dumpus", "hilpus", "jelpus", "zhorpus"], "5": ["hilpus", "shumpus", "lorpus", "dumpus", "sterpus", "scrompus", "felpus", "chorpus", "zumpus", "worpus", "lempus", "fompus"]}
experiments/m6/m6_linear_perm.json ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linear_probe_accuracy": [
3
+ [
4
+ 0.044,
5
+ 0.034,
6
+ 0.04000000000000001,
7
+ 0.03691275167785235,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ [
12
+ 0.039999999999999994,
13
+ 0.014000000000000002,
14
+ 0.039999999999999994,
15
+ 0.06375838926174497,
16
+ 0.0,
17
+ 0.0
18
+ ],
19
+ [
20
+ 0.041999999999999996,
21
+ 0.022,
22
+ 0.05600000000000001,
23
+ 0.0738255033557047,
24
+ 0.0,
25
+ 0.0
26
+ ],
27
+ [
28
+ 0.033999999999999996,
29
+ 0.033999999999999996,
30
+ 0.048,
31
+ 0.06711409395973154,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ [
36
+ 0.062,
37
+ 0.034,
38
+ 0.046000000000000006,
39
+ 0.09060402684563758,
40
+ 0.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.06,
45
+ 0.044,
46
+ 0.05600000000000001,
47
+ 0.03355704697986577,
48
+ 0.0,
49
+ 0.0
50
+ ],
51
+ [
52
+ 0.1,
53
+ 0.042,
54
+ 0.07,
55
+ 0.02348993288590604,
56
+ 0.0,
57
+ 0.0
58
+ ],
59
+ [
60
+ 0.122,
61
+ 0.028000000000000004,
62
+ 0.038,
63
+ 0.0436241610738255,
64
+ 0.0,
65
+ 0.0
66
+ ],
67
+ [
68
+ 0.21800000000000003,
69
+ 0.1,
70
+ 0.05,
71
+ 0.016778523489932886,
72
+ 0.0,
73
+ 0.0
74
+ ],
75
+ [
76
+ 0.152,
77
+ 0.06799999999999999,
78
+ 0.072,
79
+ 0.04697986577181208,
80
+ 0.0,
81
+ 0.0
82
+ ],
83
+ [
84
+ 0.134,
85
+ 0.094,
86
+ 0.126,
87
+ 0.087248322147651,
88
+ 0.0,
89
+ 0.0
90
+ ],
91
+ [
92
+ 0.11400000000000002,
93
+ 0.06000000000000001,
94
+ 0.16,
95
+ 0.28859060402684567,
96
+ 0.0,
97
+ 0.0
98
+ ],
99
+ [
100
+ 0.094,
101
+ 0.08,
102
+ 0.182,
103
+ 0.2919463087248322,
104
+ 0.0,
105
+ 0.0
106
+ ]
107
+ ],
108
+ "permutation_p_values": [
109
+ [
110
+ 0.12993503248375812,
111
+ 0.2623688155922039,
112
+ 0.13493253373313344,
113
+ 0.1904047976011994,
114
+ 1.0,
115
+ 1.0
116
+ ],
117
+ [
118
+ 0.272863568215892,
119
+ 0.7381309345327336,
120
+ 0.2638680659670165,
121
+ 0.06046976511744128,
122
+ 1.0,
123
+ 1.0
124
+ ],
125
+ [
126
+ 0.12443778110944528,
127
+ 0.4927536231884058,
128
+ 0.047976011994003,
129
+ 0.01649175412293853,
130
+ 1.0,
131
+ 1.0
132
+ ],
133
+ [
134
+ 0.27436281859070466,
135
+ 0.26636681659170414,
136
+ 0.11294352823588207,
137
+ 0.025487256371814093,
138
+ 1.0,
139
+ 1.0
140
+ ],
141
+ [
142
+ 0.01999000499750125,
143
+ 0.2733633183408296,
144
+ 0.11944027986006997,
145
+ 0.0029985007496251873,
146
+ 1.0,
147
+ 1.0
148
+ ],
149
+ [
150
+ 0.05347326336831584,
151
+ 0.11444277861069466,
152
+ 0.041979010494752625,
153
+ 0.19640179910044978,
154
+ 1.0,
155
+ 1.0
156
+ ],
157
+ [
158
+ 0.0004997501249375312,
159
+ 0.11244377811094453,
160
+ 0.017991004497751123,
161
+ 0.4447776111944028,
162
+ 1.0,
163
+ 1.0
164
+ ],
165
+ [
166
+ 0.0004997501249375312,
167
+ 0.49525237381309345,
168
+ 0.2698650674662669,
169
+ 0.22288855572213892,
170
+ 1.0,
171
+ 1.0
172
+ ],
173
+ [
174
+ 0.0004997501249375312,
175
+ 0.0009995002498750624,
176
+ 0.12343828085957022,
177
+ 0.45327336331834084,
178
+ 1.0,
179
+ 1.0
180
+ ],
181
+ [
182
+ 0.0004997501249375312,
183
+ 0.022988505747126436,
184
+ 0.0029985007496251873,
185
+ 0.18640679660169915,
186
+ 1.0,
187
+ 1.0
188
+ ],
189
+ [
190
+ 0.0004997501249375312,
191
+ 0.0004997501249375312,
192
+ 0.0004997501249375312,
193
+ 0.0024987506246876563,
194
+ 1.0,
195
+ 1.0
196
+ ],
197
+ [
198
+ 0.0004997501249375312,
199
+ 0.019490254872563718,
200
+ 0.0004997501249375312,
201
+ 0.0004997501249375312,
202
+ 1.0,
203
+ 1.0
204
+ ],
205
+ [
206
+ 0.0009995002498750624,
207
+ 0.006496751624187906,
208
+ 0.0004997501249375312,
209
+ 0.0004997501249375312,
210
+ 1.0,
211
+ 1.0
212
+ ]
213
+ ],
214
+ "significant_cells": [
215
+ {
216
+ "layer": 6,
217
+ "position": 0,
218
+ "accuracy": 0.1,
219
+ "p_value": 0.0004997501249375312
220
+ },
221
+ {
222
+ "layer": 7,
223
+ "position": 0,
224
+ "accuracy": 0.122,
225
+ "p_value": 0.0004997501249375312
226
+ },
227
+ {
228
+ "layer": 8,
229
+ "position": 0,
230
+ "accuracy": 0.21800000000000003,
231
+ "p_value": 0.0004997501249375312
232
+ },
233
+ {
234
+ "layer": 9,
235
+ "position": 0,
236
+ "accuracy": 0.152,
237
+ "p_value": 0.0004997501249375312
238
+ },
239
+ {
240
+ "layer": 10,
241
+ "position": 0,
242
+ "accuracy": 0.134,
243
+ "p_value": 0.0004997501249375312
244
+ },
245
+ {
246
+ "layer": 11,
247
+ "position": 0,
248
+ "accuracy": 0.11400000000000002,
249
+ "p_value": 0.0004997501249375312
250
+ },
251
+ {
252
+ "layer": 10,
253
+ "position": 1,
254
+ "accuracy": 0.094,
255
+ "p_value": 0.0004997501249375312
256
+ },
257
+ {
258
+ "layer": 10,
259
+ "position": 2,
260
+ "accuracy": 0.126,
261
+ "p_value": 0.0004997501249375312
262
+ },
263
+ {
264
+ "layer": 11,
265
+ "position": 2,
266
+ "accuracy": 0.16,
267
+ "p_value": 0.0004997501249375312
268
+ },
269
+ {
270
+ "layer": 12,
271
+ "position": 2,
272
+ "accuracy": 0.182,
273
+ "p_value": 0.0004997501249375312
274
+ },
275
+ {
276
+ "layer": 11,
277
+ "position": 3,
278
+ "accuracy": 0.28859060402684567,
279
+ "p_value": 0.0004997501249375312
280
+ },
281
+ {
282
+ "layer": 12,
283
+ "position": 3,
284
+ "accuracy": 0.2919463087248322,
285
+ "p_value": 0.0004997501249375312
286
+ }
287
+ ],
288
+ "n_significant_cells": 12,
289
+ "bonferroni_threshold": 0.000641025641025641,
290
+ "n_samples_per_position": {
291
+ "0": 500,
292
+ "1": 500,
293
+ "2": 500,
294
+ "3": 298,
295
+ "4": 81,
296
+ "5": 12
297
+ },
298
+ "n_permutations": 2000,
299
+ "peak_layer": 12,
300
+ "peak_position": 3,
301
+ "peak_accuracy": 0.2919463087248322
302
+ }
experiments/m6/m6_transplant.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "matched": 0,
3
+ "unmatched": 0,
4
+ "total": 0,
5
+ "matched_rate": 0.0,
6
+ "unmatched_rate": 0.0
7
+ }
experiments/m6/mcnemar.json ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "n_tests": 10,
3
+ "results": {
4
+ "m6_vs_m3_prosqa_test": {
5
+ "n": 500,
6
+ "target_acc": 0.948,
7
+ "ref_acc": 0.98,
8
+ "diff_pp": -3.2,
9
+ "ci_95": [
10
+ -5.1,
11
+ -1.3
12
+ ],
13
+ "contingency": {
14
+ "both_correct": 470,
15
+ "m6_only": 4,
16
+ "m3_only": 20,
17
+ "both_wrong": 6
18
+ },
19
+ "discordant": 24,
20
+ "p_value": 0.001543879508972168,
21
+ "odds_ratio": 0.2,
22
+ "p_bonferroni": 0.01543879508972168
23
+ },
24
+ "m6_vs_m3_ood_7hop": {
25
+ "n": 1000,
26
+ "target_acc": 0.769,
27
+ "ref_acc": 0.707,
28
+ "diff_pp": 6.2,
29
+ "ci_95": [
30
+ 2.82,
31
+ 9.58
32
+ ],
33
+ "contingency": {
34
+ "both_correct": 587,
35
+ "m6_only": 182,
36
+ "m3_only": 120,
37
+ "both_wrong": 111
38
+ },
39
+ "discordant": 302,
40
+ "p_value": 0.0004294605210459007,
41
+ "odds_ratio": 1.5166666666666666,
42
+ "p_bonferroni": 0.004294605210459007
43
+ },
44
+ "m6_vs_m3_ood_8hop": {
45
+ "n": 1000,
46
+ "target_acc": 0.752,
47
+ "ref_acc": 0.695,
48
+ "diff_pp": 5.7,
49
+ "ci_95": [
50
+ 2.28,
51
+ 9.12
52
+ ],
53
+ "contingency": {
54
+ "both_correct": 570,
55
+ "m6_only": 182,
56
+ "m3_only": 125,
57
+ "both_wrong": 123
58
+ },
59
+ "discordant": 307,
60
+ "p_value": 0.001354478357551105,
61
+ "odds_ratio": 1.456,
62
+ "p_bonferroni": 0.01354478357551105
63
+ },
64
+ "m6_vs_m3_ood_dag": {
65
+ "n": 1000,
66
+ "target_acc": 0.598,
67
+ "ref_acc": 0.597,
68
+ "diff_pp": 0.1,
69
+ "ci_95": [
70
+ -3.54,
71
+ 3.74
72
+ ],
73
+ "contingency": {
74
+ "both_correct": 425,
75
+ "m6_only": 173,
76
+ "m3_only": 172,
77
+ "both_wrong": 230
78
+ },
79
+ "discordant": 345,
80
+ "p_value": 1.0,
81
+ "odds_ratio": 1.005813953488372,
82
+ "p_bonferroni": 1.0
83
+ },
84
+ "m6_vs_m3_ood_dense": {
85
+ "n": 1000,
86
+ "target_acc": 0.648,
87
+ "ref_acc": 0.629,
88
+ "diff_pp": 1.9,
89
+ "ci_95": [
90
+ -1.79,
91
+ 5.59
92
+ ],
93
+ "contingency": {
94
+ "both_correct": 461,
95
+ "m6_only": 187,
96
+ "m3_only": 168,
97
+ "both_wrong": 184
98
+ },
99
+ "discordant": 355,
100
+ "p_value": 0.3394157064437944,
101
+ "odds_ratio": 1.1130952380952381,
102
+ "p_bonferroni": 1.0
103
+ },
104
+ "m6_vs_m5_prosqa_test": {
105
+ "n": 500,
106
+ "target_acc": 0.948,
107
+ "ref_acc": 0.956,
108
+ "diff_pp": -0.8,
109
+ "ci_95": [
110
+ -2.8,
111
+ 1.2
112
+ ],
113
+ "contingency": {
114
+ "both_correct": 463,
115
+ "m6_only": 11,
116
+ "m5_only": 15,
117
+ "both_wrong": 11
118
+ },
119
+ "discordant": 26,
120
+ "p_value": 0.557197093963623,
121
+ "odds_ratio": 0.7333333333333333,
122
+ "p_bonferroni": 1.0
123
+ },
124
+ "m6_vs_m5_ood_7hop": {
125
+ "n": 1000,
126
+ "target_acc": 0.769,
127
+ "ref_acc": 0.719,
128
+ "diff_pp": 5.0,
129
+ "ci_95": [
130
+ 1.69,
131
+ 8.31
132
+ ],
133
+ "contingency": {
134
+ "both_correct": 600,
135
+ "m6_only": 169,
136
+ "m5_only": 119,
137
+ "both_wrong": 112
138
+ },
139
+ "discordant": 288,
140
+ "p_value": 0.003809189023554528,
141
+ "odds_ratio": 1.4201680672268908,
142
+ "p_bonferroni": 0.03809189023554528
143
+ },
144
+ "m6_vs_m5_ood_8hop": {
145
+ "n": 1000,
146
+ "target_acc": 0.752,
147
+ "ref_acc": 0.726,
148
+ "diff_pp": 2.6,
149
+ "ci_95": [
150
+ -0.79,
151
+ 5.99
152
+ ],
153
+ "contingency": {
154
+ "both_correct": 589,
155
+ "m6_only": 163,
156
+ "m5_only": 137,
157
+ "both_wrong": 111
158
+ },
159
+ "discordant": 300,
160
+ "p_value": 0.14879255774103087,
161
+ "odds_ratio": 1.1897810218978102,
162
+ "p_bonferroni": 1.0
163
+ },
164
+ "m6_vs_m5_ood_dag": {
165
+ "n": 1000,
166
+ "target_acc": 0.598,
167
+ "ref_acc": 0.578,
168
+ "diff_pp": 2.0,
169
+ "ci_95": [
170
+ -1.66,
171
+ 5.66
172
+ ],
173
+ "contingency": {
174
+ "both_correct": 413,
175
+ "m6_only": 185,
176
+ "m5_only": 165,
177
+ "both_wrong": 237
178
+ },
179
+ "discordant": 350,
180
+ "p_value": 0.3098201765186668,
181
+ "odds_ratio": 1.121212121212121,
182
+ "p_bonferroni": 1.0
183
+ },
184
+ "m6_vs_m5_ood_dense": {
185
+ "n": 1000,
186
+ "target_acc": 0.648,
187
+ "ref_acc": 0.684,
188
+ "diff_pp": -3.6,
189
+ "ci_95": [
190
+ -7.22,
191
+ 0.02
192
+ ],
193
+ "contingency": {
194
+ "both_correct": 495,
195
+ "m6_only": 153,
196
+ "m5_only": 189,
197
+ "both_wrong": 163
198
+ },
199
+ "discordant": 342,
200
+ "p_value": 0.05825460914566845,
201
+ "odds_ratio": 0.8095238095238095,
202
+ "p_bonferroni": 0.5825460914566845
203
+ }
204
+ }
205
+ }
experiments/m6/per_sample_correctness.json ADDED
@@ -0,0 +1,4512 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prosqa_test": [
3
+ 1,
4
+ 1,
5
+ 1,
6
+ 1,
7
+ 1,
8
+ 1,
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 0,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 1,
21
+ 0,
22
+ 1,
23
+ 1,
24
+ 1,
25
+ 1,
26
+ 1,
27
+ 1,
28
+ 1,
29
+ 1,
30
+ 1,
31
+ 1,
32
+ 1,
33
+ 0,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 0,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 0,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 0,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 0,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 0,
158
+ 1,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 1,
163
+ 1,
164
+ 0,
165
+ 1,
166
+ 1,
167
+ 1,
168
+ 1,
169
+ 1,
170
+ 1,
171
+ 1,
172
+ 1,
173
+ 1,
174
+ 1,
175
+ 1,
176
+ 1,
177
+ 1,
178
+ 1,
179
+ 1,
180
+ 1,
181
+ 1,
182
+ 1,
183
+ 1,
184
+ 1,
185
+ 1,
186
+ 1,
187
+ 1,
188
+ 1,
189
+ 1,
190
+ 0,
191
+ 1,
192
+ 1,
193
+ 1,
194
+ 1,
195
+ 1,
196
+ 1,
197
+ 1,
198
+ 1,
199
+ 1,
200
+ 1,
201
+ 1,
202
+ 1,
203
+ 1,
204
+ 1,
205
+ 1,
206
+ 1,
207
+ 1,
208
+ 0,
209
+ 1,
210
+ 1,
211
+ 1,
212
+ 1,
213
+ 1,
214
+ 1,
215
+ 1,
216
+ 1,
217
+ 1,
218
+ 1,
219
+ 1,
220
+ 1,
221
+ 1,
222
+ 1,
223
+ 1,
224
+ 1,
225
+ 1,
226
+ 1,
227
+ 1,
228
+ 1,
229
+ 1,
230
+ 1,
231
+ 1,
232
+ 1,
233
+ 1,
234
+ 0,
235
+ 1,
236
+ 1,
237
+ 1,
238
+ 1,
239
+ 1,
240
+ 0,
241
+ 1,
242
+ 1,
243
+ 1,
244
+ 1,
245
+ 1,
246
+ 1,
247
+ 1,
248
+ 1,
249
+ 1,
250
+ 0,
251
+ 1,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 0,
256
+ 1,
257
+ 1,
258
+ 1,
259
+ 1,
260
+ 1,
261
+ 1,
262
+ 1,
263
+ 1,
264
+ 1,
265
+ 1,
266
+ 1,
267
+ 1,
268
+ 1,
269
+ 1,
270
+ 1,
271
+ 1,
272
+ 1,
273
+ 1,
274
+ 1,
275
+ 1,
276
+ 1,
277
+ 0,
278
+ 0,
279
+ 0,
280
+ 1,
281
+ 1,
282
+ 1,
283
+ 1,
284
+ 1,
285
+ 1,
286
+ 1,
287
+ 1,
288
+ 1,
289
+ 1,
290
+ 1,
291
+ 1,
292
+ 1,
293
+ 1,
294
+ 1,
295
+ 1,
296
+ 1,
297
+ 1,
298
+ 1,
299
+ 1,
300
+ 1,
301
+ 1,
302
+ 1,
303
+ 1,
304
+ 1,
305
+ 1,
306
+ 1,
307
+ 1,
308
+ 1,
309
+ 1,
310
+ 1,
311
+ 1,
312
+ 1,
313
+ 1,
314
+ 1,
315
+ 1,
316
+ 1,
317
+ 1,
318
+ 1,
319
+ 1,
320
+ 1,
321
+ 1,
322
+ 1,
323
+ 1,
324
+ 1,
325
+ 1,
326
+ 1,
327
+ 1,
328
+ 1,
329
+ 1,
330
+ 1,
331
+ 1,
332
+ 1,
333
+ 1,
334
+ 0,
335
+ 1,
336
+ 1,
337
+ 1,
338
+ 1,
339
+ 1,
340
+ 1,
341
+ 1,
342
+ 1,
343
+ 1,
344
+ 1,
345
+ 1,
346
+ 1,
347
+ 1,
348
+ 1,
349
+ 1,
350
+ 1,
351
+ 1,
352
+ 1,
353
+ 1,
354
+ 1,
355
+ 1,
356
+ 1,
357
+ 1,
358
+ 0,
359
+ 1,
360
+ 1,
361
+ 1,
362
+ 1,
363
+ 1,
364
+ 1,
365
+ 1,
366
+ 1,
367
+ 1,
368
+ 1,
369
+ 1,
370
+ 1,
371
+ 1,
372
+ 1,
373
+ 1,
374
+ 1,
375
+ 1,
376
+ 1,
377
+ 1,
378
+ 1,
379
+ 1,
380
+ 1,
381
+ 1,
382
+ 1,
383
+ 1,
384
+ 1,
385
+ 1,
386
+ 1,
387
+ 1,
388
+ 1,
389
+ 0,
390
+ 1,
391
+ 1,
392
+ 1,
393
+ 1,
394
+ 1,
395
+ 1,
396
+ 1,
397
+ 0,
398
+ 1,
399
+ 1,
400
+ 1,
401
+ 1,
402
+ 1,
403
+ 1,
404
+ 1,
405
+ 1,
406
+ 1,
407
+ 1,
408
+ 1,
409
+ 1,
410
+ 1,
411
+ 0,
412
+ 1,
413
+ 1,
414
+ 1,
415
+ 1,
416
+ 1,
417
+ 1,
418
+ 1,
419
+ 1,
420
+ 1,
421
+ 1,
422
+ 1,
423
+ 1,
424
+ 1,
425
+ 1,
426
+ 1,
427
+ 1,
428
+ 1,
429
+ 1,
430
+ 1,
431
+ 1,
432
+ 1,
433
+ 1,
434
+ 1,
435
+ 0,
436
+ 1,
437
+ 1,
438
+ 1,
439
+ 1,
440
+ 1,
441
+ 1,
442
+ 1,
443
+ 1,
444
+ 1,
445
+ 1,
446
+ 1,
447
+ 1,
448
+ 1,
449
+ 1,
450
+ 1,
451
+ 1,
452
+ 1,
453
+ 1,
454
+ 1,
455
+ 1,
456
+ 1,
457
+ 1,
458
+ 1,
459
+ 1,
460
+ 1,
461
+ 1,
462
+ 1,
463
+ 1,
464
+ 1,
465
+ 1,
466
+ 1,
467
+ 0,
468
+ 1,
469
+ 1,
470
+ 1,
471
+ 0,
472
+ 1,
473
+ 1,
474
+ 1,
475
+ 1,
476
+ 1,
477
+ 1,
478
+ 1,
479
+ 1,
480
+ 1,
481
+ 1,
482
+ 1,
483
+ 1,
484
+ 1,
485
+ 1,
486
+ 1,
487
+ 1,
488
+ 1,
489
+ 1,
490
+ 1,
491
+ 1,
492
+ 1,
493
+ 1,
494
+ 1,
495
+ 1,
496
+ 1,
497
+ 1,
498
+ 1,
499
+ 1,
500
+ 1,
501
+ 1,
502
+ 1
503
+ ],
504
+ "ood_7hop": [
505
+ 1,
506
+ 1,
507
+ 1,
508
+ 0,
509
+ 1,
510
+ 1,
511
+ 1,
512
+ 1,
513
+ 1,
514
+ 0,
515
+ 1,
516
+ 1,
517
+ 1,
518
+ 0,
519
+ 1,
520
+ 1,
521
+ 1,
522
+ 1,
523
+ 1,
524
+ 1,
525
+ 1,
526
+ 1,
527
+ 1,
528
+ 1,
529
+ 1,
530
+ 1,
531
+ 1,
532
+ 1,
533
+ 1,
534
+ 1,
535
+ 1,
536
+ 0,
537
+ 1,
538
+ 1,
539
+ 0,
540
+ 1,
541
+ 1,
542
+ 1,
543
+ 1,
544
+ 1,
545
+ 0,
546
+ 1,
547
+ 1,
548
+ 1,
549
+ 1,
550
+ 0,
551
+ 0,
552
+ 1,
553
+ 1,
554
+ 0,
555
+ 1,
556
+ 1,
557
+ 1,
558
+ 1,
559
+ 1,
560
+ 1,
561
+ 1,
562
+ 1,
563
+ 0,
564
+ 1,
565
+ 1,
566
+ 1,
567
+ 1,
568
+ 1,
569
+ 1,
570
+ 1,
571
+ 1,
572
+ 1,
573
+ 1,
574
+ 1,
575
+ 1,
576
+ 1,
577
+ 1,
578
+ 1,
579
+ 1,
580
+ 1,
581
+ 1,
582
+ 1,
583
+ 1,
584
+ 1,
585
+ 1,
586
+ 1,
587
+ 1,
588
+ 1,
589
+ 1,
590
+ 1,
591
+ 1,
592
+ 1,
593
+ 1,
594
+ 1,
595
+ 1,
596
+ 1,
597
+ 1,
598
+ 1,
599
+ 1,
600
+ 1,
601
+ 0,
602
+ 1,
603
+ 1,
604
+ 1,
605
+ 1,
606
+ 1,
607
+ 1,
608
+ 1,
609
+ 0,
610
+ 1,
611
+ 0,
612
+ 1,
613
+ 1,
614
+ 1,
615
+ 0,
616
+ 1,
617
+ 0,
618
+ 1,
619
+ 1,
620
+ 1,
621
+ 0,
622
+ 1,
623
+ 1,
624
+ 1,
625
+ 0,
626
+ 1,
627
+ 1,
628
+ 1,
629
+ 1,
630
+ 0,
631
+ 1,
632
+ 0,
633
+ 1,
634
+ 1,
635
+ 1,
636
+ 1,
637
+ 1,
638
+ 1,
639
+ 1,
640
+ 1,
641
+ 1,
642
+ 1,
643
+ 1,
644
+ 0,
645
+ 1,
646
+ 1,
647
+ 1,
648
+ 1,
649
+ 1,
650
+ 1,
651
+ 1,
652
+ 1,
653
+ 1,
654
+ 0,
655
+ 0,
656
+ 1,
657
+ 1,
658
+ 0,
659
+ 1,
660
+ 1,
661
+ 1,
662
+ 0,
663
+ 1,
664
+ 1,
665
+ 1,
666
+ 1,
667
+ 0,
668
+ 0,
669
+ 1,
670
+ 1,
671
+ 1,
672
+ 1,
673
+ 1,
674
+ 0,
675
+ 1,
676
+ 1,
677
+ 1,
678
+ 1,
679
+ 1,
680
+ 1,
681
+ 1,
682
+ 1,
683
+ 0,
684
+ 1,
685
+ 1,
686
+ 1,
687
+ 0,
688
+ 1,
689
+ 1,
690
+ 0,
691
+ 1,
692
+ 0,
693
+ 1,
694
+ 1,
695
+ 1,
696
+ 1,
697
+ 1,
698
+ 1,
699
+ 1,
700
+ 1,
701
+ 1,
702
+ 1,
703
+ 0,
704
+ 1,
705
+ 1,
706
+ 0,
707
+ 1,
708
+ 1,
709
+ 0,
710
+ 0,
711
+ 1,
712
+ 1,
713
+ 1,
714
+ 1,
715
+ 1,
716
+ 1,
717
+ 1,
718
+ 1,
719
+ 1,
720
+ 0,
721
+ 0,
722
+ 0,
723
+ 1,
724
+ 1,
725
+ 1,
726
+ 1,
727
+ 1,
728
+ 1,
729
+ 0,
730
+ 0,
731
+ 1,
732
+ 1,
733
+ 1,
734
+ 1,
735
+ 1,
736
+ 1,
737
+ 1,
738
+ 1,
739
+ 1,
740
+ 1,
741
+ 1,
742
+ 1,
743
+ 1,
744
+ 1,
745
+ 0,
746
+ 1,
747
+ 1,
748
+ 1,
749
+ 1,
750
+ 1,
751
+ 1,
752
+ 1,
753
+ 1,
754
+ 0,
755
+ 1,
756
+ 1,
757
+ 1,
758
+ 0,
759
+ 1,
760
+ 1,
761
+ 0,
762
+ 1,
763
+ 1,
764
+ 1,
765
+ 0,
766
+ 0,
767
+ 1,
768
+ 1,
769
+ 1,
770
+ 1,
771
+ 1,
772
+ 1,
773
+ 0,
774
+ 1,
775
+ 1,
776
+ 0,
777
+ 1,
778
+ 1,
779
+ 0,
780
+ 1,
781
+ 1,
782
+ 1,
783
+ 0,
784
+ 1,
785
+ 1,
786
+ 0,
787
+ 0,
788
+ 1,
789
+ 1,
790
+ 0,
791
+ 0,
792
+ 1,
793
+ 1,
794
+ 0,
795
+ 1,
796
+ 0,
797
+ 0,
798
+ 1,
799
+ 1,
800
+ 0,
801
+ 1,
802
+ 0,
803
+ 1,
804
+ 1,
805
+ 1,
806
+ 1,
807
+ 1,
808
+ 1,
809
+ 0,
810
+ 1,
811
+ 1,
812
+ 1,
813
+ 0,
814
+ 1,
815
+ 0,
816
+ 1,
817
+ 1,
818
+ 1,
819
+ 1,
820
+ 1,
821
+ 0,
822
+ 1,
823
+ 1,
824
+ 1,
825
+ 1,
826
+ 1,
827
+ 1,
828
+ 0,
829
+ 1,
830
+ 1,
831
+ 1,
832
+ 1,
833
+ 1,
834
+ 1,
835
+ 1,
836
+ 0,
837
+ 1,
838
+ 1,
839
+ 1,
840
+ 1,
841
+ 1,
842
+ 0,
843
+ 1,
844
+ 0,
845
+ 1,
846
+ 1,
847
+ 1,
848
+ 0,
849
+ 0,
850
+ 1,
851
+ 0,
852
+ 1,
853
+ 1,
854
+ 1,
855
+ 0,
856
+ 1,
857
+ 1,
858
+ 1,
859
+ 0,
860
+ 0,
861
+ 1,
862
+ 1,
863
+ 1,
864
+ 1,
865
+ 1,
866
+ 1,
867
+ 1,
868
+ 1,
869
+ 1,
870
+ 1,
871
+ 1,
872
+ 1,
873
+ 1,
874
+ 0,
875
+ 1,
876
+ 0,
877
+ 0,
878
+ 1,
879
+ 1,
880
+ 1,
881
+ 1,
882
+ 0,
883
+ 1,
884
+ 1,
885
+ 1,
886
+ 1,
887
+ 1,
888
+ 1,
889
+ 1,
890
+ 1,
891
+ 1,
892
+ 0,
893
+ 1,
894
+ 0,
895
+ 1,
896
+ 1,
897
+ 1,
898
+ 0,
899
+ 1,
900
+ 1,
901
+ 1,
902
+ 1,
903
+ 0,
904
+ 0,
905
+ 0,
906
+ 1,
907
+ 0,
908
+ 1,
909
+ 1,
910
+ 1,
911
+ 1,
912
+ 0,
913
+ 1,
914
+ 1,
915
+ 1,
916
+ 1,
917
+ 1,
918
+ 1,
919
+ 1,
920
+ 1,
921
+ 1,
922
+ 1,
923
+ 1,
924
+ 1,
925
+ 0,
926
+ 1,
927
+ 1,
928
+ 1,
929
+ 1,
930
+ 0,
931
+ 1,
932
+ 1,
933
+ 1,
934
+ 1,
935
+ 0,
936
+ 1,
937
+ 1,
938
+ 1,
939
+ 0,
940
+ 1,
941
+ 1,
942
+ 1,
943
+ 1,
944
+ 1,
945
+ 1,
946
+ 1,
947
+ 1,
948
+ 1,
949
+ 0,
950
+ 0,
951
+ 1,
952
+ 1,
953
+ 1,
954
+ 1,
955
+ 1,
956
+ 0,
957
+ 1,
958
+ 1,
959
+ 1,
960
+ 1,
961
+ 1,
962
+ 1,
963
+ 1,
964
+ 1,
965
+ 1,
966
+ 1,
967
+ 0,
968
+ 0,
969
+ 0,
970
+ 0,
971
+ 1,
972
+ 1,
973
+ 1,
974
+ 0,
975
+ 1,
976
+ 1,
977
+ 1,
978
+ 1,
979
+ 0,
980
+ 1,
981
+ 0,
982
+ 1,
983
+ 1,
984
+ 1,
985
+ 1,
986
+ 1,
987
+ 0,
988
+ 1,
989
+ 1,
990
+ 1,
991
+ 0,
992
+ 0,
993
+ 1,
994
+ 0,
995
+ 1,
996
+ 1,
997
+ 1,
998
+ 1,
999
+ 1,
1000
+ 1,
1001
+ 1,
1002
+ 1,
1003
+ 0,
1004
+ 1,
1005
+ 0,
1006
+ 1,
1007
+ 1,
1008
+ 1,
1009
+ 1,
1010
+ 1,
1011
+ 1,
1012
+ 1,
1013
+ 1,
1014
+ 0,
1015
+ 1,
1016
+ 1,
1017
+ 1,
1018
+ 1,
1019
+ 1,
1020
+ 1,
1021
+ 1,
1022
+ 1,
1023
+ 1,
1024
+ 1,
1025
+ 1,
1026
+ 0,
1027
+ 1,
1028
+ 0,
1029
+ 0,
1030
+ 1,
1031
+ 0,
1032
+ 1,
1033
+ 1,
1034
+ 1,
1035
+ 1,
1036
+ 0,
1037
+ 1,
1038
+ 1,
1039
+ 1,
1040
+ 1,
1041
+ 1,
1042
+ 1,
1043
+ 1,
1044
+ 0,
1045
+ 0,
1046
+ 0,
1047
+ 1,
1048
+ 1,
1049
+ 1,
1050
+ 1,
1051
+ 1,
1052
+ 1,
1053
+ 1,
1054
+ 0,
1055
+ 0,
1056
+ 0,
1057
+ 0,
1058
+ 0,
1059
+ 1,
1060
+ 1,
1061
+ 1,
1062
+ 1,
1063
+ 0,
1064
+ 1,
1065
+ 1,
1066
+ 0,
1067
+ 1,
1068
+ 1,
1069
+ 1,
1070
+ 1,
1071
+ 1,
1072
+ 1,
1073
+ 1,
1074
+ 1,
1075
+ 1,
1076
+ 1,
1077
+ 1,
1078
+ 1,
1079
+ 1,
1080
+ 0,
1081
+ 1,
1082
+ 1,
1083
+ 0,
1084
+ 1,
1085
+ 1,
1086
+ 1,
1087
+ 1,
1088
+ 1,
1089
+ 1,
1090
+ 1,
1091
+ 1,
1092
+ 0,
1093
+ 1,
1094
+ 0,
1095
+ 0,
1096
+ 1,
1097
+ 1,
1098
+ 0,
1099
+ 1,
1100
+ 0,
1101
+ 1,
1102
+ 1,
1103
+ 1,
1104
+ 1,
1105
+ 1,
1106
+ 1,
1107
+ 1,
1108
+ 1,
1109
+ 0,
1110
+ 1,
1111
+ 1,
1112
+ 1,
1113
+ 1,
1114
+ 1,
1115
+ 1,
1116
+ 1,
1117
+ 1,
1118
+ 1,
1119
+ 1,
1120
+ 1,
1121
+ 0,
1122
+ 1,
1123
+ 1,
1124
+ 1,
1125
+ 1,
1126
+ 1,
1127
+ 1,
1128
+ 0,
1129
+ 0,
1130
+ 1,
1131
+ 1,
1132
+ 0,
1133
+ 1,
1134
+ 1,
1135
+ 1,
1136
+ 0,
1137
+ 0,
1138
+ 1,
1139
+ 0,
1140
+ 0,
1141
+ 1,
1142
+ 0,
1143
+ 1,
1144
+ 1,
1145
+ 0,
1146
+ 1,
1147
+ 1,
1148
+ 1,
1149
+ 1,
1150
+ 0,
1151
+ 0,
1152
+ 1,
1153
+ 1,
1154
+ 0,
1155
+ 0,
1156
+ 1,
1157
+ 1,
1158
+ 1,
1159
+ 1,
1160
+ 0,
1161
+ 1,
1162
+ 1,
1163
+ 1,
1164
+ 1,
1165
+ 0,
1166
+ 0,
1167
+ 1,
1168
+ 0,
1169
+ 1,
1170
+ 0,
1171
+ 0,
1172
+ 1,
1173
+ 1,
1174
+ 1,
1175
+ 1,
1176
+ 0,
1177
+ 0,
1178
+ 1,
1179
+ 1,
1180
+ 1,
1181
+ 1,
1182
+ 1,
1183
+ 0,
1184
+ 1,
1185
+ 1,
1186
+ 1,
1187
+ 1,
1188
+ 1,
1189
+ 0,
1190
+ 0,
1191
+ 1,
1192
+ 1,
1193
+ 1,
1194
+ 1,
1195
+ 1,
1196
+ 0,
1197
+ 1,
1198
+ 0,
1199
+ 1,
1200
+ 1,
1201
+ 1,
1202
+ 1,
1203
+ 0,
1204
+ 1,
1205
+ 1,
1206
+ 1,
1207
+ 1,
1208
+ 1,
1209
+ 1,
1210
+ 0,
1211
+ 1,
1212
+ 0,
1213
+ 1,
1214
+ 1,
1215
+ 1,
1216
+ 1,
1217
+ 1,
1218
+ 1,
1219
+ 1,
1220
+ 1,
1221
+ 1,
1222
+ 1,
1223
+ 1,
1224
+ 1,
1225
+ 1,
1226
+ 0,
1227
+ 1,
1228
+ 1,
1229
+ 1,
1230
+ 1,
1231
+ 1,
1232
+ 1,
1233
+ 1,
1234
+ 0,
1235
+ 1,
1236
+ 1,
1237
+ 1,
1238
+ 0,
1239
+ 1,
1240
+ 0,
1241
+ 1,
1242
+ 1,
1243
+ 1,
1244
+ 1,
1245
+ 1,
1246
+ 1,
1247
+ 1,
1248
+ 1,
1249
+ 1,
1250
+ 0,
1251
+ 1,
1252
+ 0,
1253
+ 1,
1254
+ 1,
1255
+ 1,
1256
+ 1,
1257
+ 0,
1258
+ 1,
1259
+ 1,
1260
+ 1,
1261
+ 1,
1262
+ 1,
1263
+ 1,
1264
+ 0,
1265
+ 1,
1266
+ 0,
1267
+ 0,
1268
+ 1,
1269
+ 0,
1270
+ 1,
1271
+ 1,
1272
+ 1,
1273
+ 1,
1274
+ 0,
1275
+ 1,
1276
+ 1,
1277
+ 1,
1278
+ 1,
1279
+ 1,
1280
+ 1,
1281
+ 0,
1282
+ 1,
1283
+ 1,
1284
+ 0,
1285
+ 1,
1286
+ 1,
1287
+ 1,
1288
+ 1,
1289
+ 1,
1290
+ 1,
1291
+ 0,
1292
+ 1,
1293
+ 1,
1294
+ 1,
1295
+ 0,
1296
+ 1,
1297
+ 1,
1298
+ 0,
1299
+ 1,
1300
+ 1,
1301
+ 0,
1302
+ 1,
1303
+ 0,
1304
+ 1,
1305
+ 1,
1306
+ 1,
1307
+ 1,
1308
+ 1,
1309
+ 1,
1310
+ 1,
1311
+ 1,
1312
+ 1,
1313
+ 1,
1314
+ 1,
1315
+ 1,
1316
+ 1,
1317
+ 1,
1318
+ 1,
1319
+ 1,
1320
+ 1,
1321
+ 0,
1322
+ 1,
1323
+ 1,
1324
+ 1,
1325
+ 1,
1326
+ 0,
1327
+ 1,
1328
+ 1,
1329
+ 1,
1330
+ 1,
1331
+ 0,
1332
+ 1,
1333
+ 1,
1334
+ 0,
1335
+ 1,
1336
+ 1,
1337
+ 0,
1338
+ 1,
1339
+ 1,
1340
+ 1,
1341
+ 1,
1342
+ 1,
1343
+ 1,
1344
+ 0,
1345
+ 1,
1346
+ 1,
1347
+ 1,
1348
+ 1,
1349
+ 0,
1350
+ 1,
1351
+ 0,
1352
+ 0,
1353
+ 0,
1354
+ 1,
1355
+ 0,
1356
+ 1,
1357
+ 1,
1358
+ 0,
1359
+ 1,
1360
+ 1,
1361
+ 1,
1362
+ 0,
1363
+ 1,
1364
+ 1,
1365
+ 1,
1366
+ 1,
1367
+ 1,
1368
+ 1,
1369
+ 1,
1370
+ 0,
1371
+ 0,
1372
+ 1,
1373
+ 1,
1374
+ 1,
1375
+ 1,
1376
+ 1,
1377
+ 1,
1378
+ 1,
1379
+ 1,
1380
+ 1,
1381
+ 0,
1382
+ 0,
1383
+ 1,
1384
+ 1,
1385
+ 0,
1386
+ 1,
1387
+ 0,
1388
+ 1,
1389
+ 0,
1390
+ 0,
1391
+ 1,
1392
+ 1,
1393
+ 1,
1394
+ 1,
1395
+ 1,
1396
+ 0,
1397
+ 0,
1398
+ 1,
1399
+ 1,
1400
+ 1,
1401
+ 1,
1402
+ 1,
1403
+ 1,
1404
+ 1,
1405
+ 1,
1406
+ 1,
1407
+ 0,
1408
+ 0,
1409
+ 1,
1410
+ 1,
1411
+ 1,
1412
+ 1,
1413
+ 0,
1414
+ 1,
1415
+ 0,
1416
+ 1,
1417
+ 1,
1418
+ 0,
1419
+ 1,
1420
+ 1,
1421
+ 1,
1422
+ 1,
1423
+ 1,
1424
+ 0,
1425
+ 0,
1426
+ 0,
1427
+ 1,
1428
+ 1,
1429
+ 1,
1430
+ 1,
1431
+ 1,
1432
+ 1,
1433
+ 1,
1434
+ 1,
1435
+ 1,
1436
+ 0,
1437
+ 1,
1438
+ 1,
1439
+ 0,
1440
+ 1,
1441
+ 1,
1442
+ 1,
1443
+ 1,
1444
+ 1,
1445
+ 0,
1446
+ 0,
1447
+ 0,
1448
+ 0,
1449
+ 1,
1450
+ 1,
1451
+ 1,
1452
+ 1,
1453
+ 1,
1454
+ 1,
1455
+ 1,
1456
+ 1,
1457
+ 1,
1458
+ 1,
1459
+ 0,
1460
+ 1,
1461
+ 1,
1462
+ 0,
1463
+ 1,
1464
+ 1,
1465
+ 0,
1466
+ 1,
1467
+ 1,
1468
+ 0,
1469
+ 1,
1470
+ 1,
1471
+ 1,
1472
+ 1,
1473
+ 1,
1474
+ 1,
1475
+ 1,
1476
+ 1,
1477
+ 0,
1478
+ 0,
1479
+ 1,
1480
+ 0,
1481
+ 1,
1482
+ 0,
1483
+ 1,
1484
+ 0,
1485
+ 1,
1486
+ 1,
1487
+ 1,
1488
+ 0,
1489
+ 1,
1490
+ 0,
1491
+ 1,
1492
+ 0,
1493
+ 0,
1494
+ 1,
1495
+ 1,
1496
+ 1,
1497
+ 1,
1498
+ 0,
1499
+ 0,
1500
+ 0,
1501
+ 1,
1502
+ 1,
1503
+ 1,
1504
+ 1
1505
+ ],
1506
+ "ood_8hop": [
1507
+ 1,
1508
+ 1,
1509
+ 1,
1510
+ 0,
1511
+ 1,
1512
+ 1,
1513
+ 1,
1514
+ 1,
1515
+ 1,
1516
+ 1,
1517
+ 1,
1518
+ 1,
1519
+ 1,
1520
+ 1,
1521
+ 1,
1522
+ 0,
1523
+ 1,
1524
+ 1,
1525
+ 0,
1526
+ 0,
1527
+ 1,
1528
+ 1,
1529
+ 1,
1530
+ 0,
1531
+ 1,
1532
+ 1,
1533
+ 1,
1534
+ 1,
1535
+ 1,
1536
+ 1,
1537
+ 1,
1538
+ 1,
1539
+ 0,
1540
+ 1,
1541
+ 0,
1542
+ 1,
1543
+ 1,
1544
+ 1,
1545
+ 0,
1546
+ 1,
1547
+ 0,
1548
+ 1,
1549
+ 1,
1550
+ 0,
1551
+ 1,
1552
+ 1,
1553
+ 1,
1554
+ 0,
1555
+ 1,
1556
+ 1,
1557
+ 1,
1558
+ 1,
1559
+ 1,
1560
+ 1,
1561
+ 1,
1562
+ 0,
1563
+ 1,
1564
+ 0,
1565
+ 1,
1566
+ 1,
1567
+ 0,
1568
+ 1,
1569
+ 1,
1570
+ 1,
1571
+ 1,
1572
+ 1,
1573
+ 1,
1574
+ 0,
1575
+ 1,
1576
+ 1,
1577
+ 1,
1578
+ 0,
1579
+ 1,
1580
+ 1,
1581
+ 1,
1582
+ 0,
1583
+ 1,
1584
+ 1,
1585
+ 0,
1586
+ 1,
1587
+ 0,
1588
+ 1,
1589
+ 0,
1590
+ 1,
1591
+ 1,
1592
+ 1,
1593
+ 1,
1594
+ 1,
1595
+ 1,
1596
+ 0,
1597
+ 1,
1598
+ 0,
1599
+ 1,
1600
+ 1,
1601
+ 1,
1602
+ 1,
1603
+ 0,
1604
+ 1,
1605
+ 1,
1606
+ 0,
1607
+ 1,
1608
+ 1,
1609
+ 1,
1610
+ 1,
1611
+ 1,
1612
+ 0,
1613
+ 1,
1614
+ 1,
1615
+ 1,
1616
+ 0,
1617
+ 1,
1618
+ 1,
1619
+ 1,
1620
+ 1,
1621
+ 1,
1622
+ 1,
1623
+ 1,
1624
+ 1,
1625
+ 0,
1626
+ 1,
1627
+ 1,
1628
+ 1,
1629
+ 1,
1630
+ 0,
1631
+ 1,
1632
+ 1,
1633
+ 1,
1634
+ 1,
1635
+ 1,
1636
+ 1,
1637
+ 1,
1638
+ 1,
1639
+ 1,
1640
+ 1,
1641
+ 1,
1642
+ 0,
1643
+ 1,
1644
+ 1,
1645
+ 0,
1646
+ 1,
1647
+ 1,
1648
+ 1,
1649
+ 0,
1650
+ 0,
1651
+ 1,
1652
+ 1,
1653
+ 1,
1654
+ 1,
1655
+ 0,
1656
+ 0,
1657
+ 1,
1658
+ 1,
1659
+ 1,
1660
+ 1,
1661
+ 0,
1662
+ 1,
1663
+ 1,
1664
+ 1,
1665
+ 1,
1666
+ 0,
1667
+ 0,
1668
+ 1,
1669
+ 1,
1670
+ 1,
1671
+ 1,
1672
+ 1,
1673
+ 1,
1674
+ 1,
1675
+ 1,
1676
+ 0,
1677
+ 1,
1678
+ 1,
1679
+ 1,
1680
+ 1,
1681
+ 1,
1682
+ 1,
1683
+ 0,
1684
+ 1,
1685
+ 1,
1686
+ 0,
1687
+ 0,
1688
+ 1,
1689
+ 1,
1690
+ 1,
1691
+ 1,
1692
+ 0,
1693
+ 1,
1694
+ 0,
1695
+ 1,
1696
+ 1,
1697
+ 1,
1698
+ 0,
1699
+ 0,
1700
+ 1,
1701
+ 0,
1702
+ 1,
1703
+ 1,
1704
+ 1,
1705
+ 1,
1706
+ 1,
1707
+ 0,
1708
+ 0,
1709
+ 1,
1710
+ 0,
1711
+ 1,
1712
+ 1,
1713
+ 1,
1714
+ 1,
1715
+ 1,
1716
+ 0,
1717
+ 1,
1718
+ 0,
1719
+ 1,
1720
+ 0,
1721
+ 1,
1722
+ 1,
1723
+ 1,
1724
+ 1,
1725
+ 1,
1726
+ 1,
1727
+ 0,
1728
+ 0,
1729
+ 0,
1730
+ 1,
1731
+ 1,
1732
+ 1,
1733
+ 1,
1734
+ 1,
1735
+ 0,
1736
+ 0,
1737
+ 1,
1738
+ 1,
1739
+ 1,
1740
+ 1,
1741
+ 0,
1742
+ 0,
1743
+ 0,
1744
+ 1,
1745
+ 0,
1746
+ 1,
1747
+ 1,
1748
+ 1,
1749
+ 0,
1750
+ 0,
1751
+ 1,
1752
+ 1,
1753
+ 0,
1754
+ 0,
1755
+ 0,
1756
+ 0,
1757
+ 0,
1758
+ 1,
1759
+ 0,
1760
+ 1,
1761
+ 0,
1762
+ 1,
1763
+ 1,
1764
+ 1,
1765
+ 1,
1766
+ 1,
1767
+ 0,
1768
+ 1,
1769
+ 0,
1770
+ 0,
1771
+ 1,
1772
+ 0,
1773
+ 1,
1774
+ 1,
1775
+ 1,
1776
+ 0,
1777
+ 0,
1778
+ 1,
1779
+ 1,
1780
+ 1,
1781
+ 1,
1782
+ 0,
1783
+ 1,
1784
+ 1,
1785
+ 1,
1786
+ 1,
1787
+ 1,
1788
+ 1,
1789
+ 1,
1790
+ 1,
1791
+ 0,
1792
+ 1,
1793
+ 1,
1794
+ 1,
1795
+ 1,
1796
+ 1,
1797
+ 1,
1798
+ 1,
1799
+ 1,
1800
+ 1,
1801
+ 1,
1802
+ 1,
1803
+ 1,
1804
+ 1,
1805
+ 1,
1806
+ 0,
1807
+ 0,
1808
+ 1,
1809
+ 1,
1810
+ 0,
1811
+ 1,
1812
+ 1,
1813
+ 1,
1814
+ 0,
1815
+ 1,
1816
+ 1,
1817
+ 1,
1818
+ 0,
1819
+ 1,
1820
+ 1,
1821
+ 1,
1822
+ 1,
1823
+ 0,
1824
+ 1,
1825
+ 1,
1826
+ 1,
1827
+ 1,
1828
+ 1,
1829
+ 1,
1830
+ 1,
1831
+ 0,
1832
+ 1,
1833
+ 0,
1834
+ 1,
1835
+ 1,
1836
+ 0,
1837
+ 0,
1838
+ 1,
1839
+ 1,
1840
+ 1,
1841
+ 1,
1842
+ 1,
1843
+ 1,
1844
+ 1,
1845
+ 1,
1846
+ 1,
1847
+ 1,
1848
+ 1,
1849
+ 1,
1850
+ 1,
1851
+ 1,
1852
+ 1,
1853
+ 1,
1854
+ 1,
1855
+ 1,
1856
+ 1,
1857
+ 1,
1858
+ 1,
1859
+ 1,
1860
+ 0,
1861
+ 0,
1862
+ 1,
1863
+ 1,
1864
+ 1,
1865
+ 0,
1866
+ 1,
1867
+ 1,
1868
+ 1,
1869
+ 1,
1870
+ 1,
1871
+ 1,
1872
+ 1,
1873
+ 0,
1874
+ 1,
1875
+ 0,
1876
+ 1,
1877
+ 0,
1878
+ 1,
1879
+ 1,
1880
+ 1,
1881
+ 1,
1882
+ 1,
1883
+ 1,
1884
+ 1,
1885
+ 1,
1886
+ 1,
1887
+ 0,
1888
+ 1,
1889
+ 1,
1890
+ 1,
1891
+ 1,
1892
+ 0,
1893
+ 0,
1894
+ 0,
1895
+ 1,
1896
+ 1,
1897
+ 0,
1898
+ 0,
1899
+ 0,
1900
+ 1,
1901
+ 0,
1902
+ 1,
1903
+ 1,
1904
+ 1,
1905
+ 0,
1906
+ 0,
1907
+ 1,
1908
+ 1,
1909
+ 0,
1910
+ 1,
1911
+ 1,
1912
+ 1,
1913
+ 1,
1914
+ 1,
1915
+ 1,
1916
+ 0,
1917
+ 1,
1918
+ 1,
1919
+ 0,
1920
+ 1,
1921
+ 1,
1922
+ 1,
1923
+ 1,
1924
+ 0,
1925
+ 1,
1926
+ 1,
1927
+ 1,
1928
+ 0,
1929
+ 1,
1930
+ 0,
1931
+ 1,
1932
+ 1,
1933
+ 1,
1934
+ 1,
1935
+ 1,
1936
+ 1,
1937
+ 1,
1938
+ 1,
1939
+ 1,
1940
+ 1,
1941
+ 1,
1942
+ 1,
1943
+ 1,
1944
+ 0,
1945
+ 1,
1946
+ 1,
1947
+ 1,
1948
+ 0,
1949
+ 1,
1950
+ 1,
1951
+ 0,
1952
+ 1,
1953
+ 0,
1954
+ 1,
1955
+ 1,
1956
+ 1,
1957
+ 0,
1958
+ 1,
1959
+ 1,
1960
+ 1,
1961
+ 1,
1962
+ 0,
1963
+ 1,
1964
+ 0,
1965
+ 1,
1966
+ 1,
1967
+ 1,
1968
+ 0,
1969
+ 1,
1970
+ 1,
1971
+ 0,
1972
+ 0,
1973
+ 0,
1974
+ 1,
1975
+ 1,
1976
+ 1,
1977
+ 1,
1978
+ 1,
1979
+ 1,
1980
+ 1,
1981
+ 1,
1982
+ 1,
1983
+ 1,
1984
+ 1,
1985
+ 1,
1986
+ 1,
1987
+ 1,
1988
+ 0,
1989
+ 1,
1990
+ 1,
1991
+ 1,
1992
+ 1,
1993
+ 1,
1994
+ 0,
1995
+ 1,
1996
+ 1,
1997
+ 1,
1998
+ 0,
1999
+ 1,
2000
+ 1,
2001
+ 1,
2002
+ 1,
2003
+ 1,
2004
+ 1,
2005
+ 1,
2006
+ 1,
2007
+ 1,
2008
+ 0,
2009
+ 1,
2010
+ 1,
2011
+ 1,
2012
+ 1,
2013
+ 1,
2014
+ 1,
2015
+ 1,
2016
+ 1,
2017
+ 1,
2018
+ 0,
2019
+ 1,
2020
+ 1,
2021
+ 1,
2022
+ 1,
2023
+ 1,
2024
+ 0,
2025
+ 1,
2026
+ 1,
2027
+ 1,
2028
+ 1,
2029
+ 1,
2030
+ 1,
2031
+ 1,
2032
+ 1,
2033
+ 1,
2034
+ 1,
2035
+ 1,
2036
+ 1,
2037
+ 1,
2038
+ 1,
2039
+ 1,
2040
+ 0,
2041
+ 0,
2042
+ 1,
2043
+ 1,
2044
+ 1,
2045
+ 1,
2046
+ 1,
2047
+ 0,
2048
+ 0,
2049
+ 1,
2050
+ 1,
2051
+ 1,
2052
+ 1,
2053
+ 1,
2054
+ 1,
2055
+ 0,
2056
+ 1,
2057
+ 0,
2058
+ 1,
2059
+ 0,
2060
+ 0,
2061
+ 1,
2062
+ 1,
2063
+ 1,
2064
+ 1,
2065
+ 1,
2066
+ 1,
2067
+ 0,
2068
+ 1,
2069
+ 0,
2070
+ 0,
2071
+ 1,
2072
+ 0,
2073
+ 1,
2074
+ 1,
2075
+ 0,
2076
+ 1,
2077
+ 1,
2078
+ 1,
2079
+ 1,
2080
+ 1,
2081
+ 0,
2082
+ 1,
2083
+ 0,
2084
+ 1,
2085
+ 0,
2086
+ 0,
2087
+ 1,
2088
+ 1,
2089
+ 1,
2090
+ 0,
2091
+ 1,
2092
+ 1,
2093
+ 1,
2094
+ 0,
2095
+ 1,
2096
+ 1,
2097
+ 1,
2098
+ 1,
2099
+ 1,
2100
+ 1,
2101
+ 1,
2102
+ 1,
2103
+ 0,
2104
+ 1,
2105
+ 1,
2106
+ 1,
2107
+ 1,
2108
+ 1,
2109
+ 0,
2110
+ 1,
2111
+ 1,
2112
+ 1,
2113
+ 1,
2114
+ 1,
2115
+ 1,
2116
+ 1,
2117
+ 0,
2118
+ 1,
2119
+ 0,
2120
+ 1,
2121
+ 1,
2122
+ 1,
2123
+ 1,
2124
+ 1,
2125
+ 0,
2126
+ 1,
2127
+ 1,
2128
+ 1,
2129
+ 0,
2130
+ 0,
2131
+ 1,
2132
+ 1,
2133
+ 0,
2134
+ 1,
2135
+ 1,
2136
+ 1,
2137
+ 0,
2138
+ 0,
2139
+ 1,
2140
+ 1,
2141
+ 1,
2142
+ 1,
2143
+ 0,
2144
+ 1,
2145
+ 1,
2146
+ 1,
2147
+ 0,
2148
+ 1,
2149
+ 1,
2150
+ 1,
2151
+ 1,
2152
+ 1,
2153
+ 1,
2154
+ 0,
2155
+ 1,
2156
+ 1,
2157
+ 1,
2158
+ 0,
2159
+ 1,
2160
+ 0,
2161
+ 0,
2162
+ 1,
2163
+ 1,
2164
+ 1,
2165
+ 1,
2166
+ 1,
2167
+ 1,
2168
+ 1,
2169
+ 1,
2170
+ 1,
2171
+ 0,
2172
+ 0,
2173
+ 1,
2174
+ 1,
2175
+ 1,
2176
+ 1,
2177
+ 1,
2178
+ 1,
2179
+ 0,
2180
+ 1,
2181
+ 1,
2182
+ 0,
2183
+ 0,
2184
+ 0,
2185
+ 1,
2186
+ 1,
2187
+ 0,
2188
+ 1,
2189
+ 1,
2190
+ 0,
2191
+ 1,
2192
+ 1,
2193
+ 1,
2194
+ 1,
2195
+ 0,
2196
+ 0,
2197
+ 1,
2198
+ 1,
2199
+ 1,
2200
+ 1,
2201
+ 1,
2202
+ 1,
2203
+ 1,
2204
+ 1,
2205
+ 1,
2206
+ 1,
2207
+ 1,
2208
+ 1,
2209
+ 1,
2210
+ 1,
2211
+ 1,
2212
+ 1,
2213
+ 1,
2214
+ 1,
2215
+ 1,
2216
+ 1,
2217
+ 1,
2218
+ 1,
2219
+ 1,
2220
+ 1,
2221
+ 1,
2222
+ 1,
2223
+ 1,
2224
+ 1,
2225
+ 1,
2226
+ 1,
2227
+ 1,
2228
+ 1,
2229
+ 1,
2230
+ 0,
2231
+ 0,
2232
+ 1,
2233
+ 0,
2234
+ 1,
2235
+ 0,
2236
+ 1,
2237
+ 1,
2238
+ 1,
2239
+ 0,
2240
+ 1,
2241
+ 0,
2242
+ 1,
2243
+ 1,
2244
+ 1,
2245
+ 1,
2246
+ 1,
2247
+ 1,
2248
+ 1,
2249
+ 1,
2250
+ 1,
2251
+ 1,
2252
+ 1,
2253
+ 1,
2254
+ 1,
2255
+ 1,
2256
+ 1,
2257
+ 1,
2258
+ 1,
2259
+ 1,
2260
+ 1,
2261
+ 1,
2262
+ 1,
2263
+ 1,
2264
+ 0,
2265
+ 0,
2266
+ 1,
2267
+ 0,
2268
+ 1,
2269
+ 0,
2270
+ 1,
2271
+ 1,
2272
+ 0,
2273
+ 1,
2274
+ 0,
2275
+ 0,
2276
+ 0,
2277
+ 1,
2278
+ 1,
2279
+ 1,
2280
+ 0,
2281
+ 1,
2282
+ 1,
2283
+ 0,
2284
+ 0,
2285
+ 0,
2286
+ 1,
2287
+ 1,
2288
+ 0,
2289
+ 0,
2290
+ 1,
2291
+ 0,
2292
+ 1,
2293
+ 1,
2294
+ 1,
2295
+ 1,
2296
+ 1,
2297
+ 1,
2298
+ 1,
2299
+ 0,
2300
+ 1,
2301
+ 1,
2302
+ 0,
2303
+ 1,
2304
+ 1,
2305
+ 1,
2306
+ 1,
2307
+ 1,
2308
+ 1,
2309
+ 1,
2310
+ 1,
2311
+ 0,
2312
+ 1,
2313
+ 1,
2314
+ 0,
2315
+ 1,
2316
+ 1,
2317
+ 1,
2318
+ 1,
2319
+ 1,
2320
+ 0,
2321
+ 1,
2322
+ 1,
2323
+ 1,
2324
+ 1,
2325
+ 1,
2326
+ 1,
2327
+ 0,
2328
+ 1,
2329
+ 1,
2330
+ 1,
2331
+ 1,
2332
+ 1,
2333
+ 0,
2334
+ 0,
2335
+ 1,
2336
+ 1,
2337
+ 1,
2338
+ 0,
2339
+ 1,
2340
+ 1,
2341
+ 0,
2342
+ 1,
2343
+ 1,
2344
+ 0,
2345
+ 1,
2346
+ 0,
2347
+ 1,
2348
+ 1,
2349
+ 1,
2350
+ 1,
2351
+ 1,
2352
+ 1,
2353
+ 0,
2354
+ 1,
2355
+ 1,
2356
+ 1,
2357
+ 1,
2358
+ 0,
2359
+ 1,
2360
+ 1,
2361
+ 0,
2362
+ 1,
2363
+ 0,
2364
+ 0,
2365
+ 1,
2366
+ 0,
2367
+ 1,
2368
+ 1,
2369
+ 1,
2370
+ 1,
2371
+ 1,
2372
+ 1,
2373
+ 0,
2374
+ 0,
2375
+ 1,
2376
+ 1,
2377
+ 0,
2378
+ 0,
2379
+ 0,
2380
+ 1,
2381
+ 1,
2382
+ 1,
2383
+ 0,
2384
+ 0,
2385
+ 1,
2386
+ 0,
2387
+ 1,
2388
+ 1,
2389
+ 1,
2390
+ 1,
2391
+ 0,
2392
+ 1,
2393
+ 1,
2394
+ 1,
2395
+ 1,
2396
+ 0,
2397
+ 1,
2398
+ 1,
2399
+ 1,
2400
+ 0,
2401
+ 0,
2402
+ 1,
2403
+ 0,
2404
+ 1,
2405
+ 1,
2406
+ 0,
2407
+ 1,
2408
+ 0,
2409
+ 0,
2410
+ 1,
2411
+ 1,
2412
+ 1,
2413
+ 0,
2414
+ 1,
2415
+ 0,
2416
+ 1,
2417
+ 0,
2418
+ 1,
2419
+ 0,
2420
+ 1,
2421
+ 0,
2422
+ 1,
2423
+ 1,
2424
+ 1,
2425
+ 1,
2426
+ 1,
2427
+ 1,
2428
+ 0,
2429
+ 0,
2430
+ 1,
2431
+ 1,
2432
+ 0,
2433
+ 1,
2434
+ 1,
2435
+ 1,
2436
+ 0,
2437
+ 0,
2438
+ 1,
2439
+ 1,
2440
+ 1,
2441
+ 0,
2442
+ 1,
2443
+ 1,
2444
+ 0,
2445
+ 1,
2446
+ 1,
2447
+ 1,
2448
+ 1,
2449
+ 1,
2450
+ 0,
2451
+ 0,
2452
+ 1,
2453
+ 1,
2454
+ 1,
2455
+ 1,
2456
+ 1,
2457
+ 1,
2458
+ 0,
2459
+ 1,
2460
+ 0,
2461
+ 1,
2462
+ 1,
2463
+ 1,
2464
+ 0,
2465
+ 1,
2466
+ 1,
2467
+ 1,
2468
+ 0,
2469
+ 1,
2470
+ 1,
2471
+ 1,
2472
+ 1,
2473
+ 1,
2474
+ 1,
2475
+ 1,
2476
+ 0,
2477
+ 0,
2478
+ 1,
2479
+ 1,
2480
+ 1,
2481
+ 1,
2482
+ 1,
2483
+ 1,
2484
+ 1,
2485
+ 1,
2486
+ 1,
2487
+ 1,
2488
+ 1,
2489
+ 1,
2490
+ 1,
2491
+ 1,
2492
+ 1,
2493
+ 1,
2494
+ 1,
2495
+ 1,
2496
+ 1,
2497
+ 1,
2498
+ 1,
2499
+ 0,
2500
+ 1,
2501
+ 1,
2502
+ 1,
2503
+ 1,
2504
+ 1,
2505
+ 1,
2506
+ 1
2507
+ ],
2508
+ "ood_dag": [
2509
+ 1,
2510
+ 0,
2511
+ 0,
2512
+ 1,
2513
+ 1,
2514
+ 1,
2515
+ 1,
2516
+ 1,
2517
+ 0,
2518
+ 1,
2519
+ 0,
2520
+ 1,
2521
+ 1,
2522
+ 0,
2523
+ 1,
2524
+ 0,
2525
+ 0,
2526
+ 0,
2527
+ 1,
2528
+ 1,
2529
+ 1,
2530
+ 1,
2531
+ 1,
2532
+ 0,
2533
+ 0,
2534
+ 0,
2535
+ 1,
2536
+ 1,
2537
+ 1,
2538
+ 1,
2539
+ 1,
2540
+ 1,
2541
+ 0,
2542
+ 1,
2543
+ 0,
2544
+ 1,
2545
+ 0,
2546
+ 1,
2547
+ 1,
2548
+ 1,
2549
+ 0,
2550
+ 1,
2551
+ 0,
2552
+ 1,
2553
+ 1,
2554
+ 0,
2555
+ 0,
2556
+ 1,
2557
+ 0,
2558
+ 0,
2559
+ 1,
2560
+ 0,
2561
+ 1,
2562
+ 1,
2563
+ 1,
2564
+ 0,
2565
+ 0,
2566
+ 1,
2567
+ 0,
2568
+ 1,
2569
+ 1,
2570
+ 0,
2571
+ 0,
2572
+ 1,
2573
+ 1,
2574
+ 0,
2575
+ 1,
2576
+ 0,
2577
+ 1,
2578
+ 1,
2579
+ 1,
2580
+ 1,
2581
+ 0,
2582
+ 0,
2583
+ 0,
2584
+ 1,
2585
+ 1,
2586
+ 1,
2587
+ 1,
2588
+ 0,
2589
+ 1,
2590
+ 1,
2591
+ 0,
2592
+ 0,
2593
+ 1,
2594
+ 0,
2595
+ 1,
2596
+ 0,
2597
+ 0,
2598
+ 1,
2599
+ 0,
2600
+ 1,
2601
+ 0,
2602
+ 1,
2603
+ 1,
2604
+ 1,
2605
+ 1,
2606
+ 1,
2607
+ 0,
2608
+ 1,
2609
+ 1,
2610
+ 1,
2611
+ 1,
2612
+ 0,
2613
+ 1,
2614
+ 1,
2615
+ 0,
2616
+ 0,
2617
+ 1,
2618
+ 1,
2619
+ 1,
2620
+ 1,
2621
+ 0,
2622
+ 0,
2623
+ 1,
2624
+ 0,
2625
+ 1,
2626
+ 0,
2627
+ 1,
2628
+ 1,
2629
+ 0,
2630
+ 1,
2631
+ 1,
2632
+ 1,
2633
+ 1,
2634
+ 1,
2635
+ 0,
2636
+ 0,
2637
+ 1,
2638
+ 0,
2639
+ 0,
2640
+ 1,
2641
+ 0,
2642
+ 1,
2643
+ 0,
2644
+ 1,
2645
+ 0,
2646
+ 0,
2647
+ 1,
2648
+ 1,
2649
+ 1,
2650
+ 1,
2651
+ 1,
2652
+ 0,
2653
+ 1,
2654
+ 1,
2655
+ 0,
2656
+ 1,
2657
+ 1,
2658
+ 0,
2659
+ 1,
2660
+ 0,
2661
+ 0,
2662
+ 1,
2663
+ 1,
2664
+ 1,
2665
+ 1,
2666
+ 0,
2667
+ 1,
2668
+ 0,
2669
+ 1,
2670
+ 1,
2671
+ 1,
2672
+ 1,
2673
+ 1,
2674
+ 1,
2675
+ 1,
2676
+ 1,
2677
+ 1,
2678
+ 0,
2679
+ 1,
2680
+ 1,
2681
+ 0,
2682
+ 1,
2683
+ 0,
2684
+ 1,
2685
+ 1,
2686
+ 1,
2687
+ 0,
2688
+ 0,
2689
+ 1,
2690
+ 1,
2691
+ 0,
2692
+ 1,
2693
+ 1,
2694
+ 0,
2695
+ 1,
2696
+ 1,
2697
+ 0,
2698
+ 1,
2699
+ 1,
2700
+ 0,
2701
+ 1,
2702
+ 0,
2703
+ 0,
2704
+ 1,
2705
+ 0,
2706
+ 0,
2707
+ 0,
2708
+ 0,
2709
+ 0,
2710
+ 1,
2711
+ 1,
2712
+ 0,
2713
+ 0,
2714
+ 1,
2715
+ 0,
2716
+ 1,
2717
+ 0,
2718
+ 1,
2719
+ 0,
2720
+ 0,
2721
+ 1,
2722
+ 0,
2723
+ 1,
2724
+ 1,
2725
+ 1,
2726
+ 1,
2727
+ 0,
2728
+ 0,
2729
+ 0,
2730
+ 1,
2731
+ 0,
2732
+ 1,
2733
+ 0,
2734
+ 1,
2735
+ 0,
2736
+ 1,
2737
+ 1,
2738
+ 0,
2739
+ 1,
2740
+ 1,
2741
+ 0,
2742
+ 1,
2743
+ 1,
2744
+ 0,
2745
+ 0,
2746
+ 0,
2747
+ 0,
2748
+ 1,
2749
+ 1,
2750
+ 1,
2751
+ 1,
2752
+ 0,
2753
+ 1,
2754
+ 1,
2755
+ 0,
2756
+ 1,
2757
+ 1,
2758
+ 1,
2759
+ 1,
2760
+ 0,
2761
+ 0,
2762
+ 1,
2763
+ 0,
2764
+ 0,
2765
+ 0,
2766
+ 0,
2767
+ 1,
2768
+ 1,
2769
+ 1,
2770
+ 0,
2771
+ 0,
2772
+ 1,
2773
+ 0,
2774
+ 0,
2775
+ 1,
2776
+ 1,
2777
+ 1,
2778
+ 0,
2779
+ 0,
2780
+ 0,
2781
+ 0,
2782
+ 1,
2783
+ 1,
2784
+ 1,
2785
+ 0,
2786
+ 0,
2787
+ 0,
2788
+ 1,
2789
+ 0,
2790
+ 1,
2791
+ 0,
2792
+ 1,
2793
+ 0,
2794
+ 0,
2795
+ 1,
2796
+ 1,
2797
+ 1,
2798
+ 1,
2799
+ 1,
2800
+ 1,
2801
+ 0,
2802
+ 1,
2803
+ 0,
2804
+ 1,
2805
+ 0,
2806
+ 1,
2807
+ 1,
2808
+ 0,
2809
+ 1,
2810
+ 0,
2811
+ 1,
2812
+ 0,
2813
+ 1,
2814
+ 1,
2815
+ 1,
2816
+ 0,
2817
+ 1,
2818
+ 1,
2819
+ 1,
2820
+ 0,
2821
+ 1,
2822
+ 1,
2823
+ 1,
2824
+ 1,
2825
+ 0,
2826
+ 1,
2827
+ 0,
2828
+ 1,
2829
+ 1,
2830
+ 0,
2831
+ 1,
2832
+ 1,
2833
+ 1,
2834
+ 1,
2835
+ 0,
2836
+ 0,
2837
+ 1,
2838
+ 0,
2839
+ 1,
2840
+ 1,
2841
+ 0,
2842
+ 1,
2843
+ 1,
2844
+ 0,
2845
+ 0,
2846
+ 1,
2847
+ 1,
2848
+ 1,
2849
+ 0,
2850
+ 0,
2851
+ 1,
2852
+ 0,
2853
+ 0,
2854
+ 0,
2855
+ 1,
2856
+ 1,
2857
+ 1,
2858
+ 1,
2859
+ 1,
2860
+ 1,
2861
+ 1,
2862
+ 1,
2863
+ 1,
2864
+ 0,
2865
+ 1,
2866
+ 1,
2867
+ 1,
2868
+ 1,
2869
+ 1,
2870
+ 1,
2871
+ 0,
2872
+ 0,
2873
+ 1,
2874
+ 0,
2875
+ 1,
2876
+ 0,
2877
+ 1,
2878
+ 0,
2879
+ 0,
2880
+ 0,
2881
+ 0,
2882
+ 1,
2883
+ 0,
2884
+ 1,
2885
+ 1,
2886
+ 1,
2887
+ 1,
2888
+ 1,
2889
+ 1,
2890
+ 1,
2891
+ 1,
2892
+ 1,
2893
+ 1,
2894
+ 1,
2895
+ 0,
2896
+ 0,
2897
+ 0,
2898
+ 1,
2899
+ 1,
2900
+ 0,
2901
+ 1,
2902
+ 1,
2903
+ 0,
2904
+ 0,
2905
+ 1,
2906
+ 1,
2907
+ 0,
2908
+ 1,
2909
+ 0,
2910
+ 1,
2911
+ 0,
2912
+ 1,
2913
+ 0,
2914
+ 1,
2915
+ 0,
2916
+ 0,
2917
+ 0,
2918
+ 1,
2919
+ 1,
2920
+ 1,
2921
+ 1,
2922
+ 0,
2923
+ 0,
2924
+ 1,
2925
+ 1,
2926
+ 0,
2927
+ 0,
2928
+ 1,
2929
+ 1,
2930
+ 0,
2931
+ 0,
2932
+ 1,
2933
+ 1,
2934
+ 0,
2935
+ 1,
2936
+ 0,
2937
+ 0,
2938
+ 1,
2939
+ 1,
2940
+ 1,
2941
+ 0,
2942
+ 1,
2943
+ 0,
2944
+ 1,
2945
+ 0,
2946
+ 1,
2947
+ 0,
2948
+ 1,
2949
+ 1,
2950
+ 0,
2951
+ 0,
2952
+ 0,
2953
+ 0,
2954
+ 1,
2955
+ 1,
2956
+ 0,
2957
+ 1,
2958
+ 0,
2959
+ 1,
2960
+ 1,
2961
+ 1,
2962
+ 1,
2963
+ 1,
2964
+ 1,
2965
+ 0,
2966
+ 1,
2967
+ 1,
2968
+ 1,
2969
+ 0,
2970
+ 0,
2971
+ 1,
2972
+ 0,
2973
+ 0,
2974
+ 1,
2975
+ 0,
2976
+ 1,
2977
+ 1,
2978
+ 0,
2979
+ 0,
2980
+ 1,
2981
+ 0,
2982
+ 0,
2983
+ 1,
2984
+ 1,
2985
+ 0,
2986
+ 1,
2987
+ 1,
2988
+ 0,
2989
+ 1,
2990
+ 1,
2991
+ 0,
2992
+ 1,
2993
+ 1,
2994
+ 1,
2995
+ 1,
2996
+ 1,
2997
+ 0,
2998
+ 0,
2999
+ 1,
3000
+ 0,
3001
+ 0,
3002
+ 1,
3003
+ 0,
3004
+ 0,
3005
+ 0,
3006
+ 0,
3007
+ 0,
3008
+ 0,
3009
+ 0,
3010
+ 0,
3011
+ 0,
3012
+ 0,
3013
+ 1,
3014
+ 0,
3015
+ 1,
3016
+ 1,
3017
+ 0,
3018
+ 1,
3019
+ 0,
3020
+ 1,
3021
+ 1,
3022
+ 1,
3023
+ 0,
3024
+ 1,
3025
+ 1,
3026
+ 0,
3027
+ 1,
3028
+ 1,
3029
+ 1,
3030
+ 0,
3031
+ 1,
3032
+ 1,
3033
+ 0,
3034
+ 1,
3035
+ 1,
3036
+ 0,
3037
+ 1,
3038
+ 1,
3039
+ 1,
3040
+ 0,
3041
+ 0,
3042
+ 1,
3043
+ 1,
3044
+ 1,
3045
+ 1,
3046
+ 1,
3047
+ 0,
3048
+ 1,
3049
+ 1,
3050
+ 0,
3051
+ 0,
3052
+ 1,
3053
+ 1,
3054
+ 0,
3055
+ 1,
3056
+ 1,
3057
+ 1,
3058
+ 0,
3059
+ 0,
3060
+ 0,
3061
+ 1,
3062
+ 1,
3063
+ 1,
3064
+ 1,
3065
+ 1,
3066
+ 0,
3067
+ 1,
3068
+ 1,
3069
+ 1,
3070
+ 1,
3071
+ 0,
3072
+ 1,
3073
+ 1,
3074
+ 1,
3075
+ 1,
3076
+ 0,
3077
+ 1,
3078
+ 1,
3079
+ 1,
3080
+ 1,
3081
+ 1,
3082
+ 1,
3083
+ 0,
3084
+ 0,
3085
+ 1,
3086
+ 1,
3087
+ 1,
3088
+ 1,
3089
+ 0,
3090
+ 0,
3091
+ 1,
3092
+ 1,
3093
+ 1,
3094
+ 1,
3095
+ 1,
3096
+ 1,
3097
+ 1,
3098
+ 0,
3099
+ 0,
3100
+ 0,
3101
+ 0,
3102
+ 1,
3103
+ 1,
3104
+ 1,
3105
+ 0,
3106
+ 1,
3107
+ 0,
3108
+ 1,
3109
+ 0,
3110
+ 1,
3111
+ 1,
3112
+ 0,
3113
+ 0,
3114
+ 1,
3115
+ 1,
3116
+ 0,
3117
+ 0,
3118
+ 1,
3119
+ 0,
3120
+ 0,
3121
+ 1,
3122
+ 0,
3123
+ 1,
3124
+ 1,
3125
+ 0,
3126
+ 1,
3127
+ 1,
3128
+ 1,
3129
+ 1,
3130
+ 1,
3131
+ 1,
3132
+ 0,
3133
+ 1,
3134
+ 1,
3135
+ 0,
3136
+ 0,
3137
+ 0,
3138
+ 1,
3139
+ 0,
3140
+ 1,
3141
+ 1,
3142
+ 1,
3143
+ 0,
3144
+ 0,
3145
+ 1,
3146
+ 1,
3147
+ 1,
3148
+ 1,
3149
+ 1,
3150
+ 1,
3151
+ 1,
3152
+ 1,
3153
+ 1,
3154
+ 1,
3155
+ 0,
3156
+ 0,
3157
+ 0,
3158
+ 0,
3159
+ 1,
3160
+ 1,
3161
+ 1,
3162
+ 1,
3163
+ 0,
3164
+ 0,
3165
+ 1,
3166
+ 1,
3167
+ 0,
3168
+ 1,
3169
+ 1,
3170
+ 1,
3171
+ 0,
3172
+ 1,
3173
+ 1,
3174
+ 1,
3175
+ 1,
3176
+ 1,
3177
+ 1,
3178
+ 0,
3179
+ 1,
3180
+ 1,
3181
+ 0,
3182
+ 1,
3183
+ 1,
3184
+ 1,
3185
+ 0,
3186
+ 1,
3187
+ 1,
3188
+ 1,
3189
+ 1,
3190
+ 1,
3191
+ 1,
3192
+ 0,
3193
+ 0,
3194
+ 0,
3195
+ 1,
3196
+ 0,
3197
+ 0,
3198
+ 0,
3199
+ 0,
3200
+ 1,
3201
+ 1,
3202
+ 1,
3203
+ 1,
3204
+ 1,
3205
+ 0,
3206
+ 1,
3207
+ 0,
3208
+ 0,
3209
+ 1,
3210
+ 1,
3211
+ 0,
3212
+ 1,
3213
+ 1,
3214
+ 0,
3215
+ 1,
3216
+ 1,
3217
+ 0,
3218
+ 1,
3219
+ 0,
3220
+ 1,
3221
+ 0,
3222
+ 0,
3223
+ 1,
3224
+ 0,
3225
+ 1,
3226
+ 1,
3227
+ 1,
3228
+ 0,
3229
+ 1,
3230
+ 1,
3231
+ 1,
3232
+ 1,
3233
+ 1,
3234
+ 1,
3235
+ 0,
3236
+ 1,
3237
+ 0,
3238
+ 1,
3239
+ 0,
3240
+ 0,
3241
+ 1,
3242
+ 1,
3243
+ 0,
3244
+ 1,
3245
+ 0,
3246
+ 0,
3247
+ 1,
3248
+ 0,
3249
+ 1,
3250
+ 0,
3251
+ 1,
3252
+ 1,
3253
+ 1,
3254
+ 1,
3255
+ 0,
3256
+ 1,
3257
+ 0,
3258
+ 1,
3259
+ 1,
3260
+ 1,
3261
+ 0,
3262
+ 0,
3263
+ 0,
3264
+ 1,
3265
+ 1,
3266
+ 0,
3267
+ 0,
3268
+ 1,
3269
+ 0,
3270
+ 0,
3271
+ 0,
3272
+ 1,
3273
+ 0,
3274
+ 1,
3275
+ 1,
3276
+ 0,
3277
+ 1,
3278
+ 0,
3279
+ 0,
3280
+ 1,
3281
+ 1,
3282
+ 1,
3283
+ 0,
3284
+ 1,
3285
+ 1,
3286
+ 1,
3287
+ 0,
3288
+ 0,
3289
+ 1,
3290
+ 0,
3291
+ 1,
3292
+ 1,
3293
+ 1,
3294
+ 1,
3295
+ 1,
3296
+ 1,
3297
+ 1,
3298
+ 1,
3299
+ 0,
3300
+ 0,
3301
+ 1,
3302
+ 0,
3303
+ 1,
3304
+ 1,
3305
+ 1,
3306
+ 1,
3307
+ 0,
3308
+ 1,
3309
+ 0,
3310
+ 1,
3311
+ 1,
3312
+ 0,
3313
+ 1,
3314
+ 0,
3315
+ 0,
3316
+ 1,
3317
+ 0,
3318
+ 0,
3319
+ 0,
3320
+ 1,
3321
+ 1,
3322
+ 1,
3323
+ 0,
3324
+ 1,
3325
+ 1,
3326
+ 1,
3327
+ 1,
3328
+ 1,
3329
+ 0,
3330
+ 0,
3331
+ 1,
3332
+ 1,
3333
+ 1,
3334
+ 0,
3335
+ 0,
3336
+ 1,
3337
+ 0,
3338
+ 1,
3339
+ 1,
3340
+ 0,
3341
+ 0,
3342
+ 0,
3343
+ 1,
3344
+ 1,
3345
+ 1,
3346
+ 1,
3347
+ 1,
3348
+ 1,
3349
+ 0,
3350
+ 1,
3351
+ 1,
3352
+ 0,
3353
+ 1,
3354
+ 1,
3355
+ 0,
3356
+ 0,
3357
+ 1,
3358
+ 1,
3359
+ 1,
3360
+ 0,
3361
+ 1,
3362
+ 1,
3363
+ 1,
3364
+ 1,
3365
+ 0,
3366
+ 0,
3367
+ 1,
3368
+ 0,
3369
+ 0,
3370
+ 1,
3371
+ 0,
3372
+ 1,
3373
+ 1,
3374
+ 1,
3375
+ 0,
3376
+ 1,
3377
+ 1,
3378
+ 1,
3379
+ 0,
3380
+ 1,
3381
+ 0,
3382
+ 0,
3383
+ 1,
3384
+ 1,
3385
+ 1,
3386
+ 0,
3387
+ 1,
3388
+ 1,
3389
+ 0,
3390
+ 1,
3391
+ 0,
3392
+ 1,
3393
+ 0,
3394
+ 0,
3395
+ 0,
3396
+ 0,
3397
+ 1,
3398
+ 1,
3399
+ 0,
3400
+ 0,
3401
+ 1,
3402
+ 1,
3403
+ 1,
3404
+ 0,
3405
+ 0,
3406
+ 1,
3407
+ 0,
3408
+ 1,
3409
+ 1,
3410
+ 0,
3411
+ 1,
3412
+ 1,
3413
+ 1,
3414
+ 0,
3415
+ 1,
3416
+ 1,
3417
+ 0,
3418
+ 0,
3419
+ 1,
3420
+ 1,
3421
+ 1,
3422
+ 1,
3423
+ 0,
3424
+ 1,
3425
+ 1,
3426
+ 0,
3427
+ 1,
3428
+ 0,
3429
+ 0,
3430
+ 0,
3431
+ 0,
3432
+ 1,
3433
+ 1,
3434
+ 1,
3435
+ 0,
3436
+ 1,
3437
+ 1,
3438
+ 1,
3439
+ 1,
3440
+ 1,
3441
+ 0,
3442
+ 1,
3443
+ 0,
3444
+ 1,
3445
+ 1,
3446
+ 1,
3447
+ 1,
3448
+ 1,
3449
+ 1,
3450
+ 1,
3451
+ 0,
3452
+ 0,
3453
+ 1,
3454
+ 0,
3455
+ 1,
3456
+ 0,
3457
+ 1,
3458
+ 0,
3459
+ 0,
3460
+ 0,
3461
+ 1,
3462
+ 0,
3463
+ 0,
3464
+ 1,
3465
+ 0,
3466
+ 1,
3467
+ 0,
3468
+ 0,
3469
+ 1,
3470
+ 1,
3471
+ 0,
3472
+ 0,
3473
+ 1,
3474
+ 1,
3475
+ 0,
3476
+ 1,
3477
+ 0,
3478
+ 1,
3479
+ 1,
3480
+ 1,
3481
+ 1,
3482
+ 1,
3483
+ 0,
3484
+ 1,
3485
+ 0,
3486
+ 0,
3487
+ 1,
3488
+ 0,
3489
+ 0,
3490
+ 0,
3491
+ 1,
3492
+ 0,
3493
+ 0,
3494
+ 1,
3495
+ 1,
3496
+ 0,
3497
+ 0,
3498
+ 1,
3499
+ 1,
3500
+ 1,
3501
+ 0,
3502
+ 0,
3503
+ 1,
3504
+ 1,
3505
+ 0,
3506
+ 1,
3507
+ 1,
3508
+ 1
3509
+ ],
3510
+ "ood_dense": [
3511
+ 1,
3512
+ 0,
3513
+ 1,
3514
+ 1,
3515
+ 0,
3516
+ 0,
3517
+ 0,
3518
+ 1,
3519
+ 1,
3520
+ 1,
3521
+ 1,
3522
+ 1,
3523
+ 0,
3524
+ 1,
3525
+ 0,
3526
+ 0,
3527
+ 0,
3528
+ 0,
3529
+ 0,
3530
+ 0,
3531
+ 1,
3532
+ 1,
3533
+ 1,
3534
+ 0,
3535
+ 1,
3536
+ 1,
3537
+ 0,
3538
+ 0,
3539
+ 1,
3540
+ 1,
3541
+ 0,
3542
+ 0,
3543
+ 1,
3544
+ 1,
3545
+ 0,
3546
+ 1,
3547
+ 0,
3548
+ 1,
3549
+ 0,
3550
+ 1,
3551
+ 1,
3552
+ 0,
3553
+ 0,
3554
+ 0,
3555
+ 1,
3556
+ 1,
3557
+ 0,
3558
+ 0,
3559
+ 1,
3560
+ 0,
3561
+ 0,
3562
+ 1,
3563
+ 1,
3564
+ 0,
3565
+ 1,
3566
+ 1,
3567
+ 0,
3568
+ 1,
3569
+ 0,
3570
+ 0,
3571
+ 1,
3572
+ 1,
3573
+ 1,
3574
+ 1,
3575
+ 0,
3576
+ 0,
3577
+ 0,
3578
+ 1,
3579
+ 1,
3580
+ 1,
3581
+ 1,
3582
+ 0,
3583
+ 1,
3584
+ 0,
3585
+ 0,
3586
+ 0,
3587
+ 1,
3588
+ 1,
3589
+ 0,
3590
+ 0,
3591
+ 1,
3592
+ 1,
3593
+ 1,
3594
+ 1,
3595
+ 0,
3596
+ 0,
3597
+ 1,
3598
+ 0,
3599
+ 1,
3600
+ 0,
3601
+ 1,
3602
+ 0,
3603
+ 1,
3604
+ 0,
3605
+ 1,
3606
+ 1,
3607
+ 0,
3608
+ 0,
3609
+ 1,
3610
+ 1,
3611
+ 1,
3612
+ 1,
3613
+ 0,
3614
+ 0,
3615
+ 0,
3616
+ 0,
3617
+ 0,
3618
+ 0,
3619
+ 0,
3620
+ 0,
3621
+ 0,
3622
+ 1,
3623
+ 1,
3624
+ 1,
3625
+ 0,
3626
+ 1,
3627
+ 0,
3628
+ 1,
3629
+ 1,
3630
+ 1,
3631
+ 1,
3632
+ 0,
3633
+ 0,
3634
+ 1,
3635
+ 1,
3636
+ 0,
3637
+ 1,
3638
+ 1,
3639
+ 1,
3640
+ 0,
3641
+ 1,
3642
+ 1,
3643
+ 1,
3644
+ 1,
3645
+ 1,
3646
+ 1,
3647
+ 0,
3648
+ 1,
3649
+ 0,
3650
+ 0,
3651
+ 1,
3652
+ 1,
3653
+ 1,
3654
+ 1,
3655
+ 1,
3656
+ 1,
3657
+ 1,
3658
+ 0,
3659
+ 0,
3660
+ 1,
3661
+ 1,
3662
+ 0,
3663
+ 1,
3664
+ 0,
3665
+ 1,
3666
+ 1,
3667
+ 1,
3668
+ 0,
3669
+ 1,
3670
+ 1,
3671
+ 1,
3672
+ 1,
3673
+ 1,
3674
+ 0,
3675
+ 1,
3676
+ 0,
3677
+ 0,
3678
+ 1,
3679
+ 0,
3680
+ 1,
3681
+ 1,
3682
+ 1,
3683
+ 0,
3684
+ 0,
3685
+ 0,
3686
+ 1,
3687
+ 0,
3688
+ 1,
3689
+ 1,
3690
+ 1,
3691
+ 1,
3692
+ 1,
3693
+ 1,
3694
+ 1,
3695
+ 0,
3696
+ 1,
3697
+ 1,
3698
+ 0,
3699
+ 0,
3700
+ 0,
3701
+ 0,
3702
+ 1,
3703
+ 1,
3704
+ 1,
3705
+ 0,
3706
+ 1,
3707
+ 1,
3708
+ 1,
3709
+ 1,
3710
+ 1,
3711
+ 1,
3712
+ 1,
3713
+ 1,
3714
+ 1,
3715
+ 1,
3716
+ 1,
3717
+ 1,
3718
+ 0,
3719
+ 1,
3720
+ 1,
3721
+ 1,
3722
+ 1,
3723
+ 0,
3724
+ 0,
3725
+ 1,
3726
+ 0,
3727
+ 1,
3728
+ 0,
3729
+ 1,
3730
+ 1,
3731
+ 0,
3732
+ 0,
3733
+ 1,
3734
+ 0,
3735
+ 1,
3736
+ 0,
3737
+ 1,
3738
+ 0,
3739
+ 1,
3740
+ 0,
3741
+ 0,
3742
+ 1,
3743
+ 1,
3744
+ 1,
3745
+ 1,
3746
+ 0,
3747
+ 1,
3748
+ 1,
3749
+ 0,
3750
+ 1,
3751
+ 1,
3752
+ 1,
3753
+ 1,
3754
+ 1,
3755
+ 0,
3756
+ 1,
3757
+ 1,
3758
+ 1,
3759
+ 0,
3760
+ 0,
3761
+ 0,
3762
+ 1,
3763
+ 1,
3764
+ 0,
3765
+ 0,
3766
+ 0,
3767
+ 1,
3768
+ 1,
3769
+ 1,
3770
+ 0,
3771
+ 1,
3772
+ 0,
3773
+ 0,
3774
+ 0,
3775
+ 1,
3776
+ 1,
3777
+ 1,
3778
+ 1,
3779
+ 1,
3780
+ 1,
3781
+ 1,
3782
+ 0,
3783
+ 1,
3784
+ 0,
3785
+ 1,
3786
+ 1,
3787
+ 1,
3788
+ 1,
3789
+ 1,
3790
+ 1,
3791
+ 0,
3792
+ 0,
3793
+ 1,
3794
+ 1,
3795
+ 0,
3796
+ 1,
3797
+ 0,
3798
+ 0,
3799
+ 0,
3800
+ 1,
3801
+ 1,
3802
+ 1,
3803
+ 1,
3804
+ 0,
3805
+ 0,
3806
+ 1,
3807
+ 1,
3808
+ 1,
3809
+ 1,
3810
+ 1,
3811
+ 0,
3812
+ 0,
3813
+ 1,
3814
+ 0,
3815
+ 1,
3816
+ 1,
3817
+ 1,
3818
+ 0,
3819
+ 1,
3820
+ 0,
3821
+ 1,
3822
+ 0,
3823
+ 1,
3824
+ 1,
3825
+ 1,
3826
+ 0,
3827
+ 1,
3828
+ 1,
3829
+ 0,
3830
+ 1,
3831
+ 0,
3832
+ 1,
3833
+ 1,
3834
+ 0,
3835
+ 1,
3836
+ 0,
3837
+ 0,
3838
+ 1,
3839
+ 1,
3840
+ 0,
3841
+ 1,
3842
+ 1,
3843
+ 1,
3844
+ 1,
3845
+ 0,
3846
+ 1,
3847
+ 1,
3848
+ 0,
3849
+ 1,
3850
+ 0,
3851
+ 0,
3852
+ 1,
3853
+ 1,
3854
+ 1,
3855
+ 1,
3856
+ 0,
3857
+ 0,
3858
+ 1,
3859
+ 1,
3860
+ 1,
3861
+ 1,
3862
+ 0,
3863
+ 1,
3864
+ 1,
3865
+ 1,
3866
+ 0,
3867
+ 1,
3868
+ 1,
3869
+ 1,
3870
+ 0,
3871
+ 1,
3872
+ 1,
3873
+ 1,
3874
+ 0,
3875
+ 1,
3876
+ 0,
3877
+ 1,
3878
+ 1,
3879
+ 1,
3880
+ 1,
3881
+ 1,
3882
+ 0,
3883
+ 1,
3884
+ 1,
3885
+ 1,
3886
+ 1,
3887
+ 1,
3888
+ 0,
3889
+ 1,
3890
+ 1,
3891
+ 1,
3892
+ 1,
3893
+ 1,
3894
+ 1,
3895
+ 1,
3896
+ 1,
3897
+ 0,
3898
+ 1,
3899
+ 0,
3900
+ 0,
3901
+ 0,
3902
+ 1,
3903
+ 1,
3904
+ 1,
3905
+ 1,
3906
+ 0,
3907
+ 0,
3908
+ 1,
3909
+ 1,
3910
+ 1,
3911
+ 1,
3912
+ 1,
3913
+ 0,
3914
+ 1,
3915
+ 1,
3916
+ 1,
3917
+ 1,
3918
+ 1,
3919
+ 0,
3920
+ 1,
3921
+ 1,
3922
+ 1,
3923
+ 1,
3924
+ 1,
3925
+ 0,
3926
+ 0,
3927
+ 1,
3928
+ 1,
3929
+ 0,
3930
+ 1,
3931
+ 0,
3932
+ 0,
3933
+ 0,
3934
+ 1,
3935
+ 0,
3936
+ 1,
3937
+ 1,
3938
+ 0,
3939
+ 0,
3940
+ 0,
3941
+ 0,
3942
+ 0,
3943
+ 0,
3944
+ 0,
3945
+ 1,
3946
+ 1,
3947
+ 1,
3948
+ 1,
3949
+ 1,
3950
+ 1,
3951
+ 1,
3952
+ 0,
3953
+ 1,
3954
+ 1,
3955
+ 1,
3956
+ 1,
3957
+ 0,
3958
+ 1,
3959
+ 0,
3960
+ 0,
3961
+ 1,
3962
+ 0,
3963
+ 0,
3964
+ 0,
3965
+ 1,
3966
+ 0,
3967
+ 1,
3968
+ 1,
3969
+ 0,
3970
+ 1,
3971
+ 1,
3972
+ 0,
3973
+ 1,
3974
+ 1,
3975
+ 0,
3976
+ 1,
3977
+ 1,
3978
+ 0,
3979
+ 0,
3980
+ 1,
3981
+ 1,
3982
+ 1,
3983
+ 1,
3984
+ 0,
3985
+ 0,
3986
+ 1,
3987
+ 0,
3988
+ 0,
3989
+ 1,
3990
+ 1,
3991
+ 1,
3992
+ 1,
3993
+ 0,
3994
+ 0,
3995
+ 1,
3996
+ 1,
3997
+ 0,
3998
+ 1,
3999
+ 1,
4000
+ 0,
4001
+ 1,
4002
+ 0,
4003
+ 1,
4004
+ 1,
4005
+ 1,
4006
+ 0,
4007
+ 1,
4008
+ 1,
4009
+ 0,
4010
+ 1,
4011
+ 1,
4012
+ 1,
4013
+ 0,
4014
+ 1,
4015
+ 0,
4016
+ 0,
4017
+ 0,
4018
+ 1,
4019
+ 0,
4020
+ 1,
4021
+ 1,
4022
+ 1,
4023
+ 1,
4024
+ 0,
4025
+ 1,
4026
+ 0,
4027
+ 1,
4028
+ 1,
4029
+ 0,
4030
+ 0,
4031
+ 1,
4032
+ 1,
4033
+ 1,
4034
+ 1,
4035
+ 1,
4036
+ 1,
4037
+ 1,
4038
+ 1,
4039
+ 0,
4040
+ 1,
4041
+ 1,
4042
+ 1,
4043
+ 1,
4044
+ 1,
4045
+ 0,
4046
+ 0,
4047
+ 1,
4048
+ 1,
4049
+ 1,
4050
+ 0,
4051
+ 1,
4052
+ 1,
4053
+ 0,
4054
+ 1,
4055
+ 1,
4056
+ 1,
4057
+ 1,
4058
+ 0,
4059
+ 1,
4060
+ 1,
4061
+ 0,
4062
+ 1,
4063
+ 1,
4064
+ 1,
4065
+ 1,
4066
+ 1,
4067
+ 1,
4068
+ 1,
4069
+ 1,
4070
+ 1,
4071
+ 0,
4072
+ 1,
4073
+ 0,
4074
+ 0,
4075
+ 1,
4076
+ 1,
4077
+ 0,
4078
+ 1,
4079
+ 1,
4080
+ 1,
4081
+ 1,
4082
+ 1,
4083
+ 1,
4084
+ 1,
4085
+ 1,
4086
+ 1,
4087
+ 1,
4088
+ 1,
4089
+ 1,
4090
+ 1,
4091
+ 0,
4092
+ 0,
4093
+ 1,
4094
+ 1,
4095
+ 1,
4096
+ 1,
4097
+ 1,
4098
+ 1,
4099
+ 1,
4100
+ 1,
4101
+ 1,
4102
+ 1,
4103
+ 1,
4104
+ 0,
4105
+ 0,
4106
+ 1,
4107
+ 0,
4108
+ 1,
4109
+ 1,
4110
+ 1,
4111
+ 1,
4112
+ 1,
4113
+ 0,
4114
+ 1,
4115
+ 0,
4116
+ 1,
4117
+ 1,
4118
+ 1,
4119
+ 0,
4120
+ 0,
4121
+ 1,
4122
+ 0,
4123
+ 1,
4124
+ 1,
4125
+ 0,
4126
+ 1,
4127
+ 1,
4128
+ 0,
4129
+ 1,
4130
+ 1,
4131
+ 1,
4132
+ 1,
4133
+ 1,
4134
+ 1,
4135
+ 1,
4136
+ 1,
4137
+ 1,
4138
+ 1,
4139
+ 0,
4140
+ 0,
4141
+ 0,
4142
+ 0,
4143
+ 1,
4144
+ 0,
4145
+ 1,
4146
+ 1,
4147
+ 1,
4148
+ 1,
4149
+ 1,
4150
+ 0,
4151
+ 0,
4152
+ 1,
4153
+ 1,
4154
+ 0,
4155
+ 1,
4156
+ 1,
4157
+ 0,
4158
+ 1,
4159
+ 0,
4160
+ 0,
4161
+ 0,
4162
+ 1,
4163
+ 1,
4164
+ 0,
4165
+ 0,
4166
+ 1,
4167
+ 1,
4168
+ 0,
4169
+ 1,
4170
+ 0,
4171
+ 1,
4172
+ 1,
4173
+ 1,
4174
+ 1,
4175
+ 1,
4176
+ 1,
4177
+ 1,
4178
+ 1,
4179
+ 0,
4180
+ 1,
4181
+ 1,
4182
+ 1,
4183
+ 1,
4184
+ 1,
4185
+ 1,
4186
+ 1,
4187
+ 1,
4188
+ 0,
4189
+ 1,
4190
+ 1,
4191
+ 0,
4192
+ 0,
4193
+ 1,
4194
+ 1,
4195
+ 1,
4196
+ 1,
4197
+ 1,
4198
+ 1,
4199
+ 0,
4200
+ 0,
4201
+ 1,
4202
+ 0,
4203
+ 1,
4204
+ 1,
4205
+ 0,
4206
+ 0,
4207
+ 1,
4208
+ 1,
4209
+ 1,
4210
+ 1,
4211
+ 1,
4212
+ 0,
4213
+ 1,
4214
+ 1,
4215
+ 0,
4216
+ 1,
4217
+ 1,
4218
+ 1,
4219
+ 1,
4220
+ 1,
4221
+ 1,
4222
+ 0,
4223
+ 1,
4224
+ 1,
4225
+ 1,
4226
+ 0,
4227
+ 0,
4228
+ 1,
4229
+ 1,
4230
+ 1,
4231
+ 1,
4232
+ 1,
4233
+ 1,
4234
+ 0,
4235
+ 1,
4236
+ 0,
4237
+ 1,
4238
+ 0,
4239
+ 1,
4240
+ 0,
4241
+ 1,
4242
+ 0,
4243
+ 1,
4244
+ 1,
4245
+ 0,
4246
+ 1,
4247
+ 1,
4248
+ 1,
4249
+ 1,
4250
+ 0,
4251
+ 0,
4252
+ 0,
4253
+ 0,
4254
+ 1,
4255
+ 1,
4256
+ 1,
4257
+ 1,
4258
+ 1,
4259
+ 1,
4260
+ 0,
4261
+ 1,
4262
+ 0,
4263
+ 1,
4264
+ 1,
4265
+ 1,
4266
+ 1,
4267
+ 1,
4268
+ 0,
4269
+ 0,
4270
+ 0,
4271
+ 1,
4272
+ 1,
4273
+ 0,
4274
+ 1,
4275
+ 1,
4276
+ 1,
4277
+ 1,
4278
+ 1,
4279
+ 1,
4280
+ 0,
4281
+ 1,
4282
+ 0,
4283
+ 1,
4284
+ 1,
4285
+ 0,
4286
+ 0,
4287
+ 1,
4288
+ 1,
4289
+ 1,
4290
+ 1,
4291
+ 0,
4292
+ 1,
4293
+ 1,
4294
+ 1,
4295
+ 1,
4296
+ 1,
4297
+ 0,
4298
+ 1,
4299
+ 0,
4300
+ 1,
4301
+ 0,
4302
+ 1,
4303
+ 0,
4304
+ 1,
4305
+ 1,
4306
+ 0,
4307
+ 1,
4308
+ 1,
4309
+ 0,
4310
+ 1,
4311
+ 1,
4312
+ 1,
4313
+ 0,
4314
+ 1,
4315
+ 1,
4316
+ 1,
4317
+ 1,
4318
+ 1,
4319
+ 0,
4320
+ 0,
4321
+ 1,
4322
+ 1,
4323
+ 0,
4324
+ 1,
4325
+ 1,
4326
+ 0,
4327
+ 1,
4328
+ 1,
4329
+ 1,
4330
+ 1,
4331
+ 1,
4332
+ 1,
4333
+ 1,
4334
+ 0,
4335
+ 0,
4336
+ 0,
4337
+ 0,
4338
+ 1,
4339
+ 0,
4340
+ 0,
4341
+ 1,
4342
+ 1,
4343
+ 1,
4344
+ 1,
4345
+ 1,
4346
+ 1,
4347
+ 1,
4348
+ 1,
4349
+ 1,
4350
+ 1,
4351
+ 0,
4352
+ 0,
4353
+ 0,
4354
+ 1,
4355
+ 1,
4356
+ 1,
4357
+ 1,
4358
+ 1,
4359
+ 0,
4360
+ 0,
4361
+ 1,
4362
+ 1,
4363
+ 1,
4364
+ 1,
4365
+ 1,
4366
+ 1,
4367
+ 1,
4368
+ 1,
4369
+ 1,
4370
+ 1,
4371
+ 0,
4372
+ 0,
4373
+ 0,
4374
+ 1,
4375
+ 1,
4376
+ 0,
4377
+ 0,
4378
+ 1,
4379
+ 1,
4380
+ 0,
4381
+ 1,
4382
+ 0,
4383
+ 1,
4384
+ 1,
4385
+ 0,
4386
+ 0,
4387
+ 0,
4388
+ 1,
4389
+ 1,
4390
+ 0,
4391
+ 1,
4392
+ 0,
4393
+ 0,
4394
+ 0,
4395
+ 1,
4396
+ 0,
4397
+ 0,
4398
+ 1,
4399
+ 1,
4400
+ 1,
4401
+ 1,
4402
+ 1,
4403
+ 1,
4404
+ 1,
4405
+ 1,
4406
+ 0,
4407
+ 1,
4408
+ 0,
4409
+ 0,
4410
+ 1,
4411
+ 1,
4412
+ 0,
4413
+ 0,
4414
+ 0,
4415
+ 0,
4416
+ 1,
4417
+ 1,
4418
+ 0,
4419
+ 1,
4420
+ 1,
4421
+ 1,
4422
+ 1,
4423
+ 1,
4424
+ 0,
4425
+ 1,
4426
+ 1,
4427
+ 0,
4428
+ 1,
4429
+ 0,
4430
+ 0,
4431
+ 1,
4432
+ 1,
4433
+ 1,
4434
+ 0,
4435
+ 0,
4436
+ 1,
4437
+ 0,
4438
+ 0,
4439
+ 1,
4440
+ 1,
4441
+ 1,
4442
+ 1,
4443
+ 1,
4444
+ 1,
4445
+ 1,
4446
+ 1,
4447
+ 1,
4448
+ 1,
4449
+ 1,
4450
+ 1,
4451
+ 0,
4452
+ 1,
4453
+ 1,
4454
+ 0,
4455
+ 0,
4456
+ 1,
4457
+ 1,
4458
+ 1,
4459
+ 1,
4460
+ 0,
4461
+ 0,
4462
+ 1,
4463
+ 0,
4464
+ 0,
4465
+ 1,
4466
+ 1,
4467
+ 0,
4468
+ 0,
4469
+ 0,
4470
+ 0,
4471
+ 1,
4472
+ 1,
4473
+ 1,
4474
+ 0,
4475
+ 1,
4476
+ 1,
4477
+ 1,
4478
+ 1,
4479
+ 1,
4480
+ 1,
4481
+ 1,
4482
+ 0,
4483
+ 1,
4484
+ 1,
4485
+ 1,
4486
+ 0,
4487
+ 0,
4488
+ 1,
4489
+ 1,
4490
+ 1,
4491
+ 0,
4492
+ 1,
4493
+ 1,
4494
+ 0,
4495
+ 0,
4496
+ 1,
4497
+ 1,
4498
+ 0,
4499
+ 0,
4500
+ 1,
4501
+ 0,
4502
+ 0,
4503
+ 0,
4504
+ 1,
4505
+ 1,
4506
+ 1,
4507
+ 1,
4508
+ 1,
4509
+ 0,
4510
+ 1
4511
+ ]
4512
+ }
experiments/m6/summary.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "m6",
3
+ "checkpoint": "/lambda/nfs/experiment/results/v9_meta_fork/prosqa-m6-pause-multipass/checkpoint_best",
4
+ "feedback_mode": "pause_multipass",
5
+ "stages_run": [
6
+ "transplant"
7
+ ],
8
+ "results": {
9
+ "transplant": {
10
+ "matched": 0,
11
+ "unmatched": 0,
12
+ "total": 0,
13
+ "matched_rate": 0.0,
14
+ "unmatched_rate": 0.0
15
+ }
16
+ }
17
+ }
experiments/m6_epoch39/accuracy.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prosqa_test": {
3
+ "accuracy": 0.95,
4
+ "correct": 475,
5
+ "total": 500
6
+ },
7
+ "ood_7hop": {
8
+ "accuracy": 0.73,
9
+ "correct": 730,
10
+ "total": 1000
11
+ },
12
+ "ood_8hop": {
13
+ "accuracy": 0.732,
14
+ "correct": 732,
15
+ "total": 1000
16
+ },
17
+ "ood_dag": {
18
+ "accuracy": 0.608,
19
+ "correct": 608,
20
+ "total": 1000
21
+ },
22
+ "ood_dense": {
23
+ "accuracy": 0.667,
24
+ "correct": 667,
25
+ "total": 1000
26
+ }
27
+ }
experiments/m6_epoch39/per_sample_correctness.json ADDED
@@ -0,0 +1,4512 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prosqa_test": [
3
+ 0,
4
+ 1,
5
+ 1,
6
+ 1,
7
+ 1,
8
+ 1,
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 1,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 1,
21
+ 1,
22
+ 1,
23
+ 1,
24
+ 1,
25
+ 1,
26
+ 1,
27
+ 1,
28
+ 1,
29
+ 1,
30
+ 1,
31
+ 1,
32
+ 1,
33
+ 0,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 0,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 0,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 0,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 0,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 0,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 0,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 1,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 1,
163
+ 0,
164
+ 1,
165
+ 1,
166
+ 1,
167
+ 1,
168
+ 1,
169
+ 1,
170
+ 1,
171
+ 0,
172
+ 1,
173
+ 1,
174
+ 1,
175
+ 1,
176
+ 1,
177
+ 1,
178
+ 1,
179
+ 1,
180
+ 1,
181
+ 1,
182
+ 1,
183
+ 1,
184
+ 1,
185
+ 1,
186
+ 1,
187
+ 1,
188
+ 1,
189
+ 1,
190
+ 1,
191
+ 1,
192
+ 1,
193
+ 1,
194
+ 1,
195
+ 1,
196
+ 1,
197
+ 0,
198
+ 1,
199
+ 1,
200
+ 1,
201
+ 1,
202
+ 1,
203
+ 0,
204
+ 1,
205
+ 1,
206
+ 1,
207
+ 1,
208
+ 1,
209
+ 1,
210
+ 1,
211
+ 1,
212
+ 1,
213
+ 1,
214
+ 1,
215
+ 1,
216
+ 1,
217
+ 1,
218
+ 1,
219
+ 1,
220
+ 1,
221
+ 1,
222
+ 0,
223
+ 1,
224
+ 1,
225
+ 1,
226
+ 1,
227
+ 1,
228
+ 1,
229
+ 1,
230
+ 1,
231
+ 1,
232
+ 1,
233
+ 1,
234
+ 0,
235
+ 1,
236
+ 1,
237
+ 1,
238
+ 1,
239
+ 1,
240
+ 0,
241
+ 1,
242
+ 1,
243
+ 1,
244
+ 1,
245
+ 1,
246
+ 1,
247
+ 1,
248
+ 1,
249
+ 1,
250
+ 1,
251
+ 1,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 0,
256
+ 1,
257
+ 1,
258
+ 1,
259
+ 1,
260
+ 1,
261
+ 1,
262
+ 1,
263
+ 1,
264
+ 1,
265
+ 1,
266
+ 1,
267
+ 1,
268
+ 1,
269
+ 1,
270
+ 1,
271
+ 1,
272
+ 1,
273
+ 1,
274
+ 1,
275
+ 1,
276
+ 1,
277
+ 1,
278
+ 0,
279
+ 0,
280
+ 1,
281
+ 1,
282
+ 1,
283
+ 1,
284
+ 1,
285
+ 1,
286
+ 1,
287
+ 1,
288
+ 1,
289
+ 1,
290
+ 1,
291
+ 1,
292
+ 1,
293
+ 1,
294
+ 1,
295
+ 1,
296
+ 1,
297
+ 1,
298
+ 1,
299
+ 1,
300
+ 1,
301
+ 1,
302
+ 1,
303
+ 1,
304
+ 1,
305
+ 1,
306
+ 1,
307
+ 1,
308
+ 1,
309
+ 1,
310
+ 1,
311
+ 1,
312
+ 1,
313
+ 1,
314
+ 1,
315
+ 1,
316
+ 1,
317
+ 1,
318
+ 1,
319
+ 1,
320
+ 1,
321
+ 1,
322
+ 1,
323
+ 1,
324
+ 1,
325
+ 1,
326
+ 1,
327
+ 1,
328
+ 0,
329
+ 1,
330
+ 1,
331
+ 1,
332
+ 1,
333
+ 1,
334
+ 1,
335
+ 1,
336
+ 1,
337
+ 1,
338
+ 1,
339
+ 1,
340
+ 1,
341
+ 1,
342
+ 1,
343
+ 1,
344
+ 1,
345
+ 1,
346
+ 1,
347
+ 1,
348
+ 1,
349
+ 1,
350
+ 1,
351
+ 1,
352
+ 1,
353
+ 1,
354
+ 1,
355
+ 1,
356
+ 1,
357
+ 1,
358
+ 1,
359
+ 1,
360
+ 1,
361
+ 1,
362
+ 1,
363
+ 1,
364
+ 1,
365
+ 1,
366
+ 1,
367
+ 1,
368
+ 1,
369
+ 1,
370
+ 1,
371
+ 1,
372
+ 1,
373
+ 1,
374
+ 1,
375
+ 1,
376
+ 1,
377
+ 1,
378
+ 1,
379
+ 1,
380
+ 1,
381
+ 1,
382
+ 1,
383
+ 1,
384
+ 1,
385
+ 1,
386
+ 1,
387
+ 1,
388
+ 1,
389
+ 0,
390
+ 1,
391
+ 1,
392
+ 0,
393
+ 1,
394
+ 1,
395
+ 1,
396
+ 1,
397
+ 1,
398
+ 1,
399
+ 1,
400
+ 1,
401
+ 1,
402
+ 1,
403
+ 1,
404
+ 1,
405
+ 1,
406
+ 1,
407
+ 1,
408
+ 1,
409
+ 1,
410
+ 1,
411
+ 1,
412
+ 1,
413
+ 1,
414
+ 1,
415
+ 1,
416
+ 0,
417
+ 1,
418
+ 1,
419
+ 1,
420
+ 1,
421
+ 1,
422
+ 1,
423
+ 1,
424
+ 1,
425
+ 1,
426
+ 1,
427
+ 1,
428
+ 1,
429
+ 1,
430
+ 1,
431
+ 1,
432
+ 1,
433
+ 1,
434
+ 1,
435
+ 1,
436
+ 1,
437
+ 1,
438
+ 1,
439
+ 1,
440
+ 1,
441
+ 1,
442
+ 1,
443
+ 1,
444
+ 1,
445
+ 1,
446
+ 1,
447
+ 1,
448
+ 1,
449
+ 1,
450
+ 1,
451
+ 1,
452
+ 1,
453
+ 1,
454
+ 1,
455
+ 1,
456
+ 1,
457
+ 1,
458
+ 1,
459
+ 1,
460
+ 1,
461
+ 1,
462
+ 1,
463
+ 1,
464
+ 1,
465
+ 1,
466
+ 1,
467
+ 1,
468
+ 1,
469
+ 0,
470
+ 0,
471
+ 0,
472
+ 1,
473
+ 1,
474
+ 1,
475
+ 1,
476
+ 1,
477
+ 1,
478
+ 1,
479
+ 1,
480
+ 1,
481
+ 1,
482
+ 1,
483
+ 1,
484
+ 1,
485
+ 1,
486
+ 1,
487
+ 1,
488
+ 1,
489
+ 1,
490
+ 1,
491
+ 1,
492
+ 1,
493
+ 1,
494
+ 1,
495
+ 1,
496
+ 1,
497
+ 1,
498
+ 1,
499
+ 1,
500
+ 1,
501
+ 1,
502
+ 1
503
+ ],
504
+ "ood_7hop": [
505
+ 1,
506
+ 1,
507
+ 1,
508
+ 1,
509
+ 1,
510
+ 1,
511
+ 1,
512
+ 1,
513
+ 0,
514
+ 0,
515
+ 0,
516
+ 1,
517
+ 1,
518
+ 0,
519
+ 1,
520
+ 1,
521
+ 1,
522
+ 1,
523
+ 1,
524
+ 1,
525
+ 1,
526
+ 1,
527
+ 1,
528
+ 1,
529
+ 1,
530
+ 0,
531
+ 1,
532
+ 0,
533
+ 1,
534
+ 1,
535
+ 1,
536
+ 0,
537
+ 1,
538
+ 1,
539
+ 1,
540
+ 1,
541
+ 1,
542
+ 0,
543
+ 1,
544
+ 0,
545
+ 0,
546
+ 1,
547
+ 1,
548
+ 1,
549
+ 1,
550
+ 0,
551
+ 0,
552
+ 0,
553
+ 1,
554
+ 0,
555
+ 1,
556
+ 1,
557
+ 1,
558
+ 1,
559
+ 1,
560
+ 1,
561
+ 0,
562
+ 1,
563
+ 0,
564
+ 1,
565
+ 0,
566
+ 1,
567
+ 1,
568
+ 0,
569
+ 1,
570
+ 1,
571
+ 1,
572
+ 1,
573
+ 1,
574
+ 1,
575
+ 1,
576
+ 1,
577
+ 0,
578
+ 1,
579
+ 1,
580
+ 1,
581
+ 1,
582
+ 1,
583
+ 0,
584
+ 1,
585
+ 1,
586
+ 1,
587
+ 1,
588
+ 0,
589
+ 1,
590
+ 1,
591
+ 1,
592
+ 1,
593
+ 0,
594
+ 1,
595
+ 0,
596
+ 1,
597
+ 1,
598
+ 0,
599
+ 1,
600
+ 0,
601
+ 0,
602
+ 1,
603
+ 1,
604
+ 0,
605
+ 1,
606
+ 0,
607
+ 0,
608
+ 1,
609
+ 0,
610
+ 1,
611
+ 1,
612
+ 0,
613
+ 0,
614
+ 1,
615
+ 1,
616
+ 1,
617
+ 1,
618
+ 1,
619
+ 1,
620
+ 1,
621
+ 1,
622
+ 1,
623
+ 0,
624
+ 0,
625
+ 1,
626
+ 1,
627
+ 1,
628
+ 1,
629
+ 1,
630
+ 0,
631
+ 1,
632
+ 0,
633
+ 0,
634
+ 1,
635
+ 1,
636
+ 1,
637
+ 1,
638
+ 1,
639
+ 1,
640
+ 1,
641
+ 1,
642
+ 1,
643
+ 0,
644
+ 1,
645
+ 1,
646
+ 1,
647
+ 1,
648
+ 0,
649
+ 1,
650
+ 1,
651
+ 1,
652
+ 1,
653
+ 1,
654
+ 0,
655
+ 1,
656
+ 1,
657
+ 1,
658
+ 1,
659
+ 0,
660
+ 0,
661
+ 1,
662
+ 1,
663
+ 1,
664
+ 1,
665
+ 1,
666
+ 1,
667
+ 1,
668
+ 0,
669
+ 1,
670
+ 1,
671
+ 1,
672
+ 0,
673
+ 1,
674
+ 0,
675
+ 0,
676
+ 1,
677
+ 1,
678
+ 1,
679
+ 1,
680
+ 1,
681
+ 0,
682
+ 0,
683
+ 0,
684
+ 1,
685
+ 1,
686
+ 1,
687
+ 1,
688
+ 1,
689
+ 1,
690
+ 1,
691
+ 1,
692
+ 1,
693
+ 0,
694
+ 1,
695
+ 1,
696
+ 1,
697
+ 0,
698
+ 1,
699
+ 1,
700
+ 1,
701
+ 1,
702
+ 1,
703
+ 0,
704
+ 1,
705
+ 0,
706
+ 1,
707
+ 1,
708
+ 1,
709
+ 0,
710
+ 0,
711
+ 1,
712
+ 1,
713
+ 1,
714
+ 0,
715
+ 1,
716
+ 1,
717
+ 1,
718
+ 0,
719
+ 1,
720
+ 1,
721
+ 0,
722
+ 1,
723
+ 1,
724
+ 1,
725
+ 1,
726
+ 0,
727
+ 1,
728
+ 0,
729
+ 0,
730
+ 1,
731
+ 1,
732
+ 1,
733
+ 1,
734
+ 1,
735
+ 1,
736
+ 1,
737
+ 1,
738
+ 1,
739
+ 0,
740
+ 1,
741
+ 1,
742
+ 1,
743
+ 1,
744
+ 1,
745
+ 0,
746
+ 1,
747
+ 0,
748
+ 1,
749
+ 1,
750
+ 1,
751
+ 1,
752
+ 1,
753
+ 1,
754
+ 0,
755
+ 1,
756
+ 0,
757
+ 1,
758
+ 1,
759
+ 1,
760
+ 1,
761
+ 0,
762
+ 1,
763
+ 1,
764
+ 1,
765
+ 0,
766
+ 1,
767
+ 1,
768
+ 1,
769
+ 1,
770
+ 1,
771
+ 0,
772
+ 0,
773
+ 0,
774
+ 1,
775
+ 0,
776
+ 0,
777
+ 1,
778
+ 1,
779
+ 0,
780
+ 1,
781
+ 1,
782
+ 1,
783
+ 1,
784
+ 1,
785
+ 1,
786
+ 0,
787
+ 1,
788
+ 1,
789
+ 1,
790
+ 0,
791
+ 1,
792
+ 1,
793
+ 1,
794
+ 1,
795
+ 1,
796
+ 1,
797
+ 1,
798
+ 1,
799
+ 1,
800
+ 1,
801
+ 1,
802
+ 0,
803
+ 1,
804
+ 0,
805
+ 1,
806
+ 1,
807
+ 1,
808
+ 1,
809
+ 1,
810
+ 0,
811
+ 1,
812
+ 1,
813
+ 0,
814
+ 0,
815
+ 1,
816
+ 1,
817
+ 1,
818
+ 1,
819
+ 1,
820
+ 1,
821
+ 1,
822
+ 0,
823
+ 1,
824
+ 1,
825
+ 1,
826
+ 1,
827
+ 0,
828
+ 1,
829
+ 1,
830
+ 1,
831
+ 1,
832
+ 1,
833
+ 1,
834
+ 0,
835
+ 1,
836
+ 1,
837
+ 1,
838
+ 1,
839
+ 1,
840
+ 0,
841
+ 1,
842
+ 1,
843
+ 0,
844
+ 0,
845
+ 0,
846
+ 1,
847
+ 1,
848
+ 0,
849
+ 0,
850
+ 0,
851
+ 0,
852
+ 1,
853
+ 1,
854
+ 1,
855
+ 0,
856
+ 1,
857
+ 1,
858
+ 1,
859
+ 1,
860
+ 0,
861
+ 0,
862
+ 1,
863
+ 1,
864
+ 1,
865
+ 1,
866
+ 1,
867
+ 1,
868
+ 1,
869
+ 0,
870
+ 1,
871
+ 1,
872
+ 1,
873
+ 1,
874
+ 1,
875
+ 1,
876
+ 1,
877
+ 0,
878
+ 1,
879
+ 1,
880
+ 1,
881
+ 1,
882
+ 0,
883
+ 0,
884
+ 1,
885
+ 0,
886
+ 0,
887
+ 1,
888
+ 1,
889
+ 0,
890
+ 1,
891
+ 1,
892
+ 1,
893
+ 1,
894
+ 0,
895
+ 1,
896
+ 0,
897
+ 1,
898
+ 0,
899
+ 1,
900
+ 1,
901
+ 1,
902
+ 1,
903
+ 1,
904
+ 1,
905
+ 1,
906
+ 1,
907
+ 1,
908
+ 1,
909
+ 1,
910
+ 0,
911
+ 0,
912
+ 0,
913
+ 1,
914
+ 1,
915
+ 1,
916
+ 0,
917
+ 1,
918
+ 1,
919
+ 1,
920
+ 1,
921
+ 1,
922
+ 0,
923
+ 0,
924
+ 1,
925
+ 0,
926
+ 1,
927
+ 1,
928
+ 1,
929
+ 0,
930
+ 0,
931
+ 1,
932
+ 0,
933
+ 1,
934
+ 1,
935
+ 0,
936
+ 1,
937
+ 1,
938
+ 1,
939
+ 0,
940
+ 1,
941
+ 1,
942
+ 1,
943
+ 1,
944
+ 1,
945
+ 0,
946
+ 1,
947
+ 1,
948
+ 1,
949
+ 0,
950
+ 0,
951
+ 1,
952
+ 1,
953
+ 1,
954
+ 1,
955
+ 1,
956
+ 1,
957
+ 1,
958
+ 1,
959
+ 1,
960
+ 1,
961
+ 0,
962
+ 1,
963
+ 1,
964
+ 1,
965
+ 1,
966
+ 0,
967
+ 1,
968
+ 1,
969
+ 1,
970
+ 0,
971
+ 1,
972
+ 0,
973
+ 1,
974
+ 1,
975
+ 1,
976
+ 0,
977
+ 1,
978
+ 1,
979
+ 1,
980
+ 0,
981
+ 1,
982
+ 1,
983
+ 1,
984
+ 1,
985
+ 1,
986
+ 0,
987
+ 1,
988
+ 0,
989
+ 1,
990
+ 1,
991
+ 1,
992
+ 0,
993
+ 1,
994
+ 0,
995
+ 0,
996
+ 1,
997
+ 0,
998
+ 1,
999
+ 1,
1000
+ 1,
1001
+ 1,
1002
+ 1,
1003
+ 0,
1004
+ 1,
1005
+ 1,
1006
+ 1,
1007
+ 1,
1008
+ 1,
1009
+ 0,
1010
+ 1,
1011
+ 1,
1012
+ 1,
1013
+ 1,
1014
+ 1,
1015
+ 1,
1016
+ 1,
1017
+ 1,
1018
+ 1,
1019
+ 0,
1020
+ 1,
1021
+ 1,
1022
+ 1,
1023
+ 1,
1024
+ 0,
1025
+ 1,
1026
+ 0,
1027
+ 1,
1028
+ 0,
1029
+ 0,
1030
+ 1,
1031
+ 0,
1032
+ 1,
1033
+ 1,
1034
+ 1,
1035
+ 1,
1036
+ 0,
1037
+ 1,
1038
+ 1,
1039
+ 0,
1040
+ 1,
1041
+ 1,
1042
+ 0,
1043
+ 1,
1044
+ 0,
1045
+ 1,
1046
+ 0,
1047
+ 1,
1048
+ 1,
1049
+ 0,
1050
+ 0,
1051
+ 1,
1052
+ 1,
1053
+ 1,
1054
+ 1,
1055
+ 1,
1056
+ 1,
1057
+ 1,
1058
+ 0,
1059
+ 1,
1060
+ 1,
1061
+ 1,
1062
+ 1,
1063
+ 1,
1064
+ 0,
1065
+ 0,
1066
+ 1,
1067
+ 1,
1068
+ 1,
1069
+ 1,
1070
+ 1,
1071
+ 1,
1072
+ 1,
1073
+ 1,
1074
+ 1,
1075
+ 0,
1076
+ 1,
1077
+ 1,
1078
+ 1,
1079
+ 1,
1080
+ 0,
1081
+ 1,
1082
+ 1,
1083
+ 1,
1084
+ 1,
1085
+ 1,
1086
+ 1,
1087
+ 1,
1088
+ 1,
1089
+ 0,
1090
+ 1,
1091
+ 1,
1092
+ 1,
1093
+ 1,
1094
+ 0,
1095
+ 0,
1096
+ 0,
1097
+ 1,
1098
+ 0,
1099
+ 1,
1100
+ 0,
1101
+ 1,
1102
+ 1,
1103
+ 1,
1104
+ 1,
1105
+ 1,
1106
+ 1,
1107
+ 1,
1108
+ 1,
1109
+ 0,
1110
+ 1,
1111
+ 1,
1112
+ 1,
1113
+ 0,
1114
+ 1,
1115
+ 1,
1116
+ 0,
1117
+ 1,
1118
+ 1,
1119
+ 1,
1120
+ 0,
1121
+ 0,
1122
+ 1,
1123
+ 0,
1124
+ 1,
1125
+ 1,
1126
+ 1,
1127
+ 0,
1128
+ 0,
1129
+ 0,
1130
+ 1,
1131
+ 1,
1132
+ 1,
1133
+ 1,
1134
+ 1,
1135
+ 1,
1136
+ 1,
1137
+ 1,
1138
+ 1,
1139
+ 1,
1140
+ 0,
1141
+ 0,
1142
+ 1,
1143
+ 1,
1144
+ 1,
1145
+ 1,
1146
+ 1,
1147
+ 1,
1148
+ 1,
1149
+ 1,
1150
+ 0,
1151
+ 1,
1152
+ 1,
1153
+ 1,
1154
+ 1,
1155
+ 1,
1156
+ 0,
1157
+ 1,
1158
+ 1,
1159
+ 1,
1160
+ 0,
1161
+ 1,
1162
+ 1,
1163
+ 1,
1164
+ 1,
1165
+ 1,
1166
+ 0,
1167
+ 1,
1168
+ 0,
1169
+ 1,
1170
+ 1,
1171
+ 0,
1172
+ 1,
1173
+ 0,
1174
+ 1,
1175
+ 1,
1176
+ 0,
1177
+ 0,
1178
+ 1,
1179
+ 0,
1180
+ 1,
1181
+ 1,
1182
+ 1,
1183
+ 1,
1184
+ 1,
1185
+ 1,
1186
+ 1,
1187
+ 0,
1188
+ 1,
1189
+ 1,
1190
+ 0,
1191
+ 1,
1192
+ 0,
1193
+ 1,
1194
+ 1,
1195
+ 1,
1196
+ 1,
1197
+ 0,
1198
+ 1,
1199
+ 1,
1200
+ 1,
1201
+ 1,
1202
+ 0,
1203
+ 0,
1204
+ 1,
1205
+ 1,
1206
+ 0,
1207
+ 0,
1208
+ 1,
1209
+ 1,
1210
+ 0,
1211
+ 1,
1212
+ 0,
1213
+ 1,
1214
+ 0,
1215
+ 1,
1216
+ 1,
1217
+ 1,
1218
+ 0,
1219
+ 1,
1220
+ 1,
1221
+ 1,
1222
+ 1,
1223
+ 1,
1224
+ 1,
1225
+ 1,
1226
+ 0,
1227
+ 1,
1228
+ 1,
1229
+ 1,
1230
+ 1,
1231
+ 0,
1232
+ 1,
1233
+ 1,
1234
+ 1,
1235
+ 1,
1236
+ 0,
1237
+ 1,
1238
+ 1,
1239
+ 1,
1240
+ 0,
1241
+ 0,
1242
+ 1,
1243
+ 0,
1244
+ 1,
1245
+ 0,
1246
+ 1,
1247
+ 1,
1248
+ 0,
1249
+ 1,
1250
+ 0,
1251
+ 1,
1252
+ 0,
1253
+ 1,
1254
+ 1,
1255
+ 0,
1256
+ 1,
1257
+ 1,
1258
+ 1,
1259
+ 1,
1260
+ 1,
1261
+ 1,
1262
+ 1,
1263
+ 0,
1264
+ 0,
1265
+ 1,
1266
+ 0,
1267
+ 1,
1268
+ 1,
1269
+ 1,
1270
+ 1,
1271
+ 1,
1272
+ 1,
1273
+ 1,
1274
+ 0,
1275
+ 1,
1276
+ 0,
1277
+ 0,
1278
+ 1,
1279
+ 1,
1280
+ 1,
1281
+ 0,
1282
+ 1,
1283
+ 1,
1284
+ 0,
1285
+ 0,
1286
+ 0,
1287
+ 0,
1288
+ 0,
1289
+ 0,
1290
+ 1,
1291
+ 1,
1292
+ 1,
1293
+ 1,
1294
+ 1,
1295
+ 0,
1296
+ 1,
1297
+ 1,
1298
+ 1,
1299
+ 1,
1300
+ 1,
1301
+ 0,
1302
+ 1,
1303
+ 1,
1304
+ 1,
1305
+ 1,
1306
+ 0,
1307
+ 1,
1308
+ 1,
1309
+ 1,
1310
+ 1,
1311
+ 0,
1312
+ 1,
1313
+ 1,
1314
+ 1,
1315
+ 1,
1316
+ 1,
1317
+ 1,
1318
+ 1,
1319
+ 1,
1320
+ 1,
1321
+ 0,
1322
+ 1,
1323
+ 1,
1324
+ 0,
1325
+ 1,
1326
+ 1,
1327
+ 1,
1328
+ 1,
1329
+ 1,
1330
+ 1,
1331
+ 0,
1332
+ 1,
1333
+ 1,
1334
+ 0,
1335
+ 1,
1336
+ 1,
1337
+ 0,
1338
+ 1,
1339
+ 1,
1340
+ 0,
1341
+ 1,
1342
+ 1,
1343
+ 1,
1344
+ 1,
1345
+ 1,
1346
+ 1,
1347
+ 0,
1348
+ 0,
1349
+ 0,
1350
+ 1,
1351
+ 1,
1352
+ 0,
1353
+ 0,
1354
+ 0,
1355
+ 0,
1356
+ 1,
1357
+ 0,
1358
+ 1,
1359
+ 0,
1360
+ 1,
1361
+ 1,
1362
+ 1,
1363
+ 1,
1364
+ 0,
1365
+ 1,
1366
+ 1,
1367
+ 0,
1368
+ 1,
1369
+ 0,
1370
+ 0,
1371
+ 1,
1372
+ 0,
1373
+ 1,
1374
+ 1,
1375
+ 1,
1376
+ 1,
1377
+ 1,
1378
+ 1,
1379
+ 1,
1380
+ 1,
1381
+ 1,
1382
+ 1,
1383
+ 1,
1384
+ 1,
1385
+ 1,
1386
+ 0,
1387
+ 0,
1388
+ 1,
1389
+ 0,
1390
+ 1,
1391
+ 1,
1392
+ 1,
1393
+ 0,
1394
+ 1,
1395
+ 1,
1396
+ 1,
1397
+ 0,
1398
+ 1,
1399
+ 1,
1400
+ 1,
1401
+ 1,
1402
+ 1,
1403
+ 1,
1404
+ 1,
1405
+ 1,
1406
+ 0,
1407
+ 1,
1408
+ 1,
1409
+ 0,
1410
+ 1,
1411
+ 0,
1412
+ 1,
1413
+ 0,
1414
+ 1,
1415
+ 1,
1416
+ 1,
1417
+ 1,
1418
+ 1,
1419
+ 1,
1420
+ 1,
1421
+ 1,
1422
+ 0,
1423
+ 0,
1424
+ 0,
1425
+ 1,
1426
+ 0,
1427
+ 1,
1428
+ 1,
1429
+ 1,
1430
+ 1,
1431
+ 1,
1432
+ 1,
1433
+ 1,
1434
+ 1,
1435
+ 1,
1436
+ 1,
1437
+ 1,
1438
+ 1,
1439
+ 0,
1440
+ 1,
1441
+ 1,
1442
+ 1,
1443
+ 1,
1444
+ 1,
1445
+ 1,
1446
+ 1,
1447
+ 0,
1448
+ 0,
1449
+ 0,
1450
+ 0,
1451
+ 1,
1452
+ 0,
1453
+ 0,
1454
+ 0,
1455
+ 1,
1456
+ 0,
1457
+ 1,
1458
+ 1,
1459
+ 1,
1460
+ 1,
1461
+ 1,
1462
+ 1,
1463
+ 1,
1464
+ 1,
1465
+ 0,
1466
+ 1,
1467
+ 1,
1468
+ 1,
1469
+ 1,
1470
+ 0,
1471
+ 1,
1472
+ 1,
1473
+ 1,
1474
+ 1,
1475
+ 1,
1476
+ 1,
1477
+ 0,
1478
+ 0,
1479
+ 1,
1480
+ 0,
1481
+ 1,
1482
+ 0,
1483
+ 1,
1484
+ 1,
1485
+ 1,
1486
+ 1,
1487
+ 1,
1488
+ 1,
1489
+ 0,
1490
+ 1,
1491
+ 1,
1492
+ 1,
1493
+ 1,
1494
+ 0,
1495
+ 1,
1496
+ 1,
1497
+ 1,
1498
+ 0,
1499
+ 1,
1500
+ 1,
1501
+ 1,
1502
+ 1,
1503
+ 1,
1504
+ 1
1505
+ ],
1506
+ "ood_8hop": [
1507
+ 1,
1508
+ 1,
1509
+ 0,
1510
+ 1,
1511
+ 1,
1512
+ 1,
1513
+ 0,
1514
+ 1,
1515
+ 1,
1516
+ 0,
1517
+ 1,
1518
+ 0,
1519
+ 1,
1520
+ 1,
1521
+ 1,
1522
+ 1,
1523
+ 1,
1524
+ 1,
1525
+ 0,
1526
+ 1,
1527
+ 1,
1528
+ 1,
1529
+ 1,
1530
+ 1,
1531
+ 1,
1532
+ 1,
1533
+ 0,
1534
+ 0,
1535
+ 1,
1536
+ 1,
1537
+ 1,
1538
+ 1,
1539
+ 0,
1540
+ 1,
1541
+ 0,
1542
+ 1,
1543
+ 1,
1544
+ 1,
1545
+ 0,
1546
+ 1,
1547
+ 1,
1548
+ 1,
1549
+ 0,
1550
+ 1,
1551
+ 1,
1552
+ 1,
1553
+ 1,
1554
+ 0,
1555
+ 1,
1556
+ 1,
1557
+ 1,
1558
+ 1,
1559
+ 1,
1560
+ 1,
1561
+ 1,
1562
+ 0,
1563
+ 1,
1564
+ 0,
1565
+ 1,
1566
+ 1,
1567
+ 0,
1568
+ 1,
1569
+ 0,
1570
+ 1,
1571
+ 1,
1572
+ 1,
1573
+ 1,
1574
+ 0,
1575
+ 1,
1576
+ 1,
1577
+ 1,
1578
+ 1,
1579
+ 1,
1580
+ 1,
1581
+ 1,
1582
+ 0,
1583
+ 1,
1584
+ 1,
1585
+ 1,
1586
+ 0,
1587
+ 1,
1588
+ 1,
1589
+ 1,
1590
+ 1,
1591
+ 1,
1592
+ 1,
1593
+ 1,
1594
+ 1,
1595
+ 1,
1596
+ 0,
1597
+ 1,
1598
+ 0,
1599
+ 1,
1600
+ 1,
1601
+ 1,
1602
+ 1,
1603
+ 1,
1604
+ 1,
1605
+ 1,
1606
+ 0,
1607
+ 1,
1608
+ 1,
1609
+ 0,
1610
+ 1,
1611
+ 0,
1612
+ 0,
1613
+ 1,
1614
+ 0,
1615
+ 1,
1616
+ 1,
1617
+ 0,
1618
+ 1,
1619
+ 1,
1620
+ 0,
1621
+ 0,
1622
+ 1,
1623
+ 1,
1624
+ 1,
1625
+ 0,
1626
+ 0,
1627
+ 1,
1628
+ 1,
1629
+ 1,
1630
+ 1,
1631
+ 1,
1632
+ 0,
1633
+ 1,
1634
+ 1,
1635
+ 1,
1636
+ 1,
1637
+ 0,
1638
+ 1,
1639
+ 0,
1640
+ 1,
1641
+ 1,
1642
+ 1,
1643
+ 1,
1644
+ 1,
1645
+ 1,
1646
+ 0,
1647
+ 1,
1648
+ 1,
1649
+ 0,
1650
+ 1,
1651
+ 1,
1652
+ 1,
1653
+ 0,
1654
+ 1,
1655
+ 1,
1656
+ 1,
1657
+ 1,
1658
+ 1,
1659
+ 1,
1660
+ 0,
1661
+ 1,
1662
+ 0,
1663
+ 1,
1664
+ 0,
1665
+ 1,
1666
+ 1,
1667
+ 1,
1668
+ 1,
1669
+ 0,
1670
+ 0,
1671
+ 1,
1672
+ 0,
1673
+ 1,
1674
+ 1,
1675
+ 0,
1676
+ 1,
1677
+ 1,
1678
+ 1,
1679
+ 1,
1680
+ 0,
1681
+ 1,
1682
+ 1,
1683
+ 0,
1684
+ 0,
1685
+ 1,
1686
+ 0,
1687
+ 0,
1688
+ 1,
1689
+ 0,
1690
+ 1,
1691
+ 1,
1692
+ 0,
1693
+ 1,
1694
+ 0,
1695
+ 0,
1696
+ 1,
1697
+ 1,
1698
+ 0,
1699
+ 1,
1700
+ 1,
1701
+ 1,
1702
+ 1,
1703
+ 1,
1704
+ 0,
1705
+ 1,
1706
+ 0,
1707
+ 1,
1708
+ 0,
1709
+ 1,
1710
+ 1,
1711
+ 0,
1712
+ 1,
1713
+ 1,
1714
+ 1,
1715
+ 1,
1716
+ 1,
1717
+ 1,
1718
+ 1,
1719
+ 1,
1720
+ 1,
1721
+ 1,
1722
+ 1,
1723
+ 1,
1724
+ 1,
1725
+ 1,
1726
+ 1,
1727
+ 1,
1728
+ 0,
1729
+ 1,
1730
+ 1,
1731
+ 0,
1732
+ 1,
1733
+ 1,
1734
+ 0,
1735
+ 0,
1736
+ 0,
1737
+ 0,
1738
+ 1,
1739
+ 1,
1740
+ 1,
1741
+ 0,
1742
+ 1,
1743
+ 0,
1744
+ 0,
1745
+ 0,
1746
+ 1,
1747
+ 1,
1748
+ 1,
1749
+ 0,
1750
+ 0,
1751
+ 0,
1752
+ 0,
1753
+ 0,
1754
+ 0,
1755
+ 0,
1756
+ 1,
1757
+ 1,
1758
+ 0,
1759
+ 1,
1760
+ 1,
1761
+ 1,
1762
+ 1,
1763
+ 1,
1764
+ 1,
1765
+ 1,
1766
+ 1,
1767
+ 1,
1768
+ 1,
1769
+ 0,
1770
+ 1,
1771
+ 0,
1772
+ 1,
1773
+ 1,
1774
+ 1,
1775
+ 1,
1776
+ 1,
1777
+ 1,
1778
+ 1,
1779
+ 1,
1780
+ 1,
1781
+ 1,
1782
+ 1,
1783
+ 1,
1784
+ 1,
1785
+ 1,
1786
+ 1,
1787
+ 1,
1788
+ 1,
1789
+ 1,
1790
+ 1,
1791
+ 0,
1792
+ 1,
1793
+ 1,
1794
+ 1,
1795
+ 1,
1796
+ 0,
1797
+ 1,
1798
+ 1,
1799
+ 1,
1800
+ 1,
1801
+ 1,
1802
+ 1,
1803
+ 1,
1804
+ 1,
1805
+ 1,
1806
+ 1,
1807
+ 1,
1808
+ 1,
1809
+ 0,
1810
+ 1,
1811
+ 1,
1812
+ 1,
1813
+ 0,
1814
+ 0,
1815
+ 0,
1816
+ 1,
1817
+ 1,
1818
+ 1,
1819
+ 1,
1820
+ 1,
1821
+ 1,
1822
+ 1,
1823
+ 0,
1824
+ 1,
1825
+ 1,
1826
+ 1,
1827
+ 1,
1828
+ 0,
1829
+ 0,
1830
+ 0,
1831
+ 1,
1832
+ 1,
1833
+ 0,
1834
+ 1,
1835
+ 1,
1836
+ 0,
1837
+ 1,
1838
+ 1,
1839
+ 0,
1840
+ 1,
1841
+ 1,
1842
+ 1,
1843
+ 1,
1844
+ 1,
1845
+ 1,
1846
+ 1,
1847
+ 1,
1848
+ 1,
1849
+ 1,
1850
+ 0,
1851
+ 0,
1852
+ 1,
1853
+ 1,
1854
+ 1,
1855
+ 1,
1856
+ 1,
1857
+ 1,
1858
+ 1,
1859
+ 1,
1860
+ 0,
1861
+ 1,
1862
+ 1,
1863
+ 1,
1864
+ 1,
1865
+ 0,
1866
+ 1,
1867
+ 1,
1868
+ 1,
1869
+ 1,
1870
+ 1,
1871
+ 0,
1872
+ 1,
1873
+ 0,
1874
+ 1,
1875
+ 0,
1876
+ 1,
1877
+ 0,
1878
+ 1,
1879
+ 1,
1880
+ 1,
1881
+ 1,
1882
+ 1,
1883
+ 0,
1884
+ 0,
1885
+ 1,
1886
+ 1,
1887
+ 1,
1888
+ 1,
1889
+ 0,
1890
+ 1,
1891
+ 1,
1892
+ 1,
1893
+ 0,
1894
+ 0,
1895
+ 1,
1896
+ 1,
1897
+ 0,
1898
+ 1,
1899
+ 0,
1900
+ 0,
1901
+ 1,
1902
+ 1,
1903
+ 0,
1904
+ 1,
1905
+ 0,
1906
+ 1,
1907
+ 1,
1908
+ 1,
1909
+ 0,
1910
+ 1,
1911
+ 1,
1912
+ 1,
1913
+ 1,
1914
+ 1,
1915
+ 1,
1916
+ 0,
1917
+ 1,
1918
+ 1,
1919
+ 1,
1920
+ 1,
1921
+ 1,
1922
+ 0,
1923
+ 1,
1924
+ 0,
1925
+ 1,
1926
+ 1,
1927
+ 1,
1928
+ 1,
1929
+ 1,
1930
+ 1,
1931
+ 1,
1932
+ 1,
1933
+ 0,
1934
+ 1,
1935
+ 1,
1936
+ 1,
1937
+ 1,
1938
+ 1,
1939
+ 1,
1940
+ 1,
1941
+ 1,
1942
+ 1,
1943
+ 1,
1944
+ 1,
1945
+ 1,
1946
+ 1,
1947
+ 1,
1948
+ 1,
1949
+ 1,
1950
+ 1,
1951
+ 0,
1952
+ 0,
1953
+ 0,
1954
+ 1,
1955
+ 1,
1956
+ 1,
1957
+ 0,
1958
+ 1,
1959
+ 1,
1960
+ 1,
1961
+ 1,
1962
+ 0,
1963
+ 1,
1964
+ 1,
1965
+ 1,
1966
+ 1,
1967
+ 0,
1968
+ 0,
1969
+ 1,
1970
+ 1,
1971
+ 0,
1972
+ 1,
1973
+ 1,
1974
+ 1,
1975
+ 1,
1976
+ 1,
1977
+ 0,
1978
+ 1,
1979
+ 1,
1980
+ 1,
1981
+ 1,
1982
+ 1,
1983
+ 1,
1984
+ 1,
1985
+ 1,
1986
+ 0,
1987
+ 1,
1988
+ 1,
1989
+ 1,
1990
+ 1,
1991
+ 1,
1992
+ 0,
1993
+ 1,
1994
+ 0,
1995
+ 1,
1996
+ 1,
1997
+ 1,
1998
+ 1,
1999
+ 1,
2000
+ 1,
2001
+ 1,
2002
+ 1,
2003
+ 1,
2004
+ 1,
2005
+ 1,
2006
+ 1,
2007
+ 1,
2008
+ 1,
2009
+ 0,
2010
+ 1,
2011
+ 0,
2012
+ 0,
2013
+ 1,
2014
+ 1,
2015
+ 1,
2016
+ 1,
2017
+ 1,
2018
+ 0,
2019
+ 1,
2020
+ 0,
2021
+ 1,
2022
+ 1,
2023
+ 1,
2024
+ 0,
2025
+ 0,
2026
+ 1,
2027
+ 1,
2028
+ 1,
2029
+ 1,
2030
+ 1,
2031
+ 1,
2032
+ 1,
2033
+ 0,
2034
+ 1,
2035
+ 1,
2036
+ 1,
2037
+ 0,
2038
+ 1,
2039
+ 1,
2040
+ 0,
2041
+ 0,
2042
+ 1,
2043
+ 1,
2044
+ 1,
2045
+ 1,
2046
+ 1,
2047
+ 1,
2048
+ 0,
2049
+ 1,
2050
+ 0,
2051
+ 1,
2052
+ 1,
2053
+ 1,
2054
+ 1,
2055
+ 0,
2056
+ 1,
2057
+ 0,
2058
+ 1,
2059
+ 0,
2060
+ 0,
2061
+ 1,
2062
+ 0,
2063
+ 1,
2064
+ 1,
2065
+ 1,
2066
+ 1,
2067
+ 0,
2068
+ 1,
2069
+ 0,
2070
+ 1,
2071
+ 1,
2072
+ 0,
2073
+ 1,
2074
+ 1,
2075
+ 0,
2076
+ 1,
2077
+ 1,
2078
+ 1,
2079
+ 1,
2080
+ 1,
2081
+ 1,
2082
+ 1,
2083
+ 1,
2084
+ 1,
2085
+ 0,
2086
+ 0,
2087
+ 1,
2088
+ 1,
2089
+ 1,
2090
+ 0,
2091
+ 1,
2092
+ 1,
2093
+ 1,
2094
+ 1,
2095
+ 1,
2096
+ 0,
2097
+ 1,
2098
+ 1,
2099
+ 1,
2100
+ 1,
2101
+ 1,
2102
+ 0,
2103
+ 0,
2104
+ 1,
2105
+ 1,
2106
+ 1,
2107
+ 1,
2108
+ 1,
2109
+ 1,
2110
+ 1,
2111
+ 1,
2112
+ 0,
2113
+ 1,
2114
+ 1,
2115
+ 1,
2116
+ 1,
2117
+ 1,
2118
+ 0,
2119
+ 0,
2120
+ 1,
2121
+ 1,
2122
+ 0,
2123
+ 0,
2124
+ 0,
2125
+ 0,
2126
+ 1,
2127
+ 1,
2128
+ 1,
2129
+ 0,
2130
+ 0,
2131
+ 1,
2132
+ 0,
2133
+ 1,
2134
+ 1,
2135
+ 1,
2136
+ 1,
2137
+ 0,
2138
+ 0,
2139
+ 1,
2140
+ 1,
2141
+ 1,
2142
+ 1,
2143
+ 0,
2144
+ 1,
2145
+ 1,
2146
+ 1,
2147
+ 0,
2148
+ 0,
2149
+ 0,
2150
+ 1,
2151
+ 1,
2152
+ 1,
2153
+ 1,
2154
+ 0,
2155
+ 1,
2156
+ 0,
2157
+ 1,
2158
+ 0,
2159
+ 1,
2160
+ 1,
2161
+ 0,
2162
+ 1,
2163
+ 0,
2164
+ 0,
2165
+ 1,
2166
+ 1,
2167
+ 1,
2168
+ 1,
2169
+ 1,
2170
+ 0,
2171
+ 0,
2172
+ 1,
2173
+ 1,
2174
+ 0,
2175
+ 1,
2176
+ 1,
2177
+ 1,
2178
+ 1,
2179
+ 0,
2180
+ 0,
2181
+ 1,
2182
+ 0,
2183
+ 1,
2184
+ 1,
2185
+ 1,
2186
+ 0,
2187
+ 1,
2188
+ 1,
2189
+ 1,
2190
+ 0,
2191
+ 0,
2192
+ 1,
2193
+ 1,
2194
+ 1,
2195
+ 1,
2196
+ 1,
2197
+ 1,
2198
+ 1,
2199
+ 1,
2200
+ 1,
2201
+ 1,
2202
+ 1,
2203
+ 1,
2204
+ 1,
2205
+ 1,
2206
+ 1,
2207
+ 1,
2208
+ 1,
2209
+ 1,
2210
+ 1,
2211
+ 1,
2212
+ 1,
2213
+ 0,
2214
+ 1,
2215
+ 1,
2216
+ 1,
2217
+ 1,
2218
+ 1,
2219
+ 0,
2220
+ 0,
2221
+ 1,
2222
+ 1,
2223
+ 1,
2224
+ 0,
2225
+ 1,
2226
+ 1,
2227
+ 1,
2228
+ 1,
2229
+ 1,
2230
+ 0,
2231
+ 1,
2232
+ 0,
2233
+ 1,
2234
+ 1,
2235
+ 0,
2236
+ 0,
2237
+ 1,
2238
+ 0,
2239
+ 0,
2240
+ 1,
2241
+ 0,
2242
+ 1,
2243
+ 1,
2244
+ 1,
2245
+ 1,
2246
+ 1,
2247
+ 1,
2248
+ 1,
2249
+ 0,
2250
+ 1,
2251
+ 1,
2252
+ 1,
2253
+ 1,
2254
+ 1,
2255
+ 1,
2256
+ 1,
2257
+ 0,
2258
+ 1,
2259
+ 1,
2260
+ 1,
2261
+ 1,
2262
+ 0,
2263
+ 1,
2264
+ 0,
2265
+ 0,
2266
+ 0,
2267
+ 1,
2268
+ 1,
2269
+ 0,
2270
+ 1,
2271
+ 1,
2272
+ 1,
2273
+ 1,
2274
+ 0,
2275
+ 0,
2276
+ 0,
2277
+ 1,
2278
+ 1,
2279
+ 1,
2280
+ 0,
2281
+ 1,
2282
+ 1,
2283
+ 0,
2284
+ 1,
2285
+ 0,
2286
+ 1,
2287
+ 1,
2288
+ 0,
2289
+ 1,
2290
+ 1,
2291
+ 1,
2292
+ 1,
2293
+ 1,
2294
+ 1,
2295
+ 1,
2296
+ 1,
2297
+ 1,
2298
+ 1,
2299
+ 1,
2300
+ 1,
2301
+ 1,
2302
+ 1,
2303
+ 0,
2304
+ 1,
2305
+ 1,
2306
+ 1,
2307
+ 1,
2308
+ 1,
2309
+ 1,
2310
+ 1,
2311
+ 1,
2312
+ 1,
2313
+ 1,
2314
+ 0,
2315
+ 1,
2316
+ 0,
2317
+ 0,
2318
+ 1,
2319
+ 1,
2320
+ 0,
2321
+ 1,
2322
+ 1,
2323
+ 1,
2324
+ 1,
2325
+ 1,
2326
+ 0,
2327
+ 1,
2328
+ 1,
2329
+ 1,
2330
+ 1,
2331
+ 1,
2332
+ 1,
2333
+ 1,
2334
+ 1,
2335
+ 1,
2336
+ 1,
2337
+ 1,
2338
+ 0,
2339
+ 1,
2340
+ 1,
2341
+ 0,
2342
+ 1,
2343
+ 0,
2344
+ 1,
2345
+ 1,
2346
+ 1,
2347
+ 1,
2348
+ 1,
2349
+ 1,
2350
+ 1,
2351
+ 0,
2352
+ 1,
2353
+ 0,
2354
+ 1,
2355
+ 1,
2356
+ 1,
2357
+ 0,
2358
+ 0,
2359
+ 0,
2360
+ 0,
2361
+ 0,
2362
+ 1,
2363
+ 1,
2364
+ 0,
2365
+ 1,
2366
+ 1,
2367
+ 1,
2368
+ 1,
2369
+ 1,
2370
+ 1,
2371
+ 1,
2372
+ 1,
2373
+ 1,
2374
+ 0,
2375
+ 1,
2376
+ 1,
2377
+ 0,
2378
+ 1,
2379
+ 0,
2380
+ 1,
2381
+ 1,
2382
+ 1,
2383
+ 1,
2384
+ 1,
2385
+ 1,
2386
+ 0,
2387
+ 1,
2388
+ 1,
2389
+ 1,
2390
+ 1,
2391
+ 0,
2392
+ 1,
2393
+ 0,
2394
+ 0,
2395
+ 0,
2396
+ 0,
2397
+ 1,
2398
+ 1,
2399
+ 1,
2400
+ 0,
2401
+ 1,
2402
+ 0,
2403
+ 1,
2404
+ 1,
2405
+ 1,
2406
+ 0,
2407
+ 1,
2408
+ 1,
2409
+ 1,
2410
+ 1,
2411
+ 1,
2412
+ 0,
2413
+ 1,
2414
+ 1,
2415
+ 1,
2416
+ 1,
2417
+ 1,
2418
+ 1,
2419
+ 1,
2420
+ 1,
2421
+ 0,
2422
+ 1,
2423
+ 1,
2424
+ 1,
2425
+ 1,
2426
+ 1,
2427
+ 0,
2428
+ 1,
2429
+ 0,
2430
+ 1,
2431
+ 1,
2432
+ 0,
2433
+ 1,
2434
+ 1,
2435
+ 1,
2436
+ 0,
2437
+ 1,
2438
+ 1,
2439
+ 0,
2440
+ 0,
2441
+ 1,
2442
+ 1,
2443
+ 1,
2444
+ 0,
2445
+ 1,
2446
+ 0,
2447
+ 1,
2448
+ 1,
2449
+ 1,
2450
+ 1,
2451
+ 1,
2452
+ 1,
2453
+ 1,
2454
+ 1,
2455
+ 1,
2456
+ 1,
2457
+ 0,
2458
+ 0,
2459
+ 1,
2460
+ 0,
2461
+ 1,
2462
+ 1,
2463
+ 0,
2464
+ 0,
2465
+ 0,
2466
+ 0,
2467
+ 1,
2468
+ 1,
2469
+ 1,
2470
+ 1,
2471
+ 1,
2472
+ 0,
2473
+ 1,
2474
+ 1,
2475
+ 1,
2476
+ 1,
2477
+ 0,
2478
+ 1,
2479
+ 1,
2480
+ 0,
2481
+ 1,
2482
+ 0,
2483
+ 1,
2484
+ 1,
2485
+ 1,
2486
+ 0,
2487
+ 1,
2488
+ 0,
2489
+ 1,
2490
+ 0,
2491
+ 1,
2492
+ 1,
2493
+ 0,
2494
+ 1,
2495
+ 1,
2496
+ 1,
2497
+ 0,
2498
+ 1,
2499
+ 1,
2500
+ 1,
2501
+ 1,
2502
+ 1,
2503
+ 0,
2504
+ 0,
2505
+ 0,
2506
+ 1
2507
+ ],
2508
+ "ood_dag": [
2509
+ 1,
2510
+ 1,
2511
+ 0,
2512
+ 1,
2513
+ 0,
2514
+ 1,
2515
+ 1,
2516
+ 1,
2517
+ 1,
2518
+ 1,
2519
+ 1,
2520
+ 1,
2521
+ 1,
2522
+ 1,
2523
+ 1,
2524
+ 0,
2525
+ 0,
2526
+ 1,
2527
+ 1,
2528
+ 1,
2529
+ 1,
2530
+ 1,
2531
+ 0,
2532
+ 0,
2533
+ 0,
2534
+ 0,
2535
+ 1,
2536
+ 1,
2537
+ 1,
2538
+ 1,
2539
+ 1,
2540
+ 1,
2541
+ 1,
2542
+ 0,
2543
+ 0,
2544
+ 1,
2545
+ 0,
2546
+ 1,
2547
+ 1,
2548
+ 1,
2549
+ 0,
2550
+ 1,
2551
+ 1,
2552
+ 1,
2553
+ 1,
2554
+ 0,
2555
+ 0,
2556
+ 1,
2557
+ 1,
2558
+ 0,
2559
+ 1,
2560
+ 1,
2561
+ 1,
2562
+ 1,
2563
+ 1,
2564
+ 0,
2565
+ 0,
2566
+ 1,
2567
+ 0,
2568
+ 1,
2569
+ 1,
2570
+ 0,
2571
+ 1,
2572
+ 1,
2573
+ 0,
2574
+ 1,
2575
+ 1,
2576
+ 1,
2577
+ 1,
2578
+ 1,
2579
+ 1,
2580
+ 0,
2581
+ 0,
2582
+ 0,
2583
+ 1,
2584
+ 1,
2585
+ 1,
2586
+ 1,
2587
+ 1,
2588
+ 0,
2589
+ 1,
2590
+ 1,
2591
+ 1,
2592
+ 1,
2593
+ 1,
2594
+ 0,
2595
+ 1,
2596
+ 1,
2597
+ 1,
2598
+ 0,
2599
+ 1,
2600
+ 1,
2601
+ 1,
2602
+ 1,
2603
+ 1,
2604
+ 1,
2605
+ 0,
2606
+ 1,
2607
+ 1,
2608
+ 1,
2609
+ 1,
2610
+ 1,
2611
+ 1,
2612
+ 1,
2613
+ 1,
2614
+ 1,
2615
+ 1,
2616
+ 1,
2617
+ 1,
2618
+ 1,
2619
+ 1,
2620
+ 1,
2621
+ 0,
2622
+ 0,
2623
+ 1,
2624
+ 0,
2625
+ 1,
2626
+ 0,
2627
+ 0,
2628
+ 1,
2629
+ 0,
2630
+ 1,
2631
+ 0,
2632
+ 1,
2633
+ 0,
2634
+ 0,
2635
+ 0,
2636
+ 0,
2637
+ 0,
2638
+ 0,
2639
+ 0,
2640
+ 0,
2641
+ 0,
2642
+ 1,
2643
+ 0,
2644
+ 0,
2645
+ 0,
2646
+ 1,
2647
+ 0,
2648
+ 1,
2649
+ 1,
2650
+ 1,
2651
+ 1,
2652
+ 1,
2653
+ 1,
2654
+ 1,
2655
+ 1,
2656
+ 1,
2657
+ 1,
2658
+ 0,
2659
+ 0,
2660
+ 0,
2661
+ 0,
2662
+ 1,
2663
+ 1,
2664
+ 1,
2665
+ 1,
2666
+ 0,
2667
+ 1,
2668
+ 0,
2669
+ 1,
2670
+ 1,
2671
+ 1,
2672
+ 1,
2673
+ 1,
2674
+ 1,
2675
+ 0,
2676
+ 1,
2677
+ 1,
2678
+ 1,
2679
+ 1,
2680
+ 0,
2681
+ 0,
2682
+ 1,
2683
+ 0,
2684
+ 1,
2685
+ 1,
2686
+ 1,
2687
+ 1,
2688
+ 0,
2689
+ 0,
2690
+ 1,
2691
+ 1,
2692
+ 0,
2693
+ 1,
2694
+ 0,
2695
+ 0,
2696
+ 1,
2697
+ 0,
2698
+ 0,
2699
+ 1,
2700
+ 0,
2701
+ 1,
2702
+ 0,
2703
+ 1,
2704
+ 1,
2705
+ 1,
2706
+ 1,
2707
+ 0,
2708
+ 1,
2709
+ 0,
2710
+ 1,
2711
+ 1,
2712
+ 0,
2713
+ 1,
2714
+ 0,
2715
+ 0,
2716
+ 1,
2717
+ 0,
2718
+ 1,
2719
+ 0,
2720
+ 1,
2721
+ 1,
2722
+ 0,
2723
+ 1,
2724
+ 1,
2725
+ 1,
2726
+ 1,
2727
+ 0,
2728
+ 0,
2729
+ 0,
2730
+ 0,
2731
+ 1,
2732
+ 1,
2733
+ 0,
2734
+ 0,
2735
+ 0,
2736
+ 1,
2737
+ 1,
2738
+ 0,
2739
+ 1,
2740
+ 1,
2741
+ 0,
2742
+ 1,
2743
+ 0,
2744
+ 1,
2745
+ 1,
2746
+ 1,
2747
+ 0,
2748
+ 0,
2749
+ 1,
2750
+ 1,
2751
+ 1,
2752
+ 0,
2753
+ 1,
2754
+ 1,
2755
+ 1,
2756
+ 0,
2757
+ 1,
2758
+ 1,
2759
+ 1,
2760
+ 0,
2761
+ 0,
2762
+ 1,
2763
+ 1,
2764
+ 0,
2765
+ 0,
2766
+ 0,
2767
+ 1,
2768
+ 1,
2769
+ 0,
2770
+ 0,
2771
+ 1,
2772
+ 0,
2773
+ 1,
2774
+ 1,
2775
+ 1,
2776
+ 1,
2777
+ 1,
2778
+ 0,
2779
+ 1,
2780
+ 1,
2781
+ 0,
2782
+ 1,
2783
+ 1,
2784
+ 1,
2785
+ 1,
2786
+ 1,
2787
+ 1,
2788
+ 1,
2789
+ 1,
2790
+ 0,
2791
+ 0,
2792
+ 1,
2793
+ 0,
2794
+ 0,
2795
+ 1,
2796
+ 1,
2797
+ 1,
2798
+ 1,
2799
+ 0,
2800
+ 1,
2801
+ 0,
2802
+ 1,
2803
+ 0,
2804
+ 1,
2805
+ 0,
2806
+ 1,
2807
+ 1,
2808
+ 1,
2809
+ 1,
2810
+ 1,
2811
+ 0,
2812
+ 0,
2813
+ 1,
2814
+ 1,
2815
+ 1,
2816
+ 0,
2817
+ 1,
2818
+ 0,
2819
+ 0,
2820
+ 1,
2821
+ 1,
2822
+ 0,
2823
+ 1,
2824
+ 1,
2825
+ 0,
2826
+ 1,
2827
+ 1,
2828
+ 1,
2829
+ 1,
2830
+ 1,
2831
+ 0,
2832
+ 1,
2833
+ 1,
2834
+ 1,
2835
+ 0,
2836
+ 0,
2837
+ 0,
2838
+ 0,
2839
+ 1,
2840
+ 1,
2841
+ 0,
2842
+ 1,
2843
+ 1,
2844
+ 0,
2845
+ 1,
2846
+ 0,
2847
+ 1,
2848
+ 1,
2849
+ 1,
2850
+ 1,
2851
+ 1,
2852
+ 0,
2853
+ 0,
2854
+ 0,
2855
+ 1,
2856
+ 1,
2857
+ 0,
2858
+ 1,
2859
+ 1,
2860
+ 0,
2861
+ 1,
2862
+ 1,
2863
+ 0,
2864
+ 0,
2865
+ 1,
2866
+ 1,
2867
+ 1,
2868
+ 1,
2869
+ 0,
2870
+ 1,
2871
+ 0,
2872
+ 1,
2873
+ 0,
2874
+ 0,
2875
+ 1,
2876
+ 1,
2877
+ 0,
2878
+ 0,
2879
+ 1,
2880
+ 1,
2881
+ 1,
2882
+ 1,
2883
+ 1,
2884
+ 0,
2885
+ 1,
2886
+ 0,
2887
+ 0,
2888
+ 1,
2889
+ 1,
2890
+ 1,
2891
+ 0,
2892
+ 1,
2893
+ 1,
2894
+ 0,
2895
+ 0,
2896
+ 0,
2897
+ 1,
2898
+ 1,
2899
+ 1,
2900
+ 0,
2901
+ 0,
2902
+ 1,
2903
+ 0,
2904
+ 0,
2905
+ 1,
2906
+ 1,
2907
+ 0,
2908
+ 1,
2909
+ 0,
2910
+ 1,
2911
+ 0,
2912
+ 1,
2913
+ 0,
2914
+ 1,
2915
+ 1,
2916
+ 1,
2917
+ 0,
2918
+ 1,
2919
+ 1,
2920
+ 1,
2921
+ 0,
2922
+ 0,
2923
+ 1,
2924
+ 1,
2925
+ 0,
2926
+ 0,
2927
+ 1,
2928
+ 1,
2929
+ 1,
2930
+ 0,
2931
+ 1,
2932
+ 1,
2933
+ 0,
2934
+ 0,
2935
+ 1,
2936
+ 0,
2937
+ 1,
2938
+ 0,
2939
+ 1,
2940
+ 1,
2941
+ 0,
2942
+ 1,
2943
+ 0,
2944
+ 1,
2945
+ 1,
2946
+ 1,
2947
+ 0,
2948
+ 1,
2949
+ 1,
2950
+ 0,
2951
+ 0,
2952
+ 1,
2953
+ 0,
2954
+ 0,
2955
+ 1,
2956
+ 0,
2957
+ 1,
2958
+ 0,
2959
+ 1,
2960
+ 1,
2961
+ 1,
2962
+ 0,
2963
+ 0,
2964
+ 1,
2965
+ 0,
2966
+ 1,
2967
+ 1,
2968
+ 1,
2969
+ 0,
2970
+ 0,
2971
+ 0,
2972
+ 1,
2973
+ 0,
2974
+ 0,
2975
+ 1,
2976
+ 0,
2977
+ 0,
2978
+ 0,
2979
+ 0,
2980
+ 0,
2981
+ 0,
2982
+ 1,
2983
+ 1,
2984
+ 1,
2985
+ 0,
2986
+ 1,
2987
+ 1,
2988
+ 0,
2989
+ 1,
2990
+ 1,
2991
+ 1,
2992
+ 1,
2993
+ 1,
2994
+ 0,
2995
+ 1,
2996
+ 1,
2997
+ 1,
2998
+ 1,
2999
+ 0,
3000
+ 0,
3001
+ 0,
3002
+ 1,
3003
+ 1,
3004
+ 1,
3005
+ 1,
3006
+ 1,
3007
+ 1,
3008
+ 0,
3009
+ 0,
3010
+ 0,
3011
+ 0,
3012
+ 1,
3013
+ 1,
3014
+ 0,
3015
+ 0,
3016
+ 1,
3017
+ 0,
3018
+ 0,
3019
+ 0,
3020
+ 1,
3021
+ 1,
3022
+ 1,
3023
+ 1,
3024
+ 0,
3025
+ 1,
3026
+ 1,
3027
+ 0,
3028
+ 1,
3029
+ 1,
3030
+ 0,
3031
+ 1,
3032
+ 1,
3033
+ 1,
3034
+ 1,
3035
+ 1,
3036
+ 0,
3037
+ 1,
3038
+ 1,
3039
+ 1,
3040
+ 1,
3041
+ 1,
3042
+ 0,
3043
+ 1,
3044
+ 1,
3045
+ 1,
3046
+ 1,
3047
+ 1,
3048
+ 1,
3049
+ 1,
3050
+ 1,
3051
+ 1,
3052
+ 0,
3053
+ 1,
3054
+ 1,
3055
+ 1,
3056
+ 1,
3057
+ 1,
3058
+ 0,
3059
+ 0,
3060
+ 0,
3061
+ 1,
3062
+ 1,
3063
+ 1,
3064
+ 0,
3065
+ 1,
3066
+ 1,
3067
+ 1,
3068
+ 1,
3069
+ 1,
3070
+ 1,
3071
+ 1,
3072
+ 1,
3073
+ 1,
3074
+ 1,
3075
+ 1,
3076
+ 1,
3077
+ 1,
3078
+ 0,
3079
+ 1,
3080
+ 1,
3081
+ 1,
3082
+ 0,
3083
+ 0,
3084
+ 0,
3085
+ 1,
3086
+ 1,
3087
+ 1,
3088
+ 1,
3089
+ 0,
3090
+ 0,
3091
+ 1,
3092
+ 1,
3093
+ 1,
3094
+ 1,
3095
+ 1,
3096
+ 1,
3097
+ 1,
3098
+ 1,
3099
+ 0,
3100
+ 0,
3101
+ 0,
3102
+ 1,
3103
+ 1,
3104
+ 0,
3105
+ 0,
3106
+ 1,
3107
+ 0,
3108
+ 1,
3109
+ 0,
3110
+ 1,
3111
+ 0,
3112
+ 1,
3113
+ 0,
3114
+ 1,
3115
+ 1,
3116
+ 0,
3117
+ 1,
3118
+ 0,
3119
+ 1,
3120
+ 1,
3121
+ 1,
3122
+ 1,
3123
+ 1,
3124
+ 1,
3125
+ 0,
3126
+ 1,
3127
+ 0,
3128
+ 1,
3129
+ 1,
3130
+ 1,
3131
+ 1,
3132
+ 0,
3133
+ 1,
3134
+ 1,
3135
+ 0,
3136
+ 0,
3137
+ 0,
3138
+ 0,
3139
+ 0,
3140
+ 0,
3141
+ 0,
3142
+ 1,
3143
+ 0,
3144
+ 1,
3145
+ 1,
3146
+ 1,
3147
+ 0,
3148
+ 1,
3149
+ 0,
3150
+ 1,
3151
+ 1,
3152
+ 0,
3153
+ 1,
3154
+ 1,
3155
+ 0,
3156
+ 0,
3157
+ 1,
3158
+ 0,
3159
+ 1,
3160
+ 1,
3161
+ 0,
3162
+ 1,
3163
+ 0,
3164
+ 0,
3165
+ 1,
3166
+ 1,
3167
+ 0,
3168
+ 1,
3169
+ 1,
3170
+ 1,
3171
+ 1,
3172
+ 0,
3173
+ 1,
3174
+ 1,
3175
+ 1,
3176
+ 1,
3177
+ 0,
3178
+ 0,
3179
+ 1,
3180
+ 1,
3181
+ 0,
3182
+ 1,
3183
+ 1,
3184
+ 1,
3185
+ 1,
3186
+ 1,
3187
+ 0,
3188
+ 0,
3189
+ 1,
3190
+ 0,
3191
+ 0,
3192
+ 0,
3193
+ 1,
3194
+ 0,
3195
+ 1,
3196
+ 1,
3197
+ 1,
3198
+ 0,
3199
+ 0,
3200
+ 0,
3201
+ 1,
3202
+ 0,
3203
+ 1,
3204
+ 1,
3205
+ 0,
3206
+ 1,
3207
+ 0,
3208
+ 1,
3209
+ 0,
3210
+ 1,
3211
+ 1,
3212
+ 1,
3213
+ 0,
3214
+ 1,
3215
+ 0,
3216
+ 1,
3217
+ 1,
3218
+ 1,
3219
+ 1,
3220
+ 1,
3221
+ 0,
3222
+ 0,
3223
+ 0,
3224
+ 1,
3225
+ 1,
3226
+ 1,
3227
+ 1,
3228
+ 1,
3229
+ 0,
3230
+ 0,
3231
+ 1,
3232
+ 1,
3233
+ 1,
3234
+ 0,
3235
+ 0,
3236
+ 1,
3237
+ 0,
3238
+ 1,
3239
+ 0,
3240
+ 0,
3241
+ 1,
3242
+ 1,
3243
+ 1,
3244
+ 1,
3245
+ 0,
3246
+ 0,
3247
+ 1,
3248
+ 1,
3249
+ 1,
3250
+ 0,
3251
+ 1,
3252
+ 1,
3253
+ 0,
3254
+ 1,
3255
+ 0,
3256
+ 1,
3257
+ 0,
3258
+ 1,
3259
+ 1,
3260
+ 1,
3261
+ 1,
3262
+ 1,
3263
+ 0,
3264
+ 0,
3265
+ 0,
3266
+ 0,
3267
+ 0,
3268
+ 1,
3269
+ 0,
3270
+ 1,
3271
+ 0,
3272
+ 0,
3273
+ 0,
3274
+ 1,
3275
+ 1,
3276
+ 0,
3277
+ 1,
3278
+ 1,
3279
+ 0,
3280
+ 0,
3281
+ 1,
3282
+ 1,
3283
+ 0,
3284
+ 0,
3285
+ 0,
3286
+ 1,
3287
+ 0,
3288
+ 0,
3289
+ 1,
3290
+ 0,
3291
+ 1,
3292
+ 0,
3293
+ 1,
3294
+ 1,
3295
+ 1,
3296
+ 1,
3297
+ 1,
3298
+ 1,
3299
+ 0,
3300
+ 1,
3301
+ 1,
3302
+ 1,
3303
+ 1,
3304
+ 1,
3305
+ 1,
3306
+ 1,
3307
+ 0,
3308
+ 0,
3309
+ 0,
3310
+ 1,
3311
+ 1,
3312
+ 0,
3313
+ 0,
3314
+ 0,
3315
+ 0,
3316
+ 1,
3317
+ 0,
3318
+ 1,
3319
+ 0,
3320
+ 1,
3321
+ 1,
3322
+ 1,
3323
+ 1,
3324
+ 1,
3325
+ 0,
3326
+ 1,
3327
+ 1,
3328
+ 0,
3329
+ 0,
3330
+ 1,
3331
+ 0,
3332
+ 0,
3333
+ 1,
3334
+ 0,
3335
+ 0,
3336
+ 1,
3337
+ 0,
3338
+ 1,
3339
+ 0,
3340
+ 1,
3341
+ 0,
3342
+ 0,
3343
+ 1,
3344
+ 1,
3345
+ 0,
3346
+ 1,
3347
+ 1,
3348
+ 1,
3349
+ 1,
3350
+ 1,
3351
+ 1,
3352
+ 0,
3353
+ 1,
3354
+ 0,
3355
+ 0,
3356
+ 0,
3357
+ 0,
3358
+ 1,
3359
+ 0,
3360
+ 1,
3361
+ 1,
3362
+ 0,
3363
+ 1,
3364
+ 1,
3365
+ 0,
3366
+ 1,
3367
+ 0,
3368
+ 0,
3369
+ 1,
3370
+ 1,
3371
+ 1,
3372
+ 1,
3373
+ 1,
3374
+ 0,
3375
+ 0,
3376
+ 1,
3377
+ 1,
3378
+ 0,
3379
+ 0,
3380
+ 1,
3381
+ 0,
3382
+ 0,
3383
+ 1,
3384
+ 1,
3385
+ 0,
3386
+ 0,
3387
+ 1,
3388
+ 1,
3389
+ 1,
3390
+ 0,
3391
+ 0,
3392
+ 1,
3393
+ 1,
3394
+ 0,
3395
+ 0,
3396
+ 0,
3397
+ 1,
3398
+ 1,
3399
+ 0,
3400
+ 0,
3401
+ 1,
3402
+ 1,
3403
+ 1,
3404
+ 0,
3405
+ 0,
3406
+ 0,
3407
+ 1,
3408
+ 1,
3409
+ 1,
3410
+ 1,
3411
+ 1,
3412
+ 1,
3413
+ 1,
3414
+ 0,
3415
+ 1,
3416
+ 1,
3417
+ 0,
3418
+ 1,
3419
+ 0,
3420
+ 1,
3421
+ 1,
3422
+ 0,
3423
+ 1,
3424
+ 1,
3425
+ 1,
3426
+ 0,
3427
+ 1,
3428
+ 1,
3429
+ 0,
3430
+ 0,
3431
+ 0,
3432
+ 0,
3433
+ 0,
3434
+ 0,
3435
+ 0,
3436
+ 1,
3437
+ 1,
3438
+ 1,
3439
+ 0,
3440
+ 1,
3441
+ 0,
3442
+ 1,
3443
+ 0,
3444
+ 1,
3445
+ 0,
3446
+ 1,
3447
+ 1,
3448
+ 0,
3449
+ 0,
3450
+ 0,
3451
+ 1,
3452
+ 0,
3453
+ 1,
3454
+ 1,
3455
+ 1,
3456
+ 1,
3457
+ 1,
3458
+ 1,
3459
+ 1,
3460
+ 0,
3461
+ 1,
3462
+ 1,
3463
+ 0,
3464
+ 1,
3465
+ 0,
3466
+ 1,
3467
+ 0,
3468
+ 0,
3469
+ 0,
3470
+ 0,
3471
+ 1,
3472
+ 0,
3473
+ 1,
3474
+ 1,
3475
+ 0,
3476
+ 1,
3477
+ 0,
3478
+ 1,
3479
+ 1,
3480
+ 1,
3481
+ 0,
3482
+ 1,
3483
+ 0,
3484
+ 0,
3485
+ 0,
3486
+ 0,
3487
+ 1,
3488
+ 0,
3489
+ 0,
3490
+ 0,
3491
+ 1,
3492
+ 1,
3493
+ 0,
3494
+ 1,
3495
+ 1,
3496
+ 0,
3497
+ 0,
3498
+ 0,
3499
+ 0,
3500
+ 1,
3501
+ 1,
3502
+ 1,
3503
+ 1,
3504
+ 1,
3505
+ 1,
3506
+ 1,
3507
+ 1,
3508
+ 1
3509
+ ],
3510
+ "ood_dense": [
3511
+ 1,
3512
+ 1,
3513
+ 1,
3514
+ 1,
3515
+ 1,
3516
+ 1,
3517
+ 0,
3518
+ 1,
3519
+ 1,
3520
+ 1,
3521
+ 1,
3522
+ 1,
3523
+ 0,
3524
+ 1,
3525
+ 0,
3526
+ 1,
3527
+ 1,
3528
+ 0,
3529
+ 1,
3530
+ 0,
3531
+ 1,
3532
+ 0,
3533
+ 1,
3534
+ 0,
3535
+ 0,
3536
+ 1,
3537
+ 0,
3538
+ 0,
3539
+ 1,
3540
+ 1,
3541
+ 1,
3542
+ 1,
3543
+ 1,
3544
+ 1,
3545
+ 0,
3546
+ 0,
3547
+ 0,
3548
+ 1,
3549
+ 0,
3550
+ 1,
3551
+ 0,
3552
+ 1,
3553
+ 0,
3554
+ 1,
3555
+ 0,
3556
+ 1,
3557
+ 0,
3558
+ 1,
3559
+ 1,
3560
+ 1,
3561
+ 0,
3562
+ 1,
3563
+ 0,
3564
+ 0,
3565
+ 1,
3566
+ 1,
3567
+ 0,
3568
+ 1,
3569
+ 1,
3570
+ 1,
3571
+ 1,
3572
+ 1,
3573
+ 1,
3574
+ 1,
3575
+ 0,
3576
+ 0,
3577
+ 0,
3578
+ 0,
3579
+ 0,
3580
+ 1,
3581
+ 1,
3582
+ 1,
3583
+ 0,
3584
+ 1,
3585
+ 1,
3586
+ 0,
3587
+ 0,
3588
+ 1,
3589
+ 0,
3590
+ 0,
3591
+ 0,
3592
+ 1,
3593
+ 1,
3594
+ 1,
3595
+ 0,
3596
+ 1,
3597
+ 1,
3598
+ 1,
3599
+ 0,
3600
+ 0,
3601
+ 1,
3602
+ 1,
3603
+ 1,
3604
+ 0,
3605
+ 1,
3606
+ 1,
3607
+ 0,
3608
+ 1,
3609
+ 1,
3610
+ 1,
3611
+ 1,
3612
+ 1,
3613
+ 1,
3614
+ 0,
3615
+ 1,
3616
+ 1,
3617
+ 1,
3618
+ 0,
3619
+ 0,
3620
+ 1,
3621
+ 0,
3622
+ 1,
3623
+ 1,
3624
+ 1,
3625
+ 1,
3626
+ 1,
3627
+ 0,
3628
+ 0,
3629
+ 1,
3630
+ 1,
3631
+ 0,
3632
+ 0,
3633
+ 0,
3634
+ 1,
3635
+ 1,
3636
+ 0,
3637
+ 1,
3638
+ 1,
3639
+ 1,
3640
+ 1,
3641
+ 1,
3642
+ 1,
3643
+ 1,
3644
+ 1,
3645
+ 1,
3646
+ 1,
3647
+ 0,
3648
+ 1,
3649
+ 0,
3650
+ 1,
3651
+ 1,
3652
+ 1,
3653
+ 1,
3654
+ 0,
3655
+ 1,
3656
+ 1,
3657
+ 1,
3658
+ 0,
3659
+ 1,
3660
+ 0,
3661
+ 1,
3662
+ 0,
3663
+ 1,
3664
+ 0,
3665
+ 0,
3666
+ 1,
3667
+ 1,
3668
+ 0,
3669
+ 1,
3670
+ 1,
3671
+ 1,
3672
+ 1,
3673
+ 1,
3674
+ 0,
3675
+ 1,
3676
+ 1,
3677
+ 0,
3678
+ 0,
3679
+ 1,
3680
+ 1,
3681
+ 0,
3682
+ 0,
3683
+ 0,
3684
+ 0,
3685
+ 0,
3686
+ 1,
3687
+ 0,
3688
+ 0,
3689
+ 1,
3690
+ 0,
3691
+ 1,
3692
+ 1,
3693
+ 0,
3694
+ 1,
3695
+ 1,
3696
+ 1,
3697
+ 1,
3698
+ 0,
3699
+ 0,
3700
+ 1,
3701
+ 1,
3702
+ 1,
3703
+ 1,
3704
+ 1,
3705
+ 1,
3706
+ 1,
3707
+ 1,
3708
+ 1,
3709
+ 1,
3710
+ 1,
3711
+ 1,
3712
+ 1,
3713
+ 1,
3714
+ 1,
3715
+ 1,
3716
+ 1,
3717
+ 0,
3718
+ 0,
3719
+ 1,
3720
+ 1,
3721
+ 0,
3722
+ 0,
3723
+ 0,
3724
+ 0,
3725
+ 1,
3726
+ 0,
3727
+ 1,
3728
+ 0,
3729
+ 1,
3730
+ 1,
3731
+ 1,
3732
+ 0,
3733
+ 1,
3734
+ 1,
3735
+ 0,
3736
+ 0,
3737
+ 1,
3738
+ 0,
3739
+ 1,
3740
+ 0,
3741
+ 0,
3742
+ 1,
3743
+ 1,
3744
+ 1,
3745
+ 1,
3746
+ 0,
3747
+ 1,
3748
+ 1,
3749
+ 0,
3750
+ 1,
3751
+ 1,
3752
+ 1,
3753
+ 1,
3754
+ 0,
3755
+ 0,
3756
+ 1,
3757
+ 1,
3758
+ 0,
3759
+ 1,
3760
+ 1,
3761
+ 0,
3762
+ 0,
3763
+ 1,
3764
+ 0,
3765
+ 1,
3766
+ 1,
3767
+ 1,
3768
+ 1,
3769
+ 0,
3770
+ 1,
3771
+ 1,
3772
+ 1,
3773
+ 0,
3774
+ 0,
3775
+ 0,
3776
+ 1,
3777
+ 1,
3778
+ 1,
3779
+ 1,
3780
+ 1,
3781
+ 1,
3782
+ 1,
3783
+ 1,
3784
+ 1,
3785
+ 1,
3786
+ 0,
3787
+ 1,
3788
+ 1,
3789
+ 1,
3790
+ 1,
3791
+ 1,
3792
+ 0,
3793
+ 1,
3794
+ 1,
3795
+ 1,
3796
+ 1,
3797
+ 0,
3798
+ 1,
3799
+ 0,
3800
+ 0,
3801
+ 1,
3802
+ 1,
3803
+ 1,
3804
+ 0,
3805
+ 1,
3806
+ 1,
3807
+ 1,
3808
+ 1,
3809
+ 1,
3810
+ 0,
3811
+ 1,
3812
+ 1,
3813
+ 0,
3814
+ 1,
3815
+ 1,
3816
+ 1,
3817
+ 0,
3818
+ 0,
3819
+ 1,
3820
+ 1,
3821
+ 0,
3822
+ 0,
3823
+ 1,
3824
+ 1,
3825
+ 0,
3826
+ 0,
3827
+ 1,
3828
+ 1,
3829
+ 1,
3830
+ 1,
3831
+ 1,
3832
+ 1,
3833
+ 1,
3834
+ 0,
3835
+ 1,
3836
+ 1,
3837
+ 0,
3838
+ 1,
3839
+ 1,
3840
+ 0,
3841
+ 0,
3842
+ 1,
3843
+ 0,
3844
+ 1,
3845
+ 1,
3846
+ 1,
3847
+ 1,
3848
+ 1,
3849
+ 1,
3850
+ 0,
3851
+ 0,
3852
+ 1,
3853
+ 1,
3854
+ 0,
3855
+ 1,
3856
+ 1,
3857
+ 0,
3858
+ 0,
3859
+ 1,
3860
+ 1,
3861
+ 1,
3862
+ 0,
3863
+ 1,
3864
+ 1,
3865
+ 0,
3866
+ 1,
3867
+ 1,
3868
+ 1,
3869
+ 0,
3870
+ 1,
3871
+ 1,
3872
+ 1,
3873
+ 1,
3874
+ 0,
3875
+ 1,
3876
+ 0,
3877
+ 1,
3878
+ 0,
3879
+ 1,
3880
+ 1,
3881
+ 1,
3882
+ 0,
3883
+ 1,
3884
+ 1,
3885
+ 1,
3886
+ 1,
3887
+ 1,
3888
+ 0,
3889
+ 1,
3890
+ 1,
3891
+ 1,
3892
+ 1,
3893
+ 1,
3894
+ 1,
3895
+ 1,
3896
+ 1,
3897
+ 1,
3898
+ 0,
3899
+ 0,
3900
+ 1,
3901
+ 0,
3902
+ 1,
3903
+ 0,
3904
+ 1,
3905
+ 1,
3906
+ 1,
3907
+ 1,
3908
+ 1,
3909
+ 1,
3910
+ 1,
3911
+ 1,
3912
+ 1,
3913
+ 0,
3914
+ 0,
3915
+ 1,
3916
+ 1,
3917
+ 1,
3918
+ 0,
3919
+ 0,
3920
+ 1,
3921
+ 1,
3922
+ 1,
3923
+ 1,
3924
+ 1,
3925
+ 1,
3926
+ 1,
3927
+ 1,
3928
+ 1,
3929
+ 0,
3930
+ 1,
3931
+ 1,
3932
+ 0,
3933
+ 0,
3934
+ 1,
3935
+ 0,
3936
+ 0,
3937
+ 1,
3938
+ 1,
3939
+ 0,
3940
+ 1,
3941
+ 0,
3942
+ 0,
3943
+ 0,
3944
+ 1,
3945
+ 1,
3946
+ 1,
3947
+ 0,
3948
+ 0,
3949
+ 0,
3950
+ 0,
3951
+ 0,
3952
+ 1,
3953
+ 1,
3954
+ 0,
3955
+ 1,
3956
+ 1,
3957
+ 0,
3958
+ 0,
3959
+ 0,
3960
+ 0,
3961
+ 1,
3962
+ 0,
3963
+ 0,
3964
+ 0,
3965
+ 1,
3966
+ 1,
3967
+ 1,
3968
+ 0,
3969
+ 0,
3970
+ 1,
3971
+ 1,
3972
+ 0,
3973
+ 1,
3974
+ 1,
3975
+ 0,
3976
+ 0,
3977
+ 1,
3978
+ 0,
3979
+ 1,
3980
+ 1,
3981
+ 0,
3982
+ 1,
3983
+ 1,
3984
+ 1,
3985
+ 0,
3986
+ 0,
3987
+ 1,
3988
+ 0,
3989
+ 1,
3990
+ 1,
3991
+ 1,
3992
+ 1,
3993
+ 1,
3994
+ 0,
3995
+ 0,
3996
+ 0,
3997
+ 1,
3998
+ 1,
3999
+ 1,
4000
+ 0,
4001
+ 1,
4002
+ 0,
4003
+ 1,
4004
+ 1,
4005
+ 0,
4006
+ 1,
4007
+ 1,
4008
+ 1,
4009
+ 0,
4010
+ 1,
4011
+ 1,
4012
+ 1,
4013
+ 0,
4014
+ 1,
4015
+ 1,
4016
+ 0,
4017
+ 0,
4018
+ 1,
4019
+ 0,
4020
+ 1,
4021
+ 1,
4022
+ 1,
4023
+ 1,
4024
+ 1,
4025
+ 1,
4026
+ 0,
4027
+ 1,
4028
+ 1,
4029
+ 0,
4030
+ 1,
4031
+ 1,
4032
+ 1,
4033
+ 1,
4034
+ 1,
4035
+ 0,
4036
+ 1,
4037
+ 1,
4038
+ 1,
4039
+ 1,
4040
+ 1,
4041
+ 1,
4042
+ 1,
4043
+ 1,
4044
+ 1,
4045
+ 1,
4046
+ 1,
4047
+ 1,
4048
+ 1,
4049
+ 1,
4050
+ 1,
4051
+ 1,
4052
+ 1,
4053
+ 1,
4054
+ 1,
4055
+ 0,
4056
+ 1,
4057
+ 1,
4058
+ 1,
4059
+ 1,
4060
+ 1,
4061
+ 0,
4062
+ 1,
4063
+ 1,
4064
+ 1,
4065
+ 0,
4066
+ 1,
4067
+ 1,
4068
+ 1,
4069
+ 1,
4070
+ 0,
4071
+ 0,
4072
+ 1,
4073
+ 1,
4074
+ 1,
4075
+ 1,
4076
+ 1,
4077
+ 1,
4078
+ 1,
4079
+ 1,
4080
+ 1,
4081
+ 1,
4082
+ 0,
4083
+ 1,
4084
+ 1,
4085
+ 0,
4086
+ 1,
4087
+ 1,
4088
+ 1,
4089
+ 1,
4090
+ 1,
4091
+ 0,
4092
+ 0,
4093
+ 1,
4094
+ 1,
4095
+ 0,
4096
+ 0,
4097
+ 1,
4098
+ 0,
4099
+ 1,
4100
+ 1,
4101
+ 1,
4102
+ 1,
4103
+ 1,
4104
+ 0,
4105
+ 1,
4106
+ 1,
4107
+ 0,
4108
+ 0,
4109
+ 1,
4110
+ 1,
4111
+ 1,
4112
+ 1,
4113
+ 1,
4114
+ 0,
4115
+ 0,
4116
+ 1,
4117
+ 1,
4118
+ 1,
4119
+ 1,
4120
+ 1,
4121
+ 1,
4122
+ 0,
4123
+ 1,
4124
+ 1,
4125
+ 0,
4126
+ 1,
4127
+ 0,
4128
+ 0,
4129
+ 1,
4130
+ 1,
4131
+ 1,
4132
+ 0,
4133
+ 0,
4134
+ 1,
4135
+ 0,
4136
+ 1,
4137
+ 1,
4138
+ 1,
4139
+ 1,
4140
+ 0,
4141
+ 1,
4142
+ 1,
4143
+ 1,
4144
+ 1,
4145
+ 1,
4146
+ 1,
4147
+ 1,
4148
+ 1,
4149
+ 1,
4150
+ 1,
4151
+ 0,
4152
+ 0,
4153
+ 1,
4154
+ 0,
4155
+ 1,
4156
+ 1,
4157
+ 0,
4158
+ 1,
4159
+ 0,
4160
+ 1,
4161
+ 0,
4162
+ 1,
4163
+ 1,
4164
+ 1,
4165
+ 0,
4166
+ 1,
4167
+ 1,
4168
+ 0,
4169
+ 1,
4170
+ 0,
4171
+ 1,
4172
+ 0,
4173
+ 1,
4174
+ 1,
4175
+ 1,
4176
+ 0,
4177
+ 1,
4178
+ 0,
4179
+ 1,
4180
+ 1,
4181
+ 0,
4182
+ 0,
4183
+ 1,
4184
+ 1,
4185
+ 1,
4186
+ 1,
4187
+ 1,
4188
+ 0,
4189
+ 1,
4190
+ 0,
4191
+ 0,
4192
+ 0,
4193
+ 1,
4194
+ 1,
4195
+ 1,
4196
+ 1,
4197
+ 1,
4198
+ 1,
4199
+ 0,
4200
+ 0,
4201
+ 1,
4202
+ 0,
4203
+ 1,
4204
+ 1,
4205
+ 1,
4206
+ 1,
4207
+ 1,
4208
+ 1,
4209
+ 1,
4210
+ 1,
4211
+ 1,
4212
+ 1,
4213
+ 1,
4214
+ 1,
4215
+ 0,
4216
+ 0,
4217
+ 1,
4218
+ 1,
4219
+ 0,
4220
+ 0,
4221
+ 0,
4222
+ 0,
4223
+ 0,
4224
+ 1,
4225
+ 0,
4226
+ 1,
4227
+ 0,
4228
+ 1,
4229
+ 1,
4230
+ 1,
4231
+ 1,
4232
+ 1,
4233
+ 1,
4234
+ 1,
4235
+ 1,
4236
+ 0,
4237
+ 1,
4238
+ 0,
4239
+ 1,
4240
+ 1,
4241
+ 0,
4242
+ 0,
4243
+ 1,
4244
+ 1,
4245
+ 1,
4246
+ 1,
4247
+ 0,
4248
+ 1,
4249
+ 1,
4250
+ 1,
4251
+ 0,
4252
+ 1,
4253
+ 0,
4254
+ 1,
4255
+ 1,
4256
+ 0,
4257
+ 1,
4258
+ 1,
4259
+ 1,
4260
+ 0,
4261
+ 0,
4262
+ 0,
4263
+ 1,
4264
+ 1,
4265
+ 1,
4266
+ 1,
4267
+ 1,
4268
+ 1,
4269
+ 1,
4270
+ 1,
4271
+ 0,
4272
+ 1,
4273
+ 1,
4274
+ 1,
4275
+ 1,
4276
+ 1,
4277
+ 1,
4278
+ 0,
4279
+ 1,
4280
+ 0,
4281
+ 1,
4282
+ 0,
4283
+ 1,
4284
+ 1,
4285
+ 0,
4286
+ 0,
4287
+ 0,
4288
+ 0,
4289
+ 1,
4290
+ 1,
4291
+ 0,
4292
+ 1,
4293
+ 1,
4294
+ 1,
4295
+ 1,
4296
+ 0,
4297
+ 0,
4298
+ 1,
4299
+ 1,
4300
+ 1,
4301
+ 0,
4302
+ 1,
4303
+ 1,
4304
+ 1,
4305
+ 0,
4306
+ 1,
4307
+ 0,
4308
+ 1,
4309
+ 0,
4310
+ 1,
4311
+ 1,
4312
+ 1,
4313
+ 1,
4314
+ 1,
4315
+ 1,
4316
+ 1,
4317
+ 1,
4318
+ 1,
4319
+ 0,
4320
+ 1,
4321
+ 1,
4322
+ 1,
4323
+ 1,
4324
+ 0,
4325
+ 1,
4326
+ 1,
4327
+ 1,
4328
+ 1,
4329
+ 1,
4330
+ 1,
4331
+ 1,
4332
+ 1,
4333
+ 1,
4334
+ 0,
4335
+ 0,
4336
+ 1,
4337
+ 0,
4338
+ 1,
4339
+ 0,
4340
+ 0,
4341
+ 1,
4342
+ 1,
4343
+ 0,
4344
+ 1,
4345
+ 1,
4346
+ 1,
4347
+ 1,
4348
+ 1,
4349
+ 0,
4350
+ 1,
4351
+ 1,
4352
+ 0,
4353
+ 1,
4354
+ 1,
4355
+ 1,
4356
+ 1,
4357
+ 0,
4358
+ 1,
4359
+ 0,
4360
+ 1,
4361
+ 1,
4362
+ 0,
4363
+ 1,
4364
+ 1,
4365
+ 0,
4366
+ 1,
4367
+ 1,
4368
+ 1,
4369
+ 1,
4370
+ 1,
4371
+ 0,
4372
+ 1,
4373
+ 1,
4374
+ 1,
4375
+ 0,
4376
+ 0,
4377
+ 1,
4378
+ 1,
4379
+ 1,
4380
+ 1,
4381
+ 1,
4382
+ 0,
4383
+ 1,
4384
+ 1,
4385
+ 1,
4386
+ 1,
4387
+ 0,
4388
+ 1,
4389
+ 1,
4390
+ 1,
4391
+ 1,
4392
+ 0,
4393
+ 0,
4394
+ 0,
4395
+ 1,
4396
+ 1,
4397
+ 1,
4398
+ 1,
4399
+ 1,
4400
+ 1,
4401
+ 1,
4402
+ 0,
4403
+ 1,
4404
+ 1,
4405
+ 1,
4406
+ 0,
4407
+ 1,
4408
+ 1,
4409
+ 0,
4410
+ 1,
4411
+ 1,
4412
+ 0,
4413
+ 1,
4414
+ 0,
4415
+ 0,
4416
+ 1,
4417
+ 1,
4418
+ 1,
4419
+ 0,
4420
+ 0,
4421
+ 1,
4422
+ 1,
4423
+ 1,
4424
+ 1,
4425
+ 1,
4426
+ 0,
4427
+ 1,
4428
+ 1,
4429
+ 0,
4430
+ 1,
4431
+ 1,
4432
+ 0,
4433
+ 1,
4434
+ 0,
4435
+ 1,
4436
+ 1,
4437
+ 0,
4438
+ 0,
4439
+ 1,
4440
+ 1,
4441
+ 1,
4442
+ 1,
4443
+ 1,
4444
+ 1,
4445
+ 0,
4446
+ 0,
4447
+ 1,
4448
+ 1,
4449
+ 1,
4450
+ 0,
4451
+ 1,
4452
+ 1,
4453
+ 1,
4454
+ 1,
4455
+ 1,
4456
+ 1,
4457
+ 1,
4458
+ 0,
4459
+ 1,
4460
+ 0,
4461
+ 0,
4462
+ 0,
4463
+ 0,
4464
+ 0,
4465
+ 1,
4466
+ 1,
4467
+ 0,
4468
+ 1,
4469
+ 0,
4470
+ 0,
4471
+ 1,
4472
+ 0,
4473
+ 1,
4474
+ 1,
4475
+ 0,
4476
+ 1,
4477
+ 1,
4478
+ 1,
4479
+ 0,
4480
+ 1,
4481
+ 0,
4482
+ 0,
4483
+ 0,
4484
+ 1,
4485
+ 0,
4486
+ 0,
4487
+ 0,
4488
+ 0,
4489
+ 0,
4490
+ 0,
4491
+ 0,
4492
+ 1,
4493
+ 0,
4494
+ 0,
4495
+ 1,
4496
+ 1,
4497
+ 1,
4498
+ 0,
4499
+ 0,
4500
+ 1,
4501
+ 1,
4502
+ 0,
4503
+ 0,
4504
+ 0,
4505
+ 1,
4506
+ 1,
4507
+ 1,
4508
+ 1,
4509
+ 0,
4510
+ 1
4511
+ ]
4512
+ }
experiments/m6_epoch39/summary.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "m6_epoch39",
3
+ "checkpoint": "/lambda/nfs/experiment/results/v9_meta_fork/prosqa-m6-pause-multipass/checkpoint_39",
4
+ "feedback_mode": "pause_multipass",
5
+ "stages_run": [
6
+ "accuracy"
7
+ ],
8
+ "results": {
9
+ "accuracy": {
10
+ "prosqa_test": {
11
+ "accuracy": 0.95,
12
+ "correct": 475,
13
+ "total": 500
14
+ },
15
+ "ood_7hop": {
16
+ "accuracy": 0.73,
17
+ "correct": 730,
18
+ "total": 1000
19
+ },
20
+ "ood_8hop": {
21
+ "accuracy": 0.732,
22
+ "correct": 732,
23
+ "total": 1000
24
+ },
25
+ "ood_dag": {
26
+ "accuracy": 0.608,
27
+ "correct": 608,
28
+ "total": 1000
29
+ },
30
+ "ood_dense": {
31
+ "accuracy": 0.667,
32
+ "correct": 667,
33
+ "total": 1000
34
+ }
35
+ }
36
+ }
37
+ }
experiments/mlp_probe_grid_search.json ADDED
@@ -0,0 +1,3378 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "target_cells": [
3
+ {
4
+ "model": "m3",
5
+ "layer": 0,
6
+ "position": 3
7
+ },
8
+ {
9
+ "model": "m3",
10
+ "layer": 12,
11
+ "position": 2
12
+ },
13
+ {
14
+ "model": "m5",
15
+ "layer": 12,
16
+ "position": 3
17
+ },
18
+ {
19
+ "model": "m5",
20
+ "layer": 8,
21
+ "position": 0
22
+ },
23
+ {
24
+ "model": "m5",
25
+ "layer": 12,
26
+ "position": 2
27
+ }
28
+ ],
29
+ "grid": {
30
+ "hidden_sizes": [
31
+ [
32
+ 64
33
+ ],
34
+ [
35
+ 128
36
+ ],
37
+ [
38
+ 256
39
+ ],
40
+ [
41
+ 512
42
+ ],
43
+ [
44
+ 128,
45
+ 64
46
+ ],
47
+ [
48
+ 256,
49
+ 128
50
+ ]
51
+ ],
52
+ "learning_rates": [
53
+ 0.0001,
54
+ 0.001,
55
+ 0.01
56
+ ],
57
+ "alphas": [
58
+ 0.0001,
59
+ 0.001,
60
+ 0.01,
61
+ 0.1
62
+ ]
63
+ },
64
+ "n_folds": 5,
65
+ "results": [
66
+ {
67
+ "model": "m3",
68
+ "layer": 0,
69
+ "position": 3,
70
+ "n_samples": 298,
71
+ "n_classes": 38,
72
+ "linear_accuracy": 0.5536912751677852,
73
+ "best_mlp_accuracy": 0.4597315436241611,
74
+ "mlp_advantage_pp": -9.4,
75
+ "best_config": {
76
+ "hidden_sizes": [
77
+ 512
78
+ ],
79
+ "learning_rate": 0.001,
80
+ "alpha": 0.0001,
81
+ "accuracy": 0.4597315436241611
82
+ },
83
+ "top_5_configs": [
84
+ {
85
+ "hidden_sizes": [
86
+ 512
87
+ ],
88
+ "learning_rate": 0.001,
89
+ "alpha": 0.0001,
90
+ "accuracy": 0.4597315436241611
91
+ },
92
+ {
93
+ "hidden_sizes": [
94
+ 512
95
+ ],
96
+ "learning_rate": 0.001,
97
+ "alpha": 0.001,
98
+ "accuracy": 0.4597315436241611
99
+ },
100
+ {
101
+ "hidden_sizes": [
102
+ 512
103
+ ],
104
+ "learning_rate": 0.001,
105
+ "alpha": 0.01,
106
+ "accuracy": 0.4563758389261745
107
+ },
108
+ {
109
+ "hidden_sizes": [
110
+ 512
111
+ ],
112
+ "learning_rate": 0.001,
113
+ "alpha": 0.1,
114
+ "accuracy": 0.4563758389261745
115
+ },
116
+ {
117
+ "hidden_sizes": [
118
+ 256
119
+ ],
120
+ "learning_rate": 0.01,
121
+ "alpha": 0.0001,
122
+ "accuracy": 0.40268456375838924
123
+ }
124
+ ],
125
+ "all_configs": [
126
+ {
127
+ "hidden_sizes": [
128
+ 512
129
+ ],
130
+ "learning_rate": 0.001,
131
+ "alpha": 0.0001,
132
+ "accuracy": 0.4597315436241611
133
+ },
134
+ {
135
+ "hidden_sizes": [
136
+ 512
137
+ ],
138
+ "learning_rate": 0.001,
139
+ "alpha": 0.001,
140
+ "accuracy": 0.4597315436241611
141
+ },
142
+ {
143
+ "hidden_sizes": [
144
+ 512
145
+ ],
146
+ "learning_rate": 0.001,
147
+ "alpha": 0.01,
148
+ "accuracy": 0.4563758389261745
149
+ },
150
+ {
151
+ "hidden_sizes": [
152
+ 512
153
+ ],
154
+ "learning_rate": 0.001,
155
+ "alpha": 0.1,
156
+ "accuracy": 0.4563758389261745
157
+ },
158
+ {
159
+ "hidden_sizes": [
160
+ 256
161
+ ],
162
+ "learning_rate": 0.01,
163
+ "alpha": 0.0001,
164
+ "accuracy": 0.40268456375838924
165
+ },
166
+ {
167
+ "hidden_sizes": [
168
+ 256
169
+ ],
170
+ "learning_rate": 0.01,
171
+ "alpha": 0.001,
172
+ "accuracy": 0.40268456375838924
173
+ },
174
+ {
175
+ "hidden_sizes": [
176
+ 256,
177
+ 128
178
+ ],
179
+ "learning_rate": 0.001,
180
+ "alpha": 0.1,
181
+ "accuracy": 0.38926174496644295
182
+ },
183
+ {
184
+ "hidden_sizes": [
185
+ 128,
186
+ 64
187
+ ],
188
+ "learning_rate": 0.01,
189
+ "alpha": 0.001,
190
+ "accuracy": 0.3825503355704698
191
+ },
192
+ {
193
+ "hidden_sizes": [
194
+ 256,
195
+ 128
196
+ ],
197
+ "learning_rate": 0.001,
198
+ "alpha": 0.01,
199
+ "accuracy": 0.3825503355704698
200
+ },
201
+ {
202
+ "hidden_sizes": [
203
+ 512
204
+ ],
205
+ "learning_rate": 0.01,
206
+ "alpha": 0.1,
207
+ "accuracy": 0.37919463087248323
208
+ },
209
+ {
210
+ "hidden_sizes": [
211
+ 256,
212
+ 128
213
+ ],
214
+ "learning_rate": 0.001,
215
+ "alpha": 0.0001,
216
+ "accuracy": 0.37919463087248323
217
+ },
218
+ {
219
+ "hidden_sizes": [
220
+ 256,
221
+ 128
222
+ ],
223
+ "learning_rate": 0.001,
224
+ "alpha": 0.001,
225
+ "accuracy": 0.37919463087248323
226
+ },
227
+ {
228
+ "hidden_sizes": [
229
+ 256,
230
+ 128
231
+ ],
232
+ "learning_rate": 0.01,
233
+ "alpha": 0.0001,
234
+ "accuracy": 0.37919463087248323
235
+ },
236
+ {
237
+ "hidden_sizes": [
238
+ 256
239
+ ],
240
+ "learning_rate": 0.01,
241
+ "alpha": 0.01,
242
+ "accuracy": 0.3791946308724832
243
+ },
244
+ {
245
+ "hidden_sizes": [
246
+ 256
247
+ ],
248
+ "learning_rate": 0.01,
249
+ "alpha": 0.1,
250
+ "accuracy": 0.3791946308724832
251
+ },
252
+ {
253
+ "hidden_sizes": [
254
+ 128,
255
+ 64
256
+ ],
257
+ "learning_rate": 0.01,
258
+ "alpha": 0.0001,
259
+ "accuracy": 0.37248322147651003
260
+ },
261
+ {
262
+ "hidden_sizes": [
263
+ 128
264
+ ],
265
+ "learning_rate": 0.01,
266
+ "alpha": 0.1,
267
+ "accuracy": 0.3657718120805369
268
+ },
269
+ {
270
+ "hidden_sizes": [
271
+ 128
272
+ ],
273
+ "learning_rate": 0.01,
274
+ "alpha": 0.001,
275
+ "accuracy": 0.36241610738255037
276
+ },
277
+ {
278
+ "hidden_sizes": [
279
+ 128,
280
+ 64
281
+ ],
282
+ "learning_rate": 0.01,
283
+ "alpha": 0.1,
284
+ "accuracy": 0.36241610738255037
285
+ },
286
+ {
287
+ "hidden_sizes": [
288
+ 128,
289
+ 64
290
+ ],
291
+ "learning_rate": 0.01,
292
+ "alpha": 0.01,
293
+ "accuracy": 0.3624161073825503
294
+ },
295
+ {
296
+ "hidden_sizes": [
297
+ 64
298
+ ],
299
+ "learning_rate": 0.001,
300
+ "alpha": 0.0001,
301
+ "accuracy": 0.35906040268456374
302
+ },
303
+ {
304
+ "hidden_sizes": [
305
+ 64
306
+ ],
307
+ "learning_rate": 0.001,
308
+ "alpha": 0.001,
309
+ "accuracy": 0.35906040268456374
310
+ },
311
+ {
312
+ "hidden_sizes": [
313
+ 128
314
+ ],
315
+ "learning_rate": 0.01,
316
+ "alpha": 0.0001,
317
+ "accuracy": 0.35906040268456374
318
+ },
319
+ {
320
+ "hidden_sizes": [
321
+ 128
322
+ ],
323
+ "learning_rate": 0.01,
324
+ "alpha": 0.01,
325
+ "accuracy": 0.35906040268456374
326
+ },
327
+ {
328
+ "hidden_sizes": [
329
+ 128,
330
+ 64
331
+ ],
332
+ "learning_rate": 0.001,
333
+ "alpha": 0.0001,
334
+ "accuracy": 0.35906040268456374
335
+ },
336
+ {
337
+ "hidden_sizes": [
338
+ 64
339
+ ],
340
+ "learning_rate": 0.001,
341
+ "alpha": 0.01,
342
+ "accuracy": 0.3557046979865772
343
+ },
344
+ {
345
+ "hidden_sizes": [
346
+ 64
347
+ ],
348
+ "learning_rate": 0.01,
349
+ "alpha": 0.0001,
350
+ "accuracy": 0.3557046979865772
351
+ },
352
+ {
353
+ "hidden_sizes": [
354
+ 64
355
+ ],
356
+ "learning_rate": 0.01,
357
+ "alpha": 0.01,
358
+ "accuracy": 0.35570469798657717
359
+ },
360
+ {
361
+ "hidden_sizes": [
362
+ 64
363
+ ],
364
+ "learning_rate": 0.01,
365
+ "alpha": 0.001,
366
+ "accuracy": 0.3523489932885906
367
+ },
368
+ {
369
+ "hidden_sizes": [
370
+ 128,
371
+ 64
372
+ ],
373
+ "learning_rate": 0.001,
374
+ "alpha": 0.01,
375
+ "accuracy": 0.3523489932885906
376
+ },
377
+ {
378
+ "hidden_sizes": [
379
+ 256,
380
+ 128
381
+ ],
382
+ "learning_rate": 0.01,
383
+ "alpha": 0.1,
384
+ "accuracy": 0.3523489932885906
385
+ },
386
+ {
387
+ "hidden_sizes": [
388
+ 128,
389
+ 64
390
+ ],
391
+ "learning_rate": 0.001,
392
+ "alpha": 0.1,
393
+ "accuracy": 0.348993288590604
394
+ },
395
+ {
396
+ "hidden_sizes": [
397
+ 128,
398
+ 64
399
+ ],
400
+ "learning_rate": 0.001,
401
+ "alpha": 0.001,
402
+ "accuracy": 0.34563758389261745
403
+ },
404
+ {
405
+ "hidden_sizes": [
406
+ 256,
407
+ 128
408
+ ],
409
+ "learning_rate": 0.01,
410
+ "alpha": 0.001,
411
+ "accuracy": 0.3389261744966443
412
+ },
413
+ {
414
+ "hidden_sizes": [
415
+ 512
416
+ ],
417
+ "learning_rate": 0.01,
418
+ "alpha": 0.0001,
419
+ "accuracy": 0.33557046979865773
420
+ },
421
+ {
422
+ "hidden_sizes": [
423
+ 512
424
+ ],
425
+ "learning_rate": 0.01,
426
+ "alpha": 0.001,
427
+ "accuracy": 0.33557046979865773
428
+ },
429
+ {
430
+ "hidden_sizes": [
431
+ 512
432
+ ],
433
+ "learning_rate": 0.01,
434
+ "alpha": 0.01,
435
+ "accuracy": 0.33221476510067116
436
+ },
437
+ {
438
+ "hidden_sizes": [
439
+ 256,
440
+ 128
441
+ ],
442
+ "learning_rate": 0.01,
443
+ "alpha": 0.01,
444
+ "accuracy": 0.32885906040268453
445
+ },
446
+ {
447
+ "hidden_sizes": [
448
+ 64
449
+ ],
450
+ "learning_rate": 0.01,
451
+ "alpha": 0.1,
452
+ "accuracy": 0.325503355704698
453
+ },
454
+ {
455
+ "hidden_sizes": [
456
+ 256
457
+ ],
458
+ "learning_rate": 0.001,
459
+ "alpha": 0.0001,
460
+ "accuracy": 0.325503355704698
461
+ },
462
+ {
463
+ "hidden_sizes": [
464
+ 256
465
+ ],
466
+ "learning_rate": 0.001,
467
+ "alpha": 0.001,
468
+ "accuracy": 0.325503355704698
469
+ },
470
+ {
471
+ "hidden_sizes": [
472
+ 256
473
+ ],
474
+ "learning_rate": 0.001,
475
+ "alpha": 0.01,
476
+ "accuracy": 0.325503355704698
477
+ },
478
+ {
479
+ "hidden_sizes": [
480
+ 256
481
+ ],
482
+ "learning_rate": 0.001,
483
+ "alpha": 0.1,
484
+ "accuracy": 0.325503355704698
485
+ },
486
+ {
487
+ "hidden_sizes": [
488
+ 64
489
+ ],
490
+ "learning_rate": 0.001,
491
+ "alpha": 0.1,
492
+ "accuracy": 0.3221476510067114
493
+ },
494
+ {
495
+ "hidden_sizes": [
496
+ 128
497
+ ],
498
+ "learning_rate": 0.001,
499
+ "alpha": 0.0001,
500
+ "accuracy": 0.31208053691275167
501
+ },
502
+ {
503
+ "hidden_sizes": [
504
+ 128
505
+ ],
506
+ "learning_rate": 0.001,
507
+ "alpha": 0.001,
508
+ "accuracy": 0.31208053691275167
509
+ },
510
+ {
511
+ "hidden_sizes": [
512
+ 128
513
+ ],
514
+ "learning_rate": 0.001,
515
+ "alpha": 0.1,
516
+ "accuracy": 0.31208053691275167
517
+ },
518
+ {
519
+ "hidden_sizes": [
520
+ 128
521
+ ],
522
+ "learning_rate": 0.001,
523
+ "alpha": 0.01,
524
+ "accuracy": 0.3087248322147651
525
+ },
526
+ {
527
+ "hidden_sizes": [
528
+ 256
529
+ ],
530
+ "learning_rate": 0.0001,
531
+ "alpha": 0.0001,
532
+ "accuracy": 0.0738255033557047
533
+ },
534
+ {
535
+ "hidden_sizes": [
536
+ 256
537
+ ],
538
+ "learning_rate": 0.0001,
539
+ "alpha": 0.001,
540
+ "accuracy": 0.0738255033557047
541
+ },
542
+ {
543
+ "hidden_sizes": [
544
+ 256
545
+ ],
546
+ "learning_rate": 0.0001,
547
+ "alpha": 0.01,
548
+ "accuracy": 0.0738255033557047
549
+ },
550
+ {
551
+ "hidden_sizes": [
552
+ 256
553
+ ],
554
+ "learning_rate": 0.0001,
555
+ "alpha": 0.1,
556
+ "accuracy": 0.0738255033557047
557
+ },
558
+ {
559
+ "hidden_sizes": [
560
+ 128
561
+ ],
562
+ "learning_rate": 0.0001,
563
+ "alpha": 0.0001,
564
+ "accuracy": 0.0436241610738255
565
+ },
566
+ {
567
+ "hidden_sizes": [
568
+ 128
569
+ ],
570
+ "learning_rate": 0.0001,
571
+ "alpha": 0.001,
572
+ "accuracy": 0.0436241610738255
573
+ },
574
+ {
575
+ "hidden_sizes": [
576
+ 128
577
+ ],
578
+ "learning_rate": 0.0001,
579
+ "alpha": 0.01,
580
+ "accuracy": 0.0436241610738255
581
+ },
582
+ {
583
+ "hidden_sizes": [
584
+ 128
585
+ ],
586
+ "learning_rate": 0.0001,
587
+ "alpha": 0.1,
588
+ "accuracy": 0.0436241610738255
589
+ },
590
+ {
591
+ "hidden_sizes": [
592
+ 64
593
+ ],
594
+ "learning_rate": 0.0001,
595
+ "alpha": 0.0001,
596
+ "accuracy": 0.02348993288590604
597
+ },
598
+ {
599
+ "hidden_sizes": [
600
+ 64
601
+ ],
602
+ "learning_rate": 0.0001,
603
+ "alpha": 0.001,
604
+ "accuracy": 0.02348993288590604
605
+ },
606
+ {
607
+ "hidden_sizes": [
608
+ 64
609
+ ],
610
+ "learning_rate": 0.0001,
611
+ "alpha": 0.01,
612
+ "accuracy": 0.02348993288590604
613
+ },
614
+ {
615
+ "hidden_sizes": [
616
+ 64
617
+ ],
618
+ "learning_rate": 0.0001,
619
+ "alpha": 0.1,
620
+ "accuracy": 0.02348993288590604
621
+ },
622
+ {
623
+ "hidden_sizes": [
624
+ 512
625
+ ],
626
+ "learning_rate": 0.0001,
627
+ "alpha": 0.0001,
628
+ "accuracy": 0.020134228187919462
629
+ },
630
+ {
631
+ "hidden_sizes": [
632
+ 512
633
+ ],
634
+ "learning_rate": 0.0001,
635
+ "alpha": 0.001,
636
+ "accuracy": 0.020134228187919462
637
+ },
638
+ {
639
+ "hidden_sizes": [
640
+ 512
641
+ ],
642
+ "learning_rate": 0.0001,
643
+ "alpha": 0.01,
644
+ "accuracy": 0.020134228187919462
645
+ },
646
+ {
647
+ "hidden_sizes": [
648
+ 512
649
+ ],
650
+ "learning_rate": 0.0001,
651
+ "alpha": 0.1,
652
+ "accuracy": 0.020134228187919462
653
+ },
654
+ {
655
+ "hidden_sizes": [
656
+ 256,
657
+ 128
658
+ ],
659
+ "learning_rate": 0.0001,
660
+ "alpha": 0.0001,
661
+ "accuracy": 0.020134228187919462
662
+ },
663
+ {
664
+ "hidden_sizes": [
665
+ 256,
666
+ 128
667
+ ],
668
+ "learning_rate": 0.0001,
669
+ "alpha": 0.001,
670
+ "accuracy": 0.020134228187919462
671
+ },
672
+ {
673
+ "hidden_sizes": [
674
+ 256,
675
+ 128
676
+ ],
677
+ "learning_rate": 0.0001,
678
+ "alpha": 0.01,
679
+ "accuracy": 0.020134228187919462
680
+ },
681
+ {
682
+ "hidden_sizes": [
683
+ 256,
684
+ 128
685
+ ],
686
+ "learning_rate": 0.0001,
687
+ "alpha": 0.1,
688
+ "accuracy": 0.020134228187919462
689
+ },
690
+ {
691
+ "hidden_sizes": [
692
+ 128,
693
+ 64
694
+ ],
695
+ "learning_rate": 0.0001,
696
+ "alpha": 0.0001,
697
+ "accuracy": 0.016778523489932886
698
+ },
699
+ {
700
+ "hidden_sizes": [
701
+ 128,
702
+ 64
703
+ ],
704
+ "learning_rate": 0.0001,
705
+ "alpha": 0.001,
706
+ "accuracy": 0.016778523489932886
707
+ },
708
+ {
709
+ "hidden_sizes": [
710
+ 128,
711
+ 64
712
+ ],
713
+ "learning_rate": 0.0001,
714
+ "alpha": 0.01,
715
+ "accuracy": 0.016778523489932886
716
+ },
717
+ {
718
+ "hidden_sizes": [
719
+ 128,
720
+ 64
721
+ ],
722
+ "learning_rate": 0.0001,
723
+ "alpha": 0.1,
724
+ "accuracy": 0.016778523489932886
725
+ }
726
+ ]
727
+ },
728
+ {
729
+ "model": "m3",
730
+ "layer": 12,
731
+ "position": 2,
732
+ "n_samples": 500,
733
+ "n_classes": 38,
734
+ "linear_accuracy": 0.19,
735
+ "best_mlp_accuracy": 0.292,
736
+ "mlp_advantage_pp": 10.2,
737
+ "best_config": {
738
+ "hidden_sizes": [
739
+ 512
740
+ ],
741
+ "learning_rate": 0.001,
742
+ "alpha": 0.1,
743
+ "accuracy": 0.292
744
+ },
745
+ "top_5_configs": [
746
+ {
747
+ "hidden_sizes": [
748
+ 512
749
+ ],
750
+ "learning_rate": 0.001,
751
+ "alpha": 0.1,
752
+ "accuracy": 0.292
753
+ },
754
+ {
755
+ "hidden_sizes": [
756
+ 256
757
+ ],
758
+ "learning_rate": 0.001,
759
+ "alpha": 0.001,
760
+ "accuracy": 0.28400000000000003
761
+ },
762
+ {
763
+ "hidden_sizes": [
764
+ 256
765
+ ],
766
+ "learning_rate": 0.001,
767
+ "alpha": 0.1,
768
+ "accuracy": 0.28400000000000003
769
+ },
770
+ {
771
+ "hidden_sizes": [
772
+ 256
773
+ ],
774
+ "learning_rate": 0.001,
775
+ "alpha": 0.0001,
776
+ "accuracy": 0.28200000000000003
777
+ },
778
+ {
779
+ "hidden_sizes": [
780
+ 256
781
+ ],
782
+ "learning_rate": 0.001,
783
+ "alpha": 0.01,
784
+ "accuracy": 0.276
785
+ }
786
+ ],
787
+ "all_configs": [
788
+ {
789
+ "hidden_sizes": [
790
+ 512
791
+ ],
792
+ "learning_rate": 0.001,
793
+ "alpha": 0.1,
794
+ "accuracy": 0.292
795
+ },
796
+ {
797
+ "hidden_sizes": [
798
+ 256
799
+ ],
800
+ "learning_rate": 0.001,
801
+ "alpha": 0.001,
802
+ "accuracy": 0.28400000000000003
803
+ },
804
+ {
805
+ "hidden_sizes": [
806
+ 256
807
+ ],
808
+ "learning_rate": 0.001,
809
+ "alpha": 0.1,
810
+ "accuracy": 0.28400000000000003
811
+ },
812
+ {
813
+ "hidden_sizes": [
814
+ 256
815
+ ],
816
+ "learning_rate": 0.001,
817
+ "alpha": 0.0001,
818
+ "accuracy": 0.28200000000000003
819
+ },
820
+ {
821
+ "hidden_sizes": [
822
+ 256
823
+ ],
824
+ "learning_rate": 0.001,
825
+ "alpha": 0.01,
826
+ "accuracy": 0.276
827
+ },
828
+ {
829
+ "hidden_sizes": [
830
+ 512
831
+ ],
832
+ "learning_rate": 0.001,
833
+ "alpha": 0.0001,
834
+ "accuracy": 0.26799999999999996
835
+ },
836
+ {
837
+ "hidden_sizes": [
838
+ 512
839
+ ],
840
+ "learning_rate": 0.001,
841
+ "alpha": 0.001,
842
+ "accuracy": 0.26799999999999996
843
+ },
844
+ {
845
+ "hidden_sizes": [
846
+ 512
847
+ ],
848
+ "learning_rate": 0.001,
849
+ "alpha": 0.01,
850
+ "accuracy": 0.26799999999999996
851
+ },
852
+ {
853
+ "hidden_sizes": [
854
+ 256,
855
+ 128
856
+ ],
857
+ "learning_rate": 0.001,
858
+ "alpha": 0.001,
859
+ "accuracy": 0.262
860
+ },
861
+ {
862
+ "hidden_sizes": [
863
+ 256,
864
+ 128
865
+ ],
866
+ "learning_rate": 0.001,
867
+ "alpha": 0.01,
868
+ "accuracy": 0.262
869
+ },
870
+ {
871
+ "hidden_sizes": [
872
+ 256,
873
+ 128
874
+ ],
875
+ "learning_rate": 0.001,
876
+ "alpha": 0.1,
877
+ "accuracy": 0.262
878
+ },
879
+ {
880
+ "hidden_sizes": [
881
+ 256,
882
+ 128
883
+ ],
884
+ "learning_rate": 0.001,
885
+ "alpha": 0.0001,
886
+ "accuracy": 0.256
887
+ },
888
+ {
889
+ "hidden_sizes": [
890
+ 128
891
+ ],
892
+ "learning_rate": 0.01,
893
+ "alpha": 0.1,
894
+ "accuracy": 0.252
895
+ },
896
+ {
897
+ "hidden_sizes": [
898
+ 128
899
+ ],
900
+ "learning_rate": 0.001,
901
+ "alpha": 0.1,
902
+ "accuracy": 0.246
903
+ },
904
+ {
905
+ "hidden_sizes": [
906
+ 128
907
+ ],
908
+ "learning_rate": 0.01,
909
+ "alpha": 0.01,
910
+ "accuracy": 0.246
911
+ },
912
+ {
913
+ "hidden_sizes": [
914
+ 256
915
+ ],
916
+ "learning_rate": 0.01,
917
+ "alpha": 0.1,
918
+ "accuracy": 0.246
919
+ },
920
+ {
921
+ "hidden_sizes": [
922
+ 128
923
+ ],
924
+ "learning_rate": 0.001,
925
+ "alpha": 0.0001,
926
+ "accuracy": 0.244
927
+ },
928
+ {
929
+ "hidden_sizes": [
930
+ 128,
931
+ 64
932
+ ],
933
+ "learning_rate": 0.001,
934
+ "alpha": 0.1,
935
+ "accuracy": 0.244
936
+ },
937
+ {
938
+ "hidden_sizes": [
939
+ 128
940
+ ],
941
+ "learning_rate": 0.001,
942
+ "alpha": 0.001,
943
+ "accuracy": 0.24200000000000005
944
+ },
945
+ {
946
+ "hidden_sizes": [
947
+ 128,
948
+ 64
949
+ ],
950
+ "learning_rate": 0.001,
951
+ "alpha": 0.0001,
952
+ "accuracy": 0.24
953
+ },
954
+ {
955
+ "hidden_sizes": [
956
+ 128
957
+ ],
958
+ "learning_rate": 0.001,
959
+ "alpha": 0.01,
960
+ "accuracy": 0.23800000000000004
961
+ },
962
+ {
963
+ "hidden_sizes": [
964
+ 128,
965
+ 64
966
+ ],
967
+ "learning_rate": 0.001,
968
+ "alpha": 0.01,
969
+ "accuracy": 0.236
970
+ },
971
+ {
972
+ "hidden_sizes": [
973
+ 128
974
+ ],
975
+ "learning_rate": 0.01,
976
+ "alpha": 0.0001,
977
+ "accuracy": 0.23399999999999999
978
+ },
979
+ {
980
+ "hidden_sizes": [
981
+ 128
982
+ ],
983
+ "learning_rate": 0.01,
984
+ "alpha": 0.001,
985
+ "accuracy": 0.23399999999999999
986
+ },
987
+ {
988
+ "hidden_sizes": [
989
+ 128,
990
+ 64
991
+ ],
992
+ "learning_rate": 0.001,
993
+ "alpha": 0.001,
994
+ "accuracy": 0.23399999999999999
995
+ },
996
+ {
997
+ "hidden_sizes": [
998
+ 64
999
+ ],
1000
+ "learning_rate": 0.001,
1001
+ "alpha": 0.1,
1002
+ "accuracy": 0.23200000000000004
1003
+ },
1004
+ {
1005
+ "hidden_sizes": [
1006
+ 256
1007
+ ],
1008
+ "learning_rate": 0.01,
1009
+ "alpha": 0.01,
1010
+ "accuracy": 0.23199999999999998
1011
+ },
1012
+ {
1013
+ "hidden_sizes": [
1014
+ 64
1015
+ ],
1016
+ "learning_rate": 0.001,
1017
+ "alpha": 0.0001,
1018
+ "accuracy": 0.23000000000000004
1019
+ },
1020
+ {
1021
+ "hidden_sizes": [
1022
+ 64
1023
+ ],
1024
+ "learning_rate": 0.001,
1025
+ "alpha": 0.001,
1026
+ "accuracy": 0.22800000000000004
1027
+ },
1028
+ {
1029
+ "hidden_sizes": [
1030
+ 64
1031
+ ],
1032
+ "learning_rate": 0.001,
1033
+ "alpha": 0.01,
1034
+ "accuracy": 0.22800000000000004
1035
+ },
1036
+ {
1037
+ "hidden_sizes": [
1038
+ 512
1039
+ ],
1040
+ "learning_rate": 0.01,
1041
+ "alpha": 0.01,
1042
+ "accuracy": 0.22800000000000004
1043
+ },
1044
+ {
1045
+ "hidden_sizes": [
1046
+ 512
1047
+ ],
1048
+ "learning_rate": 0.01,
1049
+ "alpha": 0.1,
1050
+ "accuracy": 0.22800000000000004
1051
+ },
1052
+ {
1053
+ "hidden_sizes": [
1054
+ 256,
1055
+ 128
1056
+ ],
1057
+ "learning_rate": 0.01,
1058
+ "alpha": 0.001,
1059
+ "accuracy": 0.22799999999999998
1060
+ },
1061
+ {
1062
+ "hidden_sizes": [
1063
+ 128,
1064
+ 64
1065
+ ],
1066
+ "learning_rate": 0.01,
1067
+ "alpha": 0.1,
1068
+ "accuracy": 0.22599999999999998
1069
+ },
1070
+ {
1071
+ "hidden_sizes": [
1072
+ 256
1073
+ ],
1074
+ "learning_rate": 0.01,
1075
+ "alpha": 0.0001,
1076
+ "accuracy": 0.22399999999999998
1077
+ },
1078
+ {
1079
+ "hidden_sizes": [
1080
+ 256
1081
+ ],
1082
+ "learning_rate": 0.01,
1083
+ "alpha": 0.001,
1084
+ "accuracy": 0.22199999999999998
1085
+ },
1086
+ {
1087
+ "hidden_sizes": [
1088
+ 128,
1089
+ 64
1090
+ ],
1091
+ "learning_rate": 0.01,
1092
+ "alpha": 0.001,
1093
+ "accuracy": 0.21400000000000002
1094
+ },
1095
+ {
1096
+ "hidden_sizes": [
1097
+ 256,
1098
+ 128
1099
+ ],
1100
+ "learning_rate": 0.01,
1101
+ "alpha": 0.0001,
1102
+ "accuracy": 0.21000000000000002
1103
+ },
1104
+ {
1105
+ "hidden_sizes": [
1106
+ 128,
1107
+ 64
1108
+ ],
1109
+ "learning_rate": 0.01,
1110
+ "alpha": 0.01,
1111
+ "accuracy": 0.198
1112
+ },
1113
+ {
1114
+ "hidden_sizes": [
1115
+ 128,
1116
+ 64
1117
+ ],
1118
+ "learning_rate": 0.01,
1119
+ "alpha": 0.0001,
1120
+ "accuracy": 0.19
1121
+ },
1122
+ {
1123
+ "hidden_sizes": [
1124
+ 64
1125
+ ],
1126
+ "learning_rate": 0.01,
1127
+ "alpha": 0.01,
1128
+ "accuracy": 0.188
1129
+ },
1130
+ {
1131
+ "hidden_sizes": [
1132
+ 512
1133
+ ],
1134
+ "learning_rate": 0.01,
1135
+ "alpha": 0.001,
1136
+ "accuracy": 0.188
1137
+ },
1138
+ {
1139
+ "hidden_sizes": [
1140
+ 64
1141
+ ],
1142
+ "learning_rate": 0.01,
1143
+ "alpha": 0.1,
1144
+ "accuracy": 0.186
1145
+ },
1146
+ {
1147
+ "hidden_sizes": [
1148
+ 256
1149
+ ],
1150
+ "learning_rate": 0.0001,
1151
+ "alpha": 0.1,
1152
+ "accuracy": 0.186
1153
+ },
1154
+ {
1155
+ "hidden_sizes": [
1156
+ 512
1157
+ ],
1158
+ "learning_rate": 0.01,
1159
+ "alpha": 0.0001,
1160
+ "accuracy": 0.186
1161
+ },
1162
+ {
1163
+ "hidden_sizes": [
1164
+ 64
1165
+ ],
1166
+ "learning_rate": 0.01,
1167
+ "alpha": 0.001,
1168
+ "accuracy": 0.184
1169
+ },
1170
+ {
1171
+ "hidden_sizes": [
1172
+ 256
1173
+ ],
1174
+ "learning_rate": 0.0001,
1175
+ "alpha": 0.0001,
1176
+ "accuracy": 0.184
1177
+ },
1178
+ {
1179
+ "hidden_sizes": [
1180
+ 256
1181
+ ],
1182
+ "learning_rate": 0.0001,
1183
+ "alpha": 0.001,
1184
+ "accuracy": 0.184
1185
+ },
1186
+ {
1187
+ "hidden_sizes": [
1188
+ 256
1189
+ ],
1190
+ "learning_rate": 0.0001,
1191
+ "alpha": 0.01,
1192
+ "accuracy": 0.184
1193
+ },
1194
+ {
1195
+ "hidden_sizes": [
1196
+ 256,
1197
+ 128
1198
+ ],
1199
+ "learning_rate": 0.01,
1200
+ "alpha": 0.01,
1201
+ "accuracy": 0.184
1202
+ },
1203
+ {
1204
+ "hidden_sizes": [
1205
+ 64
1206
+ ],
1207
+ "learning_rate": 0.01,
1208
+ "alpha": 0.0001,
1209
+ "accuracy": 0.182
1210
+ },
1211
+ {
1212
+ "hidden_sizes": [
1213
+ 512
1214
+ ],
1215
+ "learning_rate": 0.0001,
1216
+ "alpha": 0.1,
1217
+ "accuracy": 0.182
1218
+ },
1219
+ {
1220
+ "hidden_sizes": [
1221
+ 512
1222
+ ],
1223
+ "learning_rate": 0.0001,
1224
+ "alpha": 0.001,
1225
+ "accuracy": 0.178
1226
+ },
1227
+ {
1228
+ "hidden_sizes": [
1229
+ 512
1230
+ ],
1231
+ "learning_rate": 0.0001,
1232
+ "alpha": 0.01,
1233
+ "accuracy": 0.178
1234
+ },
1235
+ {
1236
+ "hidden_sizes": [
1237
+ 512
1238
+ ],
1239
+ "learning_rate": 0.0001,
1240
+ "alpha": 0.0001,
1241
+ "accuracy": 0.176
1242
+ },
1243
+ {
1244
+ "hidden_sizes": [
1245
+ 256,
1246
+ 128
1247
+ ],
1248
+ "learning_rate": 0.01,
1249
+ "alpha": 0.1,
1250
+ "accuracy": 0.174
1251
+ },
1252
+ {
1253
+ "hidden_sizes": [
1254
+ 256,
1255
+ 128
1256
+ ],
1257
+ "learning_rate": 0.0001,
1258
+ "alpha": 0.001,
1259
+ "accuracy": 0.168
1260
+ },
1261
+ {
1262
+ "hidden_sizes": [
1263
+ 256,
1264
+ 128
1265
+ ],
1266
+ "learning_rate": 0.0001,
1267
+ "alpha": 0.1,
1268
+ "accuracy": 0.168
1269
+ },
1270
+ {
1271
+ "hidden_sizes": [
1272
+ 256,
1273
+ 128
1274
+ ],
1275
+ "learning_rate": 0.0001,
1276
+ "alpha": 0.0001,
1277
+ "accuracy": 0.166
1278
+ },
1279
+ {
1280
+ "hidden_sizes": [
1281
+ 256,
1282
+ 128
1283
+ ],
1284
+ "learning_rate": 0.0001,
1285
+ "alpha": 0.01,
1286
+ "accuracy": 0.166
1287
+ },
1288
+ {
1289
+ "hidden_sizes": [
1290
+ 128
1291
+ ],
1292
+ "learning_rate": 0.0001,
1293
+ "alpha": 0.1,
1294
+ "accuracy": 0.15400000000000003
1295
+ },
1296
+ {
1297
+ "hidden_sizes": [
1298
+ 128
1299
+ ],
1300
+ "learning_rate": 0.0001,
1301
+ "alpha": 0.01,
1302
+ "accuracy": 0.15200000000000002
1303
+ },
1304
+ {
1305
+ "hidden_sizes": [
1306
+ 128
1307
+ ],
1308
+ "learning_rate": 0.0001,
1309
+ "alpha": 0.0001,
1310
+ "accuracy": 0.15000000000000002
1311
+ },
1312
+ {
1313
+ "hidden_sizes": [
1314
+ 128
1315
+ ],
1316
+ "learning_rate": 0.0001,
1317
+ "alpha": 0.001,
1318
+ "accuracy": 0.15000000000000002
1319
+ },
1320
+ {
1321
+ "hidden_sizes": [
1322
+ 128,
1323
+ 64
1324
+ ],
1325
+ "learning_rate": 0.0001,
1326
+ "alpha": 0.0001,
1327
+ "accuracy": 0.096
1328
+ },
1329
+ {
1330
+ "hidden_sizes": [
1331
+ 128,
1332
+ 64
1333
+ ],
1334
+ "learning_rate": 0.0001,
1335
+ "alpha": 0.001,
1336
+ "accuracy": 0.096
1337
+ },
1338
+ {
1339
+ "hidden_sizes": [
1340
+ 128,
1341
+ 64
1342
+ ],
1343
+ "learning_rate": 0.0001,
1344
+ "alpha": 0.01,
1345
+ "accuracy": 0.096
1346
+ },
1347
+ {
1348
+ "hidden_sizes": [
1349
+ 128,
1350
+ 64
1351
+ ],
1352
+ "learning_rate": 0.0001,
1353
+ "alpha": 0.1,
1354
+ "accuracy": 0.096
1355
+ },
1356
+ {
1357
+ "hidden_sizes": [
1358
+ 64
1359
+ ],
1360
+ "learning_rate": 0.0001,
1361
+ "alpha": 0.0001,
1362
+ "accuracy": 0.06
1363
+ },
1364
+ {
1365
+ "hidden_sizes": [
1366
+ 64
1367
+ ],
1368
+ "learning_rate": 0.0001,
1369
+ "alpha": 0.001,
1370
+ "accuracy": 0.06
1371
+ },
1372
+ {
1373
+ "hidden_sizes": [
1374
+ 64
1375
+ ],
1376
+ "learning_rate": 0.0001,
1377
+ "alpha": 0.01,
1378
+ "accuracy": 0.06
1379
+ },
1380
+ {
1381
+ "hidden_sizes": [
1382
+ 64
1383
+ ],
1384
+ "learning_rate": 0.0001,
1385
+ "alpha": 0.1,
1386
+ "accuracy": 0.06
1387
+ }
1388
+ ]
1389
+ },
1390
+ {
1391
+ "model": "m5",
1392
+ "layer": 12,
1393
+ "position": 3,
1394
+ "n_samples": 298,
1395
+ "n_classes": 38,
1396
+ "linear_accuracy": 0.5704697986577181,
1397
+ "best_mlp_accuracy": 0.4563758389261745,
1398
+ "mlp_advantage_pp": -11.41,
1399
+ "best_config": {
1400
+ "hidden_sizes": [
1401
+ 128
1402
+ ],
1403
+ "learning_rate": 0.001,
1404
+ "alpha": 0.01,
1405
+ "accuracy": 0.4563758389261745
1406
+ },
1407
+ "top_5_configs": [
1408
+ {
1409
+ "hidden_sizes": [
1410
+ 128
1411
+ ],
1412
+ "learning_rate": 0.001,
1413
+ "alpha": 0.01,
1414
+ "accuracy": 0.4563758389261745
1415
+ },
1416
+ {
1417
+ "hidden_sizes": [
1418
+ 128
1419
+ ],
1420
+ "learning_rate": 0.001,
1421
+ "alpha": 0.1,
1422
+ "accuracy": 0.4563758389261745
1423
+ },
1424
+ {
1425
+ "hidden_sizes": [
1426
+ 128
1427
+ ],
1428
+ "learning_rate": 0.001,
1429
+ "alpha": 0.0001,
1430
+ "accuracy": 0.44966442953020136
1431
+ },
1432
+ {
1433
+ "hidden_sizes": [
1434
+ 128
1435
+ ],
1436
+ "learning_rate": 0.001,
1437
+ "alpha": 0.001,
1438
+ "accuracy": 0.44966442953020136
1439
+ },
1440
+ {
1441
+ "hidden_sizes": [
1442
+ 256
1443
+ ],
1444
+ "learning_rate": 0.001,
1445
+ "alpha": 0.0001,
1446
+ "accuracy": 0.43288590604026844
1447
+ }
1448
+ ],
1449
+ "all_configs": [
1450
+ {
1451
+ "hidden_sizes": [
1452
+ 128
1453
+ ],
1454
+ "learning_rate": 0.001,
1455
+ "alpha": 0.01,
1456
+ "accuracy": 0.4563758389261745
1457
+ },
1458
+ {
1459
+ "hidden_sizes": [
1460
+ 128
1461
+ ],
1462
+ "learning_rate": 0.001,
1463
+ "alpha": 0.1,
1464
+ "accuracy": 0.4563758389261745
1465
+ },
1466
+ {
1467
+ "hidden_sizes": [
1468
+ 128
1469
+ ],
1470
+ "learning_rate": 0.001,
1471
+ "alpha": 0.0001,
1472
+ "accuracy": 0.44966442953020136
1473
+ },
1474
+ {
1475
+ "hidden_sizes": [
1476
+ 128
1477
+ ],
1478
+ "learning_rate": 0.001,
1479
+ "alpha": 0.001,
1480
+ "accuracy": 0.44966442953020136
1481
+ },
1482
+ {
1483
+ "hidden_sizes": [
1484
+ 256
1485
+ ],
1486
+ "learning_rate": 0.001,
1487
+ "alpha": 0.0001,
1488
+ "accuracy": 0.43288590604026844
1489
+ },
1490
+ {
1491
+ "hidden_sizes": [
1492
+ 256
1493
+ ],
1494
+ "learning_rate": 0.001,
1495
+ "alpha": 0.01,
1496
+ "accuracy": 0.43288590604026844
1497
+ },
1498
+ {
1499
+ "hidden_sizes": [
1500
+ 256,
1501
+ 128
1502
+ ],
1503
+ "learning_rate": 0.001,
1504
+ "alpha": 0.1,
1505
+ "accuracy": 0.43288590604026844
1506
+ },
1507
+ {
1508
+ "hidden_sizes": [
1509
+ 256
1510
+ ],
1511
+ "learning_rate": 0.001,
1512
+ "alpha": 0.001,
1513
+ "accuracy": 0.4295302013422819
1514
+ },
1515
+ {
1516
+ "hidden_sizes": [
1517
+ 128
1518
+ ],
1519
+ "learning_rate": 0.01,
1520
+ "alpha": 0.1,
1521
+ "accuracy": 0.412751677852349
1522
+ },
1523
+ {
1524
+ "hidden_sizes": [
1525
+ 256
1526
+ ],
1527
+ "learning_rate": 0.01,
1528
+ "alpha": 0.1,
1529
+ "accuracy": 0.412751677852349
1530
+ },
1531
+ {
1532
+ "hidden_sizes": [
1533
+ 128
1534
+ ],
1535
+ "learning_rate": 0.01,
1536
+ "alpha": 0.0001,
1537
+ "accuracy": 0.40939597315436244
1538
+ },
1539
+ {
1540
+ "hidden_sizes": [
1541
+ 128
1542
+ ],
1543
+ "learning_rate": 0.01,
1544
+ "alpha": 0.001,
1545
+ "accuracy": 0.40939597315436244
1546
+ },
1547
+ {
1548
+ "hidden_sizes": [
1549
+ 256
1550
+ ],
1551
+ "learning_rate": 0.01,
1552
+ "alpha": 0.001,
1553
+ "accuracy": 0.40939597315436244
1554
+ },
1555
+ {
1556
+ "hidden_sizes": [
1557
+ 256
1558
+ ],
1559
+ "learning_rate": 0.01,
1560
+ "alpha": 0.01,
1561
+ "accuracy": 0.40939597315436244
1562
+ },
1563
+ {
1564
+ "hidden_sizes": [
1565
+ 128
1566
+ ],
1567
+ "learning_rate": 0.01,
1568
+ "alpha": 0.01,
1569
+ "accuracy": 0.40604026845637586
1570
+ },
1571
+ {
1572
+ "hidden_sizes": [
1573
+ 256
1574
+ ],
1575
+ "learning_rate": 0.01,
1576
+ "alpha": 0.0001,
1577
+ "accuracy": 0.40604026845637586
1578
+ },
1579
+ {
1580
+ "hidden_sizes": [
1581
+ 256
1582
+ ],
1583
+ "learning_rate": 0.001,
1584
+ "alpha": 0.1,
1585
+ "accuracy": 0.3993288590604027
1586
+ },
1587
+ {
1588
+ "hidden_sizes": [
1589
+ 512
1590
+ ],
1591
+ "learning_rate": 0.001,
1592
+ "alpha": 0.0001,
1593
+ "accuracy": 0.3993288590604027
1594
+ },
1595
+ {
1596
+ "hidden_sizes": [
1597
+ 512
1598
+ ],
1599
+ "learning_rate": 0.001,
1600
+ "alpha": 0.1,
1601
+ "accuracy": 0.3993288590604027
1602
+ },
1603
+ {
1604
+ "hidden_sizes": [
1605
+ 512
1606
+ ],
1607
+ "learning_rate": 0.001,
1608
+ "alpha": 0.001,
1609
+ "accuracy": 0.3959731543624161
1610
+ },
1611
+ {
1612
+ "hidden_sizes": [
1613
+ 512
1614
+ ],
1615
+ "learning_rate": 0.001,
1616
+ "alpha": 0.01,
1617
+ "accuracy": 0.3959731543624161
1618
+ },
1619
+ {
1620
+ "hidden_sizes": [
1621
+ 64
1622
+ ],
1623
+ "learning_rate": 0.01,
1624
+ "alpha": 0.1,
1625
+ "accuracy": 0.3859060402684564
1626
+ },
1627
+ {
1628
+ "hidden_sizes": [
1629
+ 64
1630
+ ],
1631
+ "learning_rate": 0.01,
1632
+ "alpha": 0.0001,
1633
+ "accuracy": 0.3825503355704698
1634
+ },
1635
+ {
1636
+ "hidden_sizes": [
1637
+ 64
1638
+ ],
1639
+ "learning_rate": 0.01,
1640
+ "alpha": 0.001,
1641
+ "accuracy": 0.3825503355704698
1642
+ },
1643
+ {
1644
+ "hidden_sizes": [
1645
+ 64
1646
+ ],
1647
+ "learning_rate": 0.01,
1648
+ "alpha": 0.01,
1649
+ "accuracy": 0.3825503355704698
1650
+ },
1651
+ {
1652
+ "hidden_sizes": [
1653
+ 256,
1654
+ 128
1655
+ ],
1656
+ "learning_rate": 0.001,
1657
+ "alpha": 0.001,
1658
+ "accuracy": 0.3825503355704698
1659
+ },
1660
+ {
1661
+ "hidden_sizes": [
1662
+ 256,
1663
+ 128
1664
+ ],
1665
+ "learning_rate": 0.001,
1666
+ "alpha": 0.0001,
1667
+ "accuracy": 0.37919463087248323
1668
+ },
1669
+ {
1670
+ "hidden_sizes": [
1671
+ 256,
1672
+ 128
1673
+ ],
1674
+ "learning_rate": 0.001,
1675
+ "alpha": 0.01,
1676
+ "accuracy": 0.37919463087248323
1677
+ },
1678
+ {
1679
+ "hidden_sizes": [
1680
+ 256,
1681
+ 128
1682
+ ],
1683
+ "learning_rate": 0.01,
1684
+ "alpha": 0.01,
1685
+ "accuracy": 0.37919463087248323
1686
+ },
1687
+ {
1688
+ "hidden_sizes": [
1689
+ 256,
1690
+ 128
1691
+ ],
1692
+ "learning_rate": 0.01,
1693
+ "alpha": 0.1,
1694
+ "accuracy": 0.3791946308724832
1695
+ },
1696
+ {
1697
+ "hidden_sizes": [
1698
+ 128,
1699
+ 64
1700
+ ],
1701
+ "learning_rate": 0.001,
1702
+ "alpha": 0.001,
1703
+ "accuracy": 0.37248322147651003
1704
+ },
1705
+ {
1706
+ "hidden_sizes": [
1707
+ 256,
1708
+ 128
1709
+ ],
1710
+ "learning_rate": 0.01,
1711
+ "alpha": 0.001,
1712
+ "accuracy": 0.37248322147651003
1713
+ },
1714
+ {
1715
+ "hidden_sizes": [
1716
+ 128,
1717
+ 64
1718
+ ],
1719
+ "learning_rate": 0.001,
1720
+ "alpha": 0.01,
1721
+ "accuracy": 0.3691275167785235
1722
+ },
1723
+ {
1724
+ "hidden_sizes": [
1725
+ 512
1726
+ ],
1727
+ "learning_rate": 0.01,
1728
+ "alpha": 0.1,
1729
+ "accuracy": 0.3657718120805369
1730
+ },
1731
+ {
1732
+ "hidden_sizes": [
1733
+ 256,
1734
+ 128
1735
+ ],
1736
+ "learning_rate": 0.01,
1737
+ "alpha": 0.0001,
1738
+ "accuracy": 0.36241610738255037
1739
+ },
1740
+ {
1741
+ "hidden_sizes": [
1742
+ 512
1743
+ ],
1744
+ "learning_rate": 0.01,
1745
+ "alpha": 0.0001,
1746
+ "accuracy": 0.35570469798657717
1747
+ },
1748
+ {
1749
+ "hidden_sizes": [
1750
+ 512
1751
+ ],
1752
+ "learning_rate": 0.01,
1753
+ "alpha": 0.001,
1754
+ "accuracy": 0.35570469798657717
1755
+ },
1756
+ {
1757
+ "hidden_sizes": [
1758
+ 512
1759
+ ],
1760
+ "learning_rate": 0.01,
1761
+ "alpha": 0.01,
1762
+ "accuracy": 0.35570469798657717
1763
+ },
1764
+ {
1765
+ "hidden_sizes": [
1766
+ 128,
1767
+ 64
1768
+ ],
1769
+ "learning_rate": 0.01,
1770
+ "alpha": 0.0001,
1771
+ "accuracy": 0.30201342281879195
1772
+ },
1773
+ {
1774
+ "hidden_sizes": [
1775
+ 128,
1776
+ 64
1777
+ ],
1778
+ "learning_rate": 0.01,
1779
+ "alpha": 0.1,
1780
+ "accuracy": 0.30201342281879195
1781
+ },
1782
+ {
1783
+ "hidden_sizes": [
1784
+ 128,
1785
+ 64
1786
+ ],
1787
+ "learning_rate": 0.01,
1788
+ "alpha": 0.001,
1789
+ "accuracy": 0.2986577181208054
1790
+ },
1791
+ {
1792
+ "hidden_sizes": [
1793
+ 128,
1794
+ 64
1795
+ ],
1796
+ "learning_rate": 0.01,
1797
+ "alpha": 0.01,
1798
+ "accuracy": 0.2986577181208054
1799
+ },
1800
+ {
1801
+ "hidden_sizes": [
1802
+ 128,
1803
+ 64
1804
+ ],
1805
+ "learning_rate": 0.001,
1806
+ "alpha": 0.1,
1807
+ "accuracy": 0.22483221476510068
1808
+ },
1809
+ {
1810
+ "hidden_sizes": [
1811
+ 128,
1812
+ 64
1813
+ ],
1814
+ "learning_rate": 0.001,
1815
+ "alpha": 0.0001,
1816
+ "accuracy": 0.21140939597315436
1817
+ },
1818
+ {
1819
+ "hidden_sizes": [
1820
+ 64
1821
+ ],
1822
+ "learning_rate": 0.001,
1823
+ "alpha": 0.0001,
1824
+ "accuracy": 0.1912751677852349
1825
+ },
1826
+ {
1827
+ "hidden_sizes": [
1828
+ 64
1829
+ ],
1830
+ "learning_rate": 0.001,
1831
+ "alpha": 0.001,
1832
+ "accuracy": 0.1912751677852349
1833
+ },
1834
+ {
1835
+ "hidden_sizes": [
1836
+ 64
1837
+ ],
1838
+ "learning_rate": 0.001,
1839
+ "alpha": 0.01,
1840
+ "accuracy": 0.1912751677852349
1841
+ },
1842
+ {
1843
+ "hidden_sizes": [
1844
+ 64
1845
+ ],
1846
+ "learning_rate": 0.001,
1847
+ "alpha": 0.1,
1848
+ "accuracy": 0.1912751677852349
1849
+ },
1850
+ {
1851
+ "hidden_sizes": [
1852
+ 128
1853
+ ],
1854
+ "learning_rate": 0.0001,
1855
+ "alpha": 0.0001,
1856
+ "accuracy": 0.09731543624161074
1857
+ },
1858
+ {
1859
+ "hidden_sizes": [
1860
+ 128
1861
+ ],
1862
+ "learning_rate": 0.0001,
1863
+ "alpha": 0.001,
1864
+ "accuracy": 0.09731543624161074
1865
+ },
1866
+ {
1867
+ "hidden_sizes": [
1868
+ 128
1869
+ ],
1870
+ "learning_rate": 0.0001,
1871
+ "alpha": 0.01,
1872
+ "accuracy": 0.09731543624161074
1873
+ },
1874
+ {
1875
+ "hidden_sizes": [
1876
+ 128
1877
+ ],
1878
+ "learning_rate": 0.0001,
1879
+ "alpha": 0.1,
1880
+ "accuracy": 0.09731543624161074
1881
+ },
1882
+ {
1883
+ "hidden_sizes": [
1884
+ 256,
1885
+ 128
1886
+ ],
1887
+ "learning_rate": 0.0001,
1888
+ "alpha": 0.0001,
1889
+ "accuracy": 0.07046979865771812
1890
+ },
1891
+ {
1892
+ "hidden_sizes": [
1893
+ 256,
1894
+ 128
1895
+ ],
1896
+ "learning_rate": 0.0001,
1897
+ "alpha": 0.001,
1898
+ "accuracy": 0.07046979865771812
1899
+ },
1900
+ {
1901
+ "hidden_sizes": [
1902
+ 256,
1903
+ 128
1904
+ ],
1905
+ "learning_rate": 0.0001,
1906
+ "alpha": 0.01,
1907
+ "accuracy": 0.07046979865771812
1908
+ },
1909
+ {
1910
+ "hidden_sizes": [
1911
+ 256,
1912
+ 128
1913
+ ],
1914
+ "learning_rate": 0.0001,
1915
+ "alpha": 0.1,
1916
+ "accuracy": 0.07046979865771812
1917
+ },
1918
+ {
1919
+ "hidden_sizes": [
1920
+ 256
1921
+ ],
1922
+ "learning_rate": 0.0001,
1923
+ "alpha": 0.0001,
1924
+ "accuracy": 0.05033557046979866
1925
+ },
1926
+ {
1927
+ "hidden_sizes": [
1928
+ 256
1929
+ ],
1930
+ "learning_rate": 0.0001,
1931
+ "alpha": 0.001,
1932
+ "accuracy": 0.05033557046979866
1933
+ },
1934
+ {
1935
+ "hidden_sizes": [
1936
+ 256
1937
+ ],
1938
+ "learning_rate": 0.0001,
1939
+ "alpha": 0.01,
1940
+ "accuracy": 0.05033557046979866
1941
+ },
1942
+ {
1943
+ "hidden_sizes": [
1944
+ 256
1945
+ ],
1946
+ "learning_rate": 0.0001,
1947
+ "alpha": 0.1,
1948
+ "accuracy": 0.05033557046979866
1949
+ },
1950
+ {
1951
+ "hidden_sizes": [
1952
+ 512
1953
+ ],
1954
+ "learning_rate": 0.0001,
1955
+ "alpha": 0.0001,
1956
+ "accuracy": 0.026845637583892617
1957
+ },
1958
+ {
1959
+ "hidden_sizes": [
1960
+ 512
1961
+ ],
1962
+ "learning_rate": 0.0001,
1963
+ "alpha": 0.001,
1964
+ "accuracy": 0.026845637583892617
1965
+ },
1966
+ {
1967
+ "hidden_sizes": [
1968
+ 512
1969
+ ],
1970
+ "learning_rate": 0.0001,
1971
+ "alpha": 0.01,
1972
+ "accuracy": 0.026845637583892617
1973
+ },
1974
+ {
1975
+ "hidden_sizes": [
1976
+ 512
1977
+ ],
1978
+ "learning_rate": 0.0001,
1979
+ "alpha": 0.1,
1980
+ "accuracy": 0.026845637583892617
1981
+ },
1982
+ {
1983
+ "hidden_sizes": [
1984
+ 128,
1985
+ 64
1986
+ ],
1987
+ "learning_rate": 0.0001,
1988
+ "alpha": 0.0001,
1989
+ "accuracy": 0.020134228187919462
1990
+ },
1991
+ {
1992
+ "hidden_sizes": [
1993
+ 128,
1994
+ 64
1995
+ ],
1996
+ "learning_rate": 0.0001,
1997
+ "alpha": 0.001,
1998
+ "accuracy": 0.020134228187919462
1999
+ },
2000
+ {
2001
+ "hidden_sizes": [
2002
+ 128,
2003
+ 64
2004
+ ],
2005
+ "learning_rate": 0.0001,
2006
+ "alpha": 0.01,
2007
+ "accuracy": 0.020134228187919462
2008
+ },
2009
+ {
2010
+ "hidden_sizes": [
2011
+ 128,
2012
+ 64
2013
+ ],
2014
+ "learning_rate": 0.0001,
2015
+ "alpha": 0.1,
2016
+ "accuracy": 0.020134228187919462
2017
+ },
2018
+ {
2019
+ "hidden_sizes": [
2020
+ 64
2021
+ ],
2022
+ "learning_rate": 0.0001,
2023
+ "alpha": 0.0001,
2024
+ "accuracy": 0.006711409395973154
2025
+ },
2026
+ {
2027
+ "hidden_sizes": [
2028
+ 64
2029
+ ],
2030
+ "learning_rate": 0.0001,
2031
+ "alpha": 0.001,
2032
+ "accuracy": 0.006711409395973154
2033
+ },
2034
+ {
2035
+ "hidden_sizes": [
2036
+ 64
2037
+ ],
2038
+ "learning_rate": 0.0001,
2039
+ "alpha": 0.01,
2040
+ "accuracy": 0.006711409395973154
2041
+ },
2042
+ {
2043
+ "hidden_sizes": [
2044
+ 64
2045
+ ],
2046
+ "learning_rate": 0.0001,
2047
+ "alpha": 0.1,
2048
+ "accuracy": 0.006711409395973154
2049
+ }
2050
+ ]
2051
+ },
2052
+ {
2053
+ "model": "m5",
2054
+ "layer": 8,
2055
+ "position": 0,
2056
+ "n_samples": 500,
2057
+ "n_classes": 38,
2058
+ "linear_accuracy": 0.236,
2059
+ "best_mlp_accuracy": 0.14600000000000002,
2060
+ "mlp_advantage_pp": -9.0,
2061
+ "best_config": {
2062
+ "hidden_sizes": [
2063
+ 256
2064
+ ],
2065
+ "learning_rate": 0.01,
2066
+ "alpha": 0.01,
2067
+ "accuracy": 0.14600000000000002
2068
+ },
2069
+ "top_5_configs": [
2070
+ {
2071
+ "hidden_sizes": [
2072
+ 256
2073
+ ],
2074
+ "learning_rate": 0.01,
2075
+ "alpha": 0.01,
2076
+ "accuracy": 0.14600000000000002
2077
+ },
2078
+ {
2079
+ "hidden_sizes": [
2080
+ 64
2081
+ ],
2082
+ "learning_rate": 0.001,
2083
+ "alpha": 0.0001,
2084
+ "accuracy": 0.144
2085
+ },
2086
+ {
2087
+ "hidden_sizes": [
2088
+ 256
2089
+ ],
2090
+ "learning_rate": 0.001,
2091
+ "alpha": 0.1,
2092
+ "accuracy": 0.144
2093
+ },
2094
+ {
2095
+ "hidden_sizes": [
2096
+ 512
2097
+ ],
2098
+ "learning_rate": 0.001,
2099
+ "alpha": 0.01,
2100
+ "accuracy": 0.144
2101
+ },
2102
+ {
2103
+ "hidden_sizes": [
2104
+ 256
2105
+ ],
2106
+ "learning_rate": 0.01,
2107
+ "alpha": 0.0001,
2108
+ "accuracy": 0.14200000000000002
2109
+ }
2110
+ ],
2111
+ "all_configs": [
2112
+ {
2113
+ "hidden_sizes": [
2114
+ 256
2115
+ ],
2116
+ "learning_rate": 0.01,
2117
+ "alpha": 0.01,
2118
+ "accuracy": 0.14600000000000002
2119
+ },
2120
+ {
2121
+ "hidden_sizes": [
2122
+ 64
2123
+ ],
2124
+ "learning_rate": 0.001,
2125
+ "alpha": 0.0001,
2126
+ "accuracy": 0.144
2127
+ },
2128
+ {
2129
+ "hidden_sizes": [
2130
+ 256
2131
+ ],
2132
+ "learning_rate": 0.001,
2133
+ "alpha": 0.1,
2134
+ "accuracy": 0.144
2135
+ },
2136
+ {
2137
+ "hidden_sizes": [
2138
+ 512
2139
+ ],
2140
+ "learning_rate": 0.001,
2141
+ "alpha": 0.01,
2142
+ "accuracy": 0.144
2143
+ },
2144
+ {
2145
+ "hidden_sizes": [
2146
+ 256
2147
+ ],
2148
+ "learning_rate": 0.01,
2149
+ "alpha": 0.0001,
2150
+ "accuracy": 0.14200000000000002
2151
+ },
2152
+ {
2153
+ "hidden_sizes": [
2154
+ 64
2155
+ ],
2156
+ "learning_rate": 0.001,
2157
+ "alpha": 0.001,
2158
+ "accuracy": 0.142
2159
+ },
2160
+ {
2161
+ "hidden_sizes": [
2162
+ 64
2163
+ ],
2164
+ "learning_rate": 0.001,
2165
+ "alpha": 0.1,
2166
+ "accuracy": 0.14
2167
+ },
2168
+ {
2169
+ "hidden_sizes": [
2170
+ 512
2171
+ ],
2172
+ "learning_rate": 0.001,
2173
+ "alpha": 0.1,
2174
+ "accuracy": 0.14
2175
+ },
2176
+ {
2177
+ "hidden_sizes": [
2178
+ 64
2179
+ ],
2180
+ "learning_rate": 0.001,
2181
+ "alpha": 0.01,
2182
+ "accuracy": 0.13999999999999999
2183
+ },
2184
+ {
2185
+ "hidden_sizes": [
2186
+ 512
2187
+ ],
2188
+ "learning_rate": 0.001,
2189
+ "alpha": 0.0001,
2190
+ "accuracy": 0.13999999999999999
2191
+ },
2192
+ {
2193
+ "hidden_sizes": [
2194
+ 512
2195
+ ],
2196
+ "learning_rate": 0.001,
2197
+ "alpha": 0.001,
2198
+ "accuracy": 0.13999999999999999
2199
+ },
2200
+ {
2201
+ "hidden_sizes": [
2202
+ 256
2203
+ ],
2204
+ "learning_rate": 0.001,
2205
+ "alpha": 0.001,
2206
+ "accuracy": 0.138
2207
+ },
2208
+ {
2209
+ "hidden_sizes": [
2210
+ 256
2211
+ ],
2212
+ "learning_rate": 0.001,
2213
+ "alpha": 0.0001,
2214
+ "accuracy": 0.136
2215
+ },
2216
+ {
2217
+ "hidden_sizes": [
2218
+ 256
2219
+ ],
2220
+ "learning_rate": 0.01,
2221
+ "alpha": 0.1,
2222
+ "accuracy": 0.13199999999999998
2223
+ },
2224
+ {
2225
+ "hidden_sizes": [
2226
+ 128
2227
+ ],
2228
+ "learning_rate": 0.001,
2229
+ "alpha": 0.01,
2230
+ "accuracy": 0.128
2231
+ },
2232
+ {
2233
+ "hidden_sizes": [
2234
+ 128
2235
+ ],
2236
+ "learning_rate": 0.01,
2237
+ "alpha": 0.001,
2238
+ "accuracy": 0.128
2239
+ },
2240
+ {
2241
+ "hidden_sizes": [
2242
+ 256
2243
+ ],
2244
+ "learning_rate": 0.001,
2245
+ "alpha": 0.01,
2246
+ "accuracy": 0.128
2247
+ },
2248
+ {
2249
+ "hidden_sizes": [
2250
+ 256,
2251
+ 128
2252
+ ],
2253
+ "learning_rate": 0.01,
2254
+ "alpha": 0.01,
2255
+ "accuracy": 0.128
2256
+ },
2257
+ {
2258
+ "hidden_sizes": [
2259
+ 512
2260
+ ],
2261
+ "learning_rate": 0.01,
2262
+ "alpha": 0.1,
2263
+ "accuracy": 0.126
2264
+ },
2265
+ {
2266
+ "hidden_sizes": [
2267
+ 512
2268
+ ],
2269
+ "learning_rate": 0.01,
2270
+ "alpha": 0.01,
2271
+ "accuracy": 0.124
2272
+ },
2273
+ {
2274
+ "hidden_sizes": [
2275
+ 128
2276
+ ],
2277
+ "learning_rate": 0.001,
2278
+ "alpha": 0.0001,
2279
+ "accuracy": 0.12200000000000003
2280
+ },
2281
+ {
2282
+ "hidden_sizes": [
2283
+ 128,
2284
+ 64
2285
+ ],
2286
+ "learning_rate": 0.001,
2287
+ "alpha": 0.0001,
2288
+ "accuracy": 0.12200000000000003
2289
+ },
2290
+ {
2291
+ "hidden_sizes": [
2292
+ 128,
2293
+ 64
2294
+ ],
2295
+ "learning_rate": 0.001,
2296
+ "alpha": 0.001,
2297
+ "accuracy": 0.12200000000000003
2298
+ },
2299
+ {
2300
+ "hidden_sizes": [
2301
+ 64
2302
+ ],
2303
+ "learning_rate": 0.01,
2304
+ "alpha": 0.1,
2305
+ "accuracy": 0.122
2306
+ },
2307
+ {
2308
+ "hidden_sizes": [
2309
+ 256,
2310
+ 128
2311
+ ],
2312
+ "learning_rate": 0.001,
2313
+ "alpha": 0.001,
2314
+ "accuracy": 0.122
2315
+ },
2316
+ {
2317
+ "hidden_sizes": [
2318
+ 256,
2319
+ 128
2320
+ ],
2321
+ "learning_rate": 0.001,
2322
+ "alpha": 0.01,
2323
+ "accuracy": 0.122
2324
+ },
2325
+ {
2326
+ "hidden_sizes": [
2327
+ 128,
2328
+ 64
2329
+ ],
2330
+ "learning_rate": 0.001,
2331
+ "alpha": 0.01,
2332
+ "accuracy": 0.12
2333
+ },
2334
+ {
2335
+ "hidden_sizes": [
2336
+ 128,
2337
+ 64
2338
+ ],
2339
+ "learning_rate": 0.001,
2340
+ "alpha": 0.1,
2341
+ "accuracy": 0.12
2342
+ },
2343
+ {
2344
+ "hidden_sizes": [
2345
+ 128
2346
+ ],
2347
+ "learning_rate": 0.01,
2348
+ "alpha": 0.0001,
2349
+ "accuracy": 0.118
2350
+ },
2351
+ {
2352
+ "hidden_sizes": [
2353
+ 128
2354
+ ],
2355
+ "learning_rate": 0.001,
2356
+ "alpha": 0.001,
2357
+ "accuracy": 0.11600000000000002
2358
+ },
2359
+ {
2360
+ "hidden_sizes": [
2361
+ 512
2362
+ ],
2363
+ "learning_rate": 0.01,
2364
+ "alpha": 0.001,
2365
+ "accuracy": 0.11600000000000002
2366
+ },
2367
+ {
2368
+ "hidden_sizes": [
2369
+ 128
2370
+ ],
2371
+ "learning_rate": 0.001,
2372
+ "alpha": 0.1,
2373
+ "accuracy": 0.11400000000000002
2374
+ },
2375
+ {
2376
+ "hidden_sizes": [
2377
+ 128,
2378
+ 64
2379
+ ],
2380
+ "learning_rate": 0.01,
2381
+ "alpha": 0.01,
2382
+ "accuracy": 0.11400000000000002
2383
+ },
2384
+ {
2385
+ "hidden_sizes": [
2386
+ 64
2387
+ ],
2388
+ "learning_rate": 0.01,
2389
+ "alpha": 0.0001,
2390
+ "accuracy": 0.11399999999999999
2391
+ },
2392
+ {
2393
+ "hidden_sizes": [
2394
+ 256
2395
+ ],
2396
+ "learning_rate": 0.01,
2397
+ "alpha": 0.001,
2398
+ "accuracy": 0.11399999999999999
2399
+ },
2400
+ {
2401
+ "hidden_sizes": [
2402
+ 512
2403
+ ],
2404
+ "learning_rate": 0.01,
2405
+ "alpha": 0.0001,
2406
+ "accuracy": 0.11399999999999999
2407
+ },
2408
+ {
2409
+ "hidden_sizes": [
2410
+ 64
2411
+ ],
2412
+ "learning_rate": 0.01,
2413
+ "alpha": 0.01,
2414
+ "accuracy": 0.11200000000000002
2415
+ },
2416
+ {
2417
+ "hidden_sizes": [
2418
+ 256,
2419
+ 128
2420
+ ],
2421
+ "learning_rate": 0.01,
2422
+ "alpha": 0.0001,
2423
+ "accuracy": 0.11200000000000002
2424
+ },
2425
+ {
2426
+ "hidden_sizes": [
2427
+ 256,
2428
+ 128
2429
+ ],
2430
+ "learning_rate": 0.001,
2431
+ "alpha": 0.1,
2432
+ "accuracy": 0.11199999999999999
2433
+ },
2434
+ {
2435
+ "hidden_sizes": [
2436
+ 128
2437
+ ],
2438
+ "learning_rate": 0.01,
2439
+ "alpha": 0.01,
2440
+ "accuracy": 0.11000000000000001
2441
+ },
2442
+ {
2443
+ "hidden_sizes": [
2444
+ 128,
2445
+ 64
2446
+ ],
2447
+ "learning_rate": 0.01,
2448
+ "alpha": 0.0001,
2449
+ "accuracy": 0.11000000000000001
2450
+ },
2451
+ {
2452
+ "hidden_sizes": [
2453
+ 256,
2454
+ 128
2455
+ ],
2456
+ "learning_rate": 0.001,
2457
+ "alpha": 0.0001,
2458
+ "accuracy": 0.10999999999999999
2459
+ },
2460
+ {
2461
+ "hidden_sizes": [
2462
+ 64
2463
+ ],
2464
+ "learning_rate": 0.01,
2465
+ "alpha": 0.001,
2466
+ "accuracy": 0.10600000000000001
2467
+ },
2468
+ {
2469
+ "hidden_sizes": [
2470
+ 128
2471
+ ],
2472
+ "learning_rate": 0.01,
2473
+ "alpha": 0.1,
2474
+ "accuracy": 0.10400000000000001
2475
+ },
2476
+ {
2477
+ "hidden_sizes": [
2478
+ 256,
2479
+ 128
2480
+ ],
2481
+ "learning_rate": 0.01,
2482
+ "alpha": 0.1,
2483
+ "accuracy": 0.10400000000000001
2484
+ },
2485
+ {
2486
+ "hidden_sizes": [
2487
+ 128,
2488
+ 64
2489
+ ],
2490
+ "learning_rate": 0.01,
2491
+ "alpha": 0.1,
2492
+ "accuracy": 0.10200000000000001
2493
+ },
2494
+ {
2495
+ "hidden_sizes": [
2496
+ 256,
2497
+ 128
2498
+ ],
2499
+ "learning_rate": 0.01,
2500
+ "alpha": 0.001,
2501
+ "accuracy": 0.10200000000000001
2502
+ },
2503
+ {
2504
+ "hidden_sizes": [
2505
+ 128,
2506
+ 64
2507
+ ],
2508
+ "learning_rate": 0.01,
2509
+ "alpha": 0.001,
2510
+ "accuracy": 0.096
2511
+ },
2512
+ {
2513
+ "hidden_sizes": [
2514
+ 256
2515
+ ],
2516
+ "learning_rate": 0.0001,
2517
+ "alpha": 0.0001,
2518
+ "accuracy": 0.082
2519
+ },
2520
+ {
2521
+ "hidden_sizes": [
2522
+ 256
2523
+ ],
2524
+ "learning_rate": 0.0001,
2525
+ "alpha": 0.001,
2526
+ "accuracy": 0.082
2527
+ },
2528
+ {
2529
+ "hidden_sizes": [
2530
+ 256
2531
+ ],
2532
+ "learning_rate": 0.0001,
2533
+ "alpha": 0.01,
2534
+ "accuracy": 0.082
2535
+ },
2536
+ {
2537
+ "hidden_sizes": [
2538
+ 512
2539
+ ],
2540
+ "learning_rate": 0.0001,
2541
+ "alpha": 0.0001,
2542
+ "accuracy": 0.08
2543
+ },
2544
+ {
2545
+ "hidden_sizes": [
2546
+ 512
2547
+ ],
2548
+ "learning_rate": 0.0001,
2549
+ "alpha": 0.001,
2550
+ "accuracy": 0.08
2551
+ },
2552
+ {
2553
+ "hidden_sizes": [
2554
+ 256
2555
+ ],
2556
+ "learning_rate": 0.0001,
2557
+ "alpha": 0.1,
2558
+ "accuracy": 0.078
2559
+ },
2560
+ {
2561
+ "hidden_sizes": [
2562
+ 512
2563
+ ],
2564
+ "learning_rate": 0.0001,
2565
+ "alpha": 0.1,
2566
+ "accuracy": 0.078
2567
+ },
2568
+ {
2569
+ "hidden_sizes": [
2570
+ 512
2571
+ ],
2572
+ "learning_rate": 0.0001,
2573
+ "alpha": 0.01,
2574
+ "accuracy": 0.076
2575
+ },
2576
+ {
2577
+ "hidden_sizes": [
2578
+ 256,
2579
+ 128
2580
+ ],
2581
+ "learning_rate": 0.0001,
2582
+ "alpha": 0.001,
2583
+ "accuracy": 0.062
2584
+ },
2585
+ {
2586
+ "hidden_sizes": [
2587
+ 256,
2588
+ 128
2589
+ ],
2590
+ "learning_rate": 0.0001,
2591
+ "alpha": 0.1,
2592
+ "accuracy": 0.062
2593
+ },
2594
+ {
2595
+ "hidden_sizes": [
2596
+ 256,
2597
+ 128
2598
+ ],
2599
+ "learning_rate": 0.0001,
2600
+ "alpha": 0.0001,
2601
+ "accuracy": 0.06
2602
+ },
2603
+ {
2604
+ "hidden_sizes": [
2605
+ 256,
2606
+ 128
2607
+ ],
2608
+ "learning_rate": 0.0001,
2609
+ "alpha": 0.01,
2610
+ "accuracy": 0.06
2611
+ },
2612
+ {
2613
+ "hidden_sizes": [
2614
+ 64
2615
+ ],
2616
+ "learning_rate": 0.0001,
2617
+ "alpha": 0.0001,
2618
+ "accuracy": 0.05600000000000001
2619
+ },
2620
+ {
2621
+ "hidden_sizes": [
2622
+ 64
2623
+ ],
2624
+ "learning_rate": 0.0001,
2625
+ "alpha": 0.001,
2626
+ "accuracy": 0.05600000000000001
2627
+ },
2628
+ {
2629
+ "hidden_sizes": [
2630
+ 64
2631
+ ],
2632
+ "learning_rate": 0.0001,
2633
+ "alpha": 0.01,
2634
+ "accuracy": 0.05600000000000001
2635
+ },
2636
+ {
2637
+ "hidden_sizes": [
2638
+ 64
2639
+ ],
2640
+ "learning_rate": 0.0001,
2641
+ "alpha": 0.1,
2642
+ "accuracy": 0.05600000000000001
2643
+ },
2644
+ {
2645
+ "hidden_sizes": [
2646
+ 128,
2647
+ 64
2648
+ ],
2649
+ "learning_rate": 0.0001,
2650
+ "alpha": 0.001,
2651
+ "accuracy": 0.046
2652
+ },
2653
+ {
2654
+ "hidden_sizes": [
2655
+ 128,
2656
+ 64
2657
+ ],
2658
+ "learning_rate": 0.0001,
2659
+ "alpha": 0.0001,
2660
+ "accuracy": 0.044000000000000004
2661
+ },
2662
+ {
2663
+ "hidden_sizes": [
2664
+ 128,
2665
+ 64
2666
+ ],
2667
+ "learning_rate": 0.0001,
2668
+ "alpha": 0.01,
2669
+ "accuracy": 0.044000000000000004
2670
+ },
2671
+ {
2672
+ "hidden_sizes": [
2673
+ 128,
2674
+ 64
2675
+ ],
2676
+ "learning_rate": 0.0001,
2677
+ "alpha": 0.1,
2678
+ "accuracy": 0.044000000000000004
2679
+ },
2680
+ {
2681
+ "hidden_sizes": [
2682
+ 128
2683
+ ],
2684
+ "learning_rate": 0.0001,
2685
+ "alpha": 0.0001,
2686
+ "accuracy": 0.038
2687
+ },
2688
+ {
2689
+ "hidden_sizes": [
2690
+ 128
2691
+ ],
2692
+ "learning_rate": 0.0001,
2693
+ "alpha": 0.001,
2694
+ "accuracy": 0.038
2695
+ },
2696
+ {
2697
+ "hidden_sizes": [
2698
+ 128
2699
+ ],
2700
+ "learning_rate": 0.0001,
2701
+ "alpha": 0.01,
2702
+ "accuracy": 0.038
2703
+ },
2704
+ {
2705
+ "hidden_sizes": [
2706
+ 128
2707
+ ],
2708
+ "learning_rate": 0.0001,
2709
+ "alpha": 0.1,
2710
+ "accuracy": 0.038
2711
+ }
2712
+ ]
2713
+ },
2714
+ {
2715
+ "model": "m5",
2716
+ "layer": 12,
2717
+ "position": 2,
2718
+ "n_samples": 500,
2719
+ "n_classes": 38,
2720
+ "linear_accuracy": 0.21999999999999997,
2721
+ "best_mlp_accuracy": 0.29600000000000004,
2722
+ "mlp_advantage_pp": 7.6,
2723
+ "best_config": {
2724
+ "hidden_sizes": [
2725
+ 256
2726
+ ],
2727
+ "learning_rate": 0.001,
2728
+ "alpha": 0.0001,
2729
+ "accuracy": 0.29600000000000004
2730
+ },
2731
+ "top_5_configs": [
2732
+ {
2733
+ "hidden_sizes": [
2734
+ 256
2735
+ ],
2736
+ "learning_rate": 0.001,
2737
+ "alpha": 0.0001,
2738
+ "accuracy": 0.29600000000000004
2739
+ },
2740
+ {
2741
+ "hidden_sizes": [
2742
+ 256
2743
+ ],
2744
+ "learning_rate": 0.001,
2745
+ "alpha": 0.001,
2746
+ "accuracy": 0.29400000000000004
2747
+ },
2748
+ {
2749
+ "hidden_sizes": [
2750
+ 256
2751
+ ],
2752
+ "learning_rate": 0.001,
2753
+ "alpha": 0.01,
2754
+ "accuracy": 0.29200000000000004
2755
+ },
2756
+ {
2757
+ "hidden_sizes": [
2758
+ 256
2759
+ ],
2760
+ "learning_rate": 0.001,
2761
+ "alpha": 0.1,
2762
+ "accuracy": 0.29200000000000004
2763
+ },
2764
+ {
2765
+ "hidden_sizes": [
2766
+ 256
2767
+ ],
2768
+ "learning_rate": 0.01,
2769
+ "alpha": 0.1,
2770
+ "accuracy": 0.28
2771
+ }
2772
+ ],
2773
+ "all_configs": [
2774
+ {
2775
+ "hidden_sizes": [
2776
+ 256
2777
+ ],
2778
+ "learning_rate": 0.001,
2779
+ "alpha": 0.0001,
2780
+ "accuracy": 0.29600000000000004
2781
+ },
2782
+ {
2783
+ "hidden_sizes": [
2784
+ 256
2785
+ ],
2786
+ "learning_rate": 0.001,
2787
+ "alpha": 0.001,
2788
+ "accuracy": 0.29400000000000004
2789
+ },
2790
+ {
2791
+ "hidden_sizes": [
2792
+ 256
2793
+ ],
2794
+ "learning_rate": 0.001,
2795
+ "alpha": 0.01,
2796
+ "accuracy": 0.29200000000000004
2797
+ },
2798
+ {
2799
+ "hidden_sizes": [
2800
+ 256
2801
+ ],
2802
+ "learning_rate": 0.001,
2803
+ "alpha": 0.1,
2804
+ "accuracy": 0.29200000000000004
2805
+ },
2806
+ {
2807
+ "hidden_sizes": [
2808
+ 256
2809
+ ],
2810
+ "learning_rate": 0.01,
2811
+ "alpha": 0.1,
2812
+ "accuracy": 0.28
2813
+ },
2814
+ {
2815
+ "hidden_sizes": [
2816
+ 512
2817
+ ],
2818
+ "learning_rate": 0.001,
2819
+ "alpha": 0.001,
2820
+ "accuracy": 0.274
2821
+ },
2822
+ {
2823
+ "hidden_sizes": [
2824
+ 128
2825
+ ],
2826
+ "learning_rate": 0.01,
2827
+ "alpha": 0.1,
2828
+ "accuracy": 0.272
2829
+ },
2830
+ {
2831
+ "hidden_sizes": [
2832
+ 128
2833
+ ],
2834
+ "learning_rate": 0.001,
2835
+ "alpha": 0.0001,
2836
+ "accuracy": 0.27
2837
+ },
2838
+ {
2839
+ "hidden_sizes": [
2840
+ 512
2841
+ ],
2842
+ "learning_rate": 0.001,
2843
+ "alpha": 0.1,
2844
+ "accuracy": 0.27
2845
+ },
2846
+ {
2847
+ "hidden_sizes": [
2848
+ 128
2849
+ ],
2850
+ "learning_rate": 0.001,
2851
+ "alpha": 0.001,
2852
+ "accuracy": 0.268
2853
+ },
2854
+ {
2855
+ "hidden_sizes": [
2856
+ 128
2857
+ ],
2858
+ "learning_rate": 0.001,
2859
+ "alpha": 0.01,
2860
+ "accuracy": 0.268
2861
+ },
2862
+ {
2863
+ "hidden_sizes": [
2864
+ 256,
2865
+ 128
2866
+ ],
2867
+ "learning_rate": 0.001,
2868
+ "alpha": 0.1,
2869
+ "accuracy": 0.268
2870
+ },
2871
+ {
2872
+ "hidden_sizes": [
2873
+ 512
2874
+ ],
2875
+ "learning_rate": 0.001,
2876
+ "alpha": 0.0001,
2877
+ "accuracy": 0.266
2878
+ },
2879
+ {
2880
+ "hidden_sizes": [
2881
+ 256,
2882
+ 128
2883
+ ],
2884
+ "learning_rate": 0.001,
2885
+ "alpha": 0.01,
2886
+ "accuracy": 0.264
2887
+ },
2888
+ {
2889
+ "hidden_sizes": [
2890
+ 512
2891
+ ],
2892
+ "learning_rate": 0.001,
2893
+ "alpha": 0.01,
2894
+ "accuracy": 0.262
2895
+ },
2896
+ {
2897
+ "hidden_sizes": [
2898
+ 256,
2899
+ 128
2900
+ ],
2901
+ "learning_rate": 0.001,
2902
+ "alpha": 0.0001,
2903
+ "accuracy": 0.262
2904
+ },
2905
+ {
2906
+ "hidden_sizes": [
2907
+ 256,
2908
+ 128
2909
+ ],
2910
+ "learning_rate": 0.001,
2911
+ "alpha": 0.001,
2912
+ "accuracy": 0.26
2913
+ },
2914
+ {
2915
+ "hidden_sizes": [
2916
+ 128
2917
+ ],
2918
+ "learning_rate": 0.001,
2919
+ "alpha": 0.1,
2920
+ "accuracy": 0.258
2921
+ },
2922
+ {
2923
+ "hidden_sizes": [
2924
+ 512
2925
+ ],
2926
+ "learning_rate": 0.0001,
2927
+ "alpha": 0.0001,
2928
+ "accuracy": 0.256
2929
+ },
2930
+ {
2931
+ "hidden_sizes": [
2932
+ 512
2933
+ ],
2934
+ "learning_rate": 0.0001,
2935
+ "alpha": 0.001,
2936
+ "accuracy": 0.256
2937
+ },
2938
+ {
2939
+ "hidden_sizes": [
2940
+ 512
2941
+ ],
2942
+ "learning_rate": 0.0001,
2943
+ "alpha": 0.01,
2944
+ "accuracy": 0.256
2945
+ },
2946
+ {
2947
+ "hidden_sizes": [
2948
+ 512
2949
+ ],
2950
+ "learning_rate": 0.0001,
2951
+ "alpha": 0.1,
2952
+ "accuracy": 0.256
2953
+ },
2954
+ {
2955
+ "hidden_sizes": [
2956
+ 64
2957
+ ],
2958
+ "learning_rate": 0.01,
2959
+ "alpha": 0.1,
2960
+ "accuracy": 0.244
2961
+ },
2962
+ {
2963
+ "hidden_sizes": [
2964
+ 64
2965
+ ],
2966
+ "learning_rate": 0.001,
2967
+ "alpha": 0.0001,
2968
+ "accuracy": 0.242
2969
+ },
2970
+ {
2971
+ "hidden_sizes": [
2972
+ 64
2973
+ ],
2974
+ "learning_rate": 0.001,
2975
+ "alpha": 0.1,
2976
+ "accuracy": 0.242
2977
+ },
2978
+ {
2979
+ "hidden_sizes": [
2980
+ 128
2981
+ ],
2982
+ "learning_rate": 0.01,
2983
+ "alpha": 0.0001,
2984
+ "accuracy": 0.242
2985
+ },
2986
+ {
2987
+ "hidden_sizes": [
2988
+ 64
2989
+ ],
2990
+ "learning_rate": 0.001,
2991
+ "alpha": 0.001,
2992
+ "accuracy": 0.24
2993
+ },
2994
+ {
2995
+ "hidden_sizes": [
2996
+ 128
2997
+ ],
2998
+ "learning_rate": 0.01,
2999
+ "alpha": 0.001,
3000
+ "accuracy": 0.24
3001
+ },
3002
+ {
3003
+ "hidden_sizes": [
3004
+ 256,
3005
+ 128
3006
+ ],
3007
+ "learning_rate": 0.01,
3008
+ "alpha": 0.1,
3009
+ "accuracy": 0.24
3010
+ },
3011
+ {
3012
+ "hidden_sizes": [
3013
+ 64
3014
+ ],
3015
+ "learning_rate": 0.001,
3016
+ "alpha": 0.01,
3017
+ "accuracy": 0.23399999999999999
3018
+ },
3019
+ {
3020
+ "hidden_sizes": [
3021
+ 256
3022
+ ],
3023
+ "learning_rate": 0.01,
3024
+ "alpha": 0.0001,
3025
+ "accuracy": 0.23399999999999999
3026
+ },
3027
+ {
3028
+ "hidden_sizes": [
3029
+ 64
3030
+ ],
3031
+ "learning_rate": 0.01,
3032
+ "alpha": 0.0001,
3033
+ "accuracy": 0.23200000000000004
3034
+ },
3035
+ {
3036
+ "hidden_sizes": [
3037
+ 64
3038
+ ],
3039
+ "learning_rate": 0.01,
3040
+ "alpha": 0.001,
3041
+ "accuracy": 0.23200000000000004
3042
+ },
3043
+ {
3044
+ "hidden_sizes": [
3045
+ 128,
3046
+ 64
3047
+ ],
3048
+ "learning_rate": 0.01,
3049
+ "alpha": 0.1,
3050
+ "accuracy": 0.23199999999999998
3051
+ },
3052
+ {
3053
+ "hidden_sizes": [
3054
+ 128,
3055
+ 64
3056
+ ],
3057
+ "learning_rate": 0.001,
3058
+ "alpha": 0.1,
3059
+ "accuracy": 0.22999999999999998
3060
+ },
3061
+ {
3062
+ "hidden_sizes": [
3063
+ 128,
3064
+ 64
3065
+ ],
3066
+ "learning_rate": 0.001,
3067
+ "alpha": 0.0001,
3068
+ "accuracy": 0.22800000000000004
3069
+ },
3070
+ {
3071
+ "hidden_sizes": [
3072
+ 256
3073
+ ],
3074
+ "learning_rate": 0.01,
3075
+ "alpha": 0.01,
3076
+ "accuracy": 0.22399999999999998
3077
+ },
3078
+ {
3079
+ "hidden_sizes": [
3080
+ 256
3081
+ ],
3082
+ "learning_rate": 0.01,
3083
+ "alpha": 0.001,
3084
+ "accuracy": 0.22200000000000003
3085
+ },
3086
+ {
3087
+ "hidden_sizes": [
3088
+ 64
3089
+ ],
3090
+ "learning_rate": 0.01,
3091
+ "alpha": 0.01,
3092
+ "accuracy": 0.21800000000000003
3093
+ },
3094
+ {
3095
+ "hidden_sizes": [
3096
+ 128,
3097
+ 64
3098
+ ],
3099
+ "learning_rate": 0.001,
3100
+ "alpha": 0.01,
3101
+ "accuracy": 0.21800000000000003
3102
+ },
3103
+ {
3104
+ "hidden_sizes": [
3105
+ 512
3106
+ ],
3107
+ "learning_rate": 0.01,
3108
+ "alpha": 0.1,
3109
+ "accuracy": 0.21799999999999997
3110
+ },
3111
+ {
3112
+ "hidden_sizes": [
3113
+ 128,
3114
+ 64
3115
+ ],
3116
+ "learning_rate": 0.001,
3117
+ "alpha": 0.001,
3118
+ "accuracy": 0.21600000000000003
3119
+ },
3120
+ {
3121
+ "hidden_sizes": [
3122
+ 128
3123
+ ],
3124
+ "learning_rate": 0.01,
3125
+ "alpha": 0.01,
3126
+ "accuracy": 0.21400000000000002
3127
+ },
3128
+ {
3129
+ "hidden_sizes": [
3130
+ 128,
3131
+ 64
3132
+ ],
3133
+ "learning_rate": 0.01,
3134
+ "alpha": 0.01,
3135
+ "accuracy": 0.21400000000000002
3136
+ },
3137
+ {
3138
+ "hidden_sizes": [
3139
+ 128,
3140
+ 64
3141
+ ],
3142
+ "learning_rate": 0.01,
3143
+ "alpha": 0.001,
3144
+ "accuracy": 0.21200000000000002
3145
+ },
3146
+ {
3147
+ "hidden_sizes": [
3148
+ 256,
3149
+ 128
3150
+ ],
3151
+ "learning_rate": 0.01,
3152
+ "alpha": 0.001,
3153
+ "accuracy": 0.20800000000000002
3154
+ },
3155
+ {
3156
+ "hidden_sizes": [
3157
+ 512
3158
+ ],
3159
+ "learning_rate": 0.01,
3160
+ "alpha": 0.0001,
3161
+ "accuracy": 0.20400000000000001
3162
+ },
3163
+ {
3164
+ "hidden_sizes": [
3165
+ 128,
3166
+ 64
3167
+ ],
3168
+ "learning_rate": 0.01,
3169
+ "alpha": 0.0001,
3170
+ "accuracy": 0.202
3171
+ },
3172
+ {
3173
+ "hidden_sizes": [
3174
+ 256,
3175
+ 128
3176
+ ],
3177
+ "learning_rate": 0.0001,
3178
+ "alpha": 0.001,
3179
+ "accuracy": 0.2
3180
+ },
3181
+ {
3182
+ "hidden_sizes": [
3183
+ 256,
3184
+ 128
3185
+ ],
3186
+ "learning_rate": 0.0001,
3187
+ "alpha": 0.1,
3188
+ "accuracy": 0.198
3189
+ },
3190
+ {
3191
+ "hidden_sizes": [
3192
+ 256,
3193
+ 128
3194
+ ],
3195
+ "learning_rate": 0.01,
3196
+ "alpha": 0.01,
3197
+ "accuracy": 0.198
3198
+ },
3199
+ {
3200
+ "hidden_sizes": [
3201
+ 512
3202
+ ],
3203
+ "learning_rate": 0.01,
3204
+ "alpha": 0.001,
3205
+ "accuracy": 0.194
3206
+ },
3207
+ {
3208
+ "hidden_sizes": [
3209
+ 256,
3210
+ 128
3211
+ ],
3212
+ "learning_rate": 0.0001,
3213
+ "alpha": 0.01,
3214
+ "accuracy": 0.192
3215
+ },
3216
+ {
3217
+ "hidden_sizes": [
3218
+ 256,
3219
+ 128
3220
+ ],
3221
+ "learning_rate": 0.01,
3222
+ "alpha": 0.0001,
3223
+ "accuracy": 0.192
3224
+ },
3225
+ {
3226
+ "hidden_sizes": [
3227
+ 256,
3228
+ 128
3229
+ ],
3230
+ "learning_rate": 0.0001,
3231
+ "alpha": 0.0001,
3232
+ "accuracy": 0.188
3233
+ },
3234
+ {
3235
+ "hidden_sizes": [
3236
+ 512
3237
+ ],
3238
+ "learning_rate": 0.01,
3239
+ "alpha": 0.01,
3240
+ "accuracy": 0.186
3241
+ },
3242
+ {
3243
+ "hidden_sizes": [
3244
+ 256
3245
+ ],
3246
+ "learning_rate": 0.0001,
3247
+ "alpha": 0.01,
3248
+ "accuracy": 0.148
3249
+ },
3250
+ {
3251
+ "hidden_sizes": [
3252
+ 256
3253
+ ],
3254
+ "learning_rate": 0.0001,
3255
+ "alpha": 0.1,
3256
+ "accuracy": 0.148
3257
+ },
3258
+ {
3259
+ "hidden_sizes": [
3260
+ 256
3261
+ ],
3262
+ "learning_rate": 0.0001,
3263
+ "alpha": 0.0001,
3264
+ "accuracy": 0.146
3265
+ },
3266
+ {
3267
+ "hidden_sizes": [
3268
+ 256
3269
+ ],
3270
+ "learning_rate": 0.0001,
3271
+ "alpha": 0.001,
3272
+ "accuracy": 0.146
3273
+ },
3274
+ {
3275
+ "hidden_sizes": [
3276
+ 128
3277
+ ],
3278
+ "learning_rate": 0.0001,
3279
+ "alpha": 0.0001,
3280
+ "accuracy": 0.10800000000000001
3281
+ },
3282
+ {
3283
+ "hidden_sizes": [
3284
+ 128
3285
+ ],
3286
+ "learning_rate": 0.0001,
3287
+ "alpha": 0.001,
3288
+ "accuracy": 0.10800000000000001
3289
+ },
3290
+ {
3291
+ "hidden_sizes": [
3292
+ 128
3293
+ ],
3294
+ "learning_rate": 0.0001,
3295
+ "alpha": 0.01,
3296
+ "accuracy": 0.10800000000000001
3297
+ },
3298
+ {
3299
+ "hidden_sizes": [
3300
+ 128
3301
+ ],
3302
+ "learning_rate": 0.0001,
3303
+ "alpha": 0.1,
3304
+ "accuracy": 0.10800000000000001
3305
+ },
3306
+ {
3307
+ "hidden_sizes": [
3308
+ 128,
3309
+ 64
3310
+ ],
3311
+ "learning_rate": 0.0001,
3312
+ "alpha": 0.0001,
3313
+ "accuracy": 0.05800000000000001
3314
+ },
3315
+ {
3316
+ "hidden_sizes": [
3317
+ 128,
3318
+ 64
3319
+ ],
3320
+ "learning_rate": 0.0001,
3321
+ "alpha": 0.001,
3322
+ "accuracy": 0.05800000000000001
3323
+ },
3324
+ {
3325
+ "hidden_sizes": [
3326
+ 128,
3327
+ 64
3328
+ ],
3329
+ "learning_rate": 0.0001,
3330
+ "alpha": 0.1,
3331
+ "accuracy": 0.04
3332
+ },
3333
+ {
3334
+ "hidden_sizes": [
3335
+ 128,
3336
+ 64
3337
+ ],
3338
+ "learning_rate": 0.0001,
3339
+ "alpha": 0.01,
3340
+ "accuracy": 0.038
3341
+ },
3342
+ {
3343
+ "hidden_sizes": [
3344
+ 64
3345
+ ],
3346
+ "learning_rate": 0.0001,
3347
+ "alpha": 0.0001,
3348
+ "accuracy": 0.034
3349
+ },
3350
+ {
3351
+ "hidden_sizes": [
3352
+ 64
3353
+ ],
3354
+ "learning_rate": 0.0001,
3355
+ "alpha": 0.001,
3356
+ "accuracy": 0.034
3357
+ },
3358
+ {
3359
+ "hidden_sizes": [
3360
+ 64
3361
+ ],
3362
+ "learning_rate": 0.0001,
3363
+ "alpha": 0.01,
3364
+ "accuracy": 0.034
3365
+ },
3366
+ {
3367
+ "hidden_sizes": [
3368
+ 64
3369
+ ],
3370
+ "learning_rate": 0.0001,
3371
+ "alpha": 0.1,
3372
+ "accuracy": 0.034
3373
+ }
3374
+ ]
3375
+ }
3376
+ ],
3377
+ "conclusion": "mlp_advantage"
3378
+ }
experiments/ood/detailed_outputs.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01fd4b74e51e8a32b98441d18a46884b00a3c3a3b33f13ba6cdae88627bf1271
3
+ size 13945991
experiments/ood/log.txt ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ======================================================================
2
+ OOD GENERALIZATION EXPERIMENT
3
+ ======================================================================
4
+ Checkpoint dir: /lambda/nfs/experiment/results/v9_meta_fork
5
+ Data dir: data/
6
+ Output dir: /lambda/nfs/experiment/results/v9_meta_fork/experiments/ood/
7
+ Seed: 0
8
+ Num thoughts: 6
9
+
10
+ Loading test sets...
11
+ prosqa_test: 500 samples
12
+ ood_7hop: 1000 samples
13
+ ood_8hop: 1000 samples
14
+ ood_dag: 1000 samples
15
+ ood_dense: 1000 samples
16
+
17
+ ============================================================
18
+ Loading model: m1
19
+ ============================================================
20
+ [load_model] Plain GPT-2 loaded from /lambda/nfs/experiment/results/v9_meta_fork/prosqa-cot/checkpoint_50
21
+ Missing keys: []
22
+ Unexpected keys: []
23
+ Type: cot, feedback_mode: None
24
+
25
+ Evaluating m1 on prosqa_test (500 samples)...
26
+ The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
27
+ Sample 0: GT='Sally is a sterpus.' Pred='Sally is a sterpus.' Correct=True
28
+ Sample 1: GT='Bob is a shumpus.' Pred='Bob is a zhorpus.' Correct=False
29
+ Sample 2: GT='Max is a boompus.' Pred='Max is a boompus.' Correct=True
30
+ Progress: 100/500 acc=0.8000
31
+ Progress: 200/500 acc=0.8300
32
+ Progress: 300/500 acc=0.8167
33
+ Progress: 400/500 acc=0.8225
34
+ Progress: 500/500 acc=0.8300
35
+ m1 / prosqa_test: accuracy=0.8300 (303.2s)
36
+
37
+ Evaluating m1 on ood_7hop (1000 samples)...
38
+ Sample 0: GT='Davis is a boompus.' Pred='Davis is a lempus.' Correct=False
39
+ Sample 1: GT='Sally is a yerpus.' Pred='Sally is a zumpus.' Correct=False
40
+ Sample 2: GT='Wren is a yumpus.' Pred='Every boompus is a gerpus. Every scrompus is a quimpus. Every quimpus is a timpus. Every numpus is a yimpus. Every brimpus is a timpus. Every worpus is a brimpus. Every yimpus is a brimpus. Every yimpus is a timpus. Every quimpus is a kerpus. Wren is a numpus. Every rempus is a worpus. Every boompus is a storpus. Every rempus is a numpus. Every boompus is a timpus. Every yumpus is a timpus. Every worpus is a boompus. Every kerpus is a storpus. Wren is a storpus. Wren is a yimpus. Every gerpus is a storpus. Every boompus is a numpus. Every boompus is a yimpus. Every gerpus is a scrompus. Every kerpus is a yumpus. Every quimpus is a numpus. Every worpus is a storpus. Every kerpus is a yimpus. Every timpus is a brimpus. Every rempus is a storpus. Is Wren a yumpus or a brimpus?
41
+ Wren is a yimpus.
42
+ Every yimpus is a brimpus.
43
+ Every brimpus is a timpus.
44
+ Every timpus is a brimpus.
45
+ Every brimpus is a timpus.
46
+ Every timpus is a brimpus.
47
+ Every brimpus is a timpus.
48
+ Every timpus is a brimpus.
49
+ Every brimpus is a timpus.
50
+ Every timpus is a brimpus.
51
+ Every brimpus is a timpus.
52
+ Every timpus is a brimpus' Correct=False
53
+ Progress: 100/1000 acc=0.1700
54
+ Progress: 200/1000 acc=0.1350
55
+ Progress: 300/1000 acc=0.1033
56
+ Progress: 400/1000 acc=0.1150
57
+ Progress: 500/1000 acc=0.1100
58
+ Progress: 600/1000 acc=0.1050
59
+ Progress: 700/1000 acc=0.1057
60
+ Progress: 800/1000 acc=0.1100
61
+ Progress: 900/1000 acc=0.1056
62
+ Progress: 1000/1000 acc=0.1070
63
+ m1 / ood_7hop: accuracy=0.1070 (969.6s)
64
+
65
+ Evaluating m1 on ood_8hop (1000 samples)...
66
+ Sample 0: GT='Sally is a boompus.' Pred='Every rempus is a scrompus. Every grimpus is a scrompus. Every yumpus is a felpus. Every fompus is a brimpus. Every jompus is a rompus. Every tumpus is a hilpus. Every tumpus is a jompus. Every quimpus is a felpus. Every felpus is a yimpus. Every zhorpus is a hilpus. Every yimpus is a grimpus. Every wumpus is a shumpus. Every gorpus is a fompus. Every brimpus is a yumpus. Every zhorpus is a yimpus. Every shumpus is a grimpus. Every yimpus is a shumpus. Every yumpus is a wumpus. Sally is a felpus. Every grimpus is a bompus. Every rompus is a quimpus. Every quimpus is a scrompus. Every quimpus is a boompus. Every bompus is a hilpus. Every quimpus is a shumpus. Every yumpus is a tumpus. Is Sally a boompus or a rempus?
67
+ Sally is a felpus.
68
+ Every felpus is a yimpus.
69
+ Every yimpus is a grimpus.
70
+ Every grimpus is a bompus.
71
+ Every bompus is a yimpus.
72
+ Every yimpus is a shumpus.
73
+ Every shumpus is a grimpus.
74
+ Every grimpus is a bompus.
75
+ Every bompus is a yimpus.
76
+ Every yimpus is a shumpus.
77
+ Every shumpus is a grimpus.
78
+ Every grimpus is a bompus' Correct=False
79
+ Sample 1: GT='Bob is a lorpus.' Pred='Bob is a sterpus. Every impus is a chorpus. Every numpus is a brimpus. Every vumpus is a wumpus. Every zumpus is a sterpus. Every numpus is a gorpus. Every gorpus is a sterpus. Every zumpus is a gorpus. Every zumpus is a lorpus. Every yumpus is a chorpus. Every yumpus is a grimpus. Every hilpus is a vumpus. Bob is a brimpus. Every sterpus is a grimpus. Every brimpus is a gorpus. Every impus is a sterpus. Every lorpus is a chorpus. Every vumpus is a yumpus. Every rorpus is a gorpus. Every rorpus is a hilpus. Every brimpus is a chorpus. Every tumpus is a numpus. Every numpus is a zumpus. Every numpus is a wumpus. Every rorpus is a chorpus. Every wumpus is a sterpus. Every yumpus is a impus. Every grimpus is a brimpus. Bob is a chorpus. Every vumpus is a grimpus. Every rorpus is a grimpus. Every vumpus is a brimpus. Every impus is a tumpus. Is Bob a lorpus or a grimpus?
80
+ Bob is a brimpus.
81
+ Every brimpus is a gorpus.
82
+ Every gorpus is a sterpus.
83
+ Every sterpus is a grimpus.
84
+ Every grimpus is a brimpus.
85
+ Every brimpus is a gorpus.
86
+ Every gorpus is a sterpus.
87
+ Every sterpus is a grimpus.
88
+ Every grimpus is a brimpus.
89
+ Every brimpus is a gorpus.
90
+ Every gorpus is a sterpus.
91
+ Every sterpus is a grimpus.' Correct=False
92
+ Sample 2: GT='Bob is a kerpus.' Pred='Every dumpus is a jompus. Every dumpus is a gwompus. Every jompus is a rompus. Every storpus is a kerpus. Every rempus is a brimpus. Every gwompus is a yerpus. Every gorpus is a gwompus. Every kerpus is a quimpus. Every gorpus is a rompus. Every timpus is a rompus. Every hilpus is a jompus. Every timpus is a storpus. Every hilpus is a gorpus. Every sterpus is a hilpus. Every quimpus is a yerpus. Every timpus is a gwompus. Every gorpus is a rempus. Every brimpus is a dumpus. Every quimpus is a gwompus. Bob is a yerpus. Every dumpus is a yerpus. Every yerpus is a quimpus. Every dumpus is a timpus. Is Bob a kerpus or a yerpus?
93
+ Bob is a yerpus.
94
+ Every yerpus is a quimpus.
95
+ Every quimpus is a gwompus.
96
+ Every gwompus is a yerpus.
97
+ Every yerpus is a quimpus.
98
+ Every quimpus is a gwompus.
99
+ Every gwompus is a yerpus.
100
+ Every yerpus is a quimpus.
101
+ Every quimpus is a gwompus.
102
+ Every gwompus is a yerpus.
103
+ Every yerpus is a qu' Correct=False
104
+ Progress: 100/1000 acc=0.0600
105
+ Progress: 200/1000 acc=0.0700
106
+ Progress: 300/1000 acc=0.0633
107
+ Progress: 400/1000 acc=0.0600
108
+ Progress: 500/1000 acc=0.0680
109
+ Progress: 600/1000 acc=0.0750
110
+ Progress: 700/1000 acc=0.0771
111
+ Progress: 800/1000 acc=0.0750
112
+ Progress: 900/1000 acc=0.0789
113
+ Progress: 1000/1000 acc=0.0820
114
+ m1 / ood_8hop: accuracy=0.0820 (955.1s)
115
+
116
+ Evaluating m1 on ood_dag (1000 samples)...
117
+ Sample 0: GT='Stella is a vumpus.' Pred='Stella is a vumpus.' Correct=True
118
+ Sample 1: GT='Sam is a yimpus.' Pred='Every hilpus is a lorpus. Every storpus is a yimpus. Every jelpus is a timpus. Every timpus is a yimpus. Every lorpus is a timpus. Every timpus is a bompus. Every timpus is a fompus. Every lempus is a hilpus. Every lempus is a jelpus. Every storpus is a gwompus. Sam is a gerpus. Every lempus is a rompus. Every rompus is a lorpus. Every chorpus is a bompus. Every dumpus is a quimpus. Every lempus is a storpus. Is Sam a yimpus or a dumpus?
119
+ Sam is a gerpus.
120
+ Every gerpus is a gerpus.
121
+ Every gerpus is a gerpus.
122
+ Every gerpus is a gerpus.
123
+ Every gerpus is a gerpus.
124
+ Every gerpus is a gerpus.
125
+ Every gerpus is a gerpus.
126
+ Every gerpus is a gerpus.
127
+ Every gerpus is a gerpus.
128
+ Every gerpus is a gerpus.
129
+ Every gerpus is a gerpus.
130
+ Every gerpus is a gerpus.' Correct=False
131
+ Sample 2: GT='Wren is a zhorpus.' Pred='Every chorpus is a zhorpus. Every hilpus is a worpus. Every gwompus is a yerpus. Every yimpus is a quimpus. Every hilpus is a yimpus. Every yerpus is a bompus. Every tumpus is a grimpus. Every quimpus is a yimpus. Every worpus is a tumpus. Every bompus is a chorpus. Every gorpus is a bompus. Every scrompus is a sterpus. Every hilpus is a zhorpus. Every gwompus is a gorpus. Every gwompus is a scrompus. Every jelpus is a yimpus. Every yimpus is a jelpus. Every quimpus is a jelpus. Every scrompus is a chorpus. Every gwompus is a hilpus. Is Wren a sterpus or a zhorpus?
132
+ Wren is a yimpus.
133
+ Every yimpus is a quimpus.
134
+ Every quimpus is a yimpus.
135
+ Every yimpus is a jelpus.
136
+ Every jelpus is a yimpus.
137
+ Every yimpus is a jelpus.
138
+ Every jelpus is a yimpus.
139
+ Every yimpus is a jelpus.
140
+ Every jelpus is a yimpus.
141
+ Every yimpus is a jelpus.
142
+ Every jelpus is a yimpus.
143
+ Every' Correct=False
144
+ Progress: 100/1000 acc=0.3200
145
+ Progress: 200/1000 acc=0.2950
146
+ Progress: 300/1000 acc=0.2900
147
+ Progress: 400/1000 acc=0.2775
148
+ Progress: 500/1000 acc=0.2720
149
+ Progress: 600/1000 acc=0.2750
150
+ Progress: 700/1000 acc=0.2814
151
+ Progress: 800/1000 acc=0.2787
152
+ Progress: 900/1000 acc=0.2889
153
+ Progress: 1000/1000 acc=0.2820
154
+ m1 / ood_dag: accuracy=0.2820 (736.6s)
155
+
156
+ Evaluating m1 on ood_dense (1000 samples)...
157
+ Sample 0: GT='Sally is a yerpus.' Pred='Sally is a yerpus.' Correct=True
158
+ Sample 1: GT='Fae is a rompus.' Pred='Every hilpus is a lempus. Every chorpus is a hilpus. Every zhorpus is a numpus. Every zhorpus is a kerpus. Every lempus is a kerpus. Every zhorpus is a shumpus. Every rompus is a lempus. Every chorpus is a fompus. Every kerpus is a yerpus. Every brimpus is a rompus. Every bompus is a brimpus. Every chorpus is a dumpus. Every numpus is a yerpus. Every numpus is a fompus. Every fompus is a dumpus. Every rompus is a worpus. Every yerpus is a kerpus. Every worpus is a kerpus. Every rompus is a felpus. Every kerpus is a gerpus. Every bompus is a hilpus. Every numpus is a gerpus. Every rompus is a sterpus. Every yerpus is a lorpus. Every numpus is a lorpus. Every shumpus is a felpus. Every brimpus is a chorpus. Every numpus is a bompus. Is Fae a lempus or a rompus?
159
+ Fae is a bompus.
160
+ Every bompus is a brimpus.
161
+ Every brimpus is a rompus.
162
+ Every rompus is a lempus.
163
+ Every lempus is a kerpus.
164
+ Every kerpus is a yerpus.
165
+ Every yerpus is a kerpus.
166
+ Every kerpus is a yerpus.
167
+ Every yerpus is a kerpus.
168
+ Every kerpus is a yerpus.
169
+ Every yerpus is a kerpus.
170
+ Every kerp' Correct=False
171
+ Sample 2: GT='Carol is a quimpus.' Pred='Carol is a chorpus.' Correct=False
172
+ Progress: 100/1000 acc=0.1900
173
+ Progress: 200/1000 acc=0.1950
174
+ Progress: 300/1000 acc=0.1800
175
+ Progress: 400/1000 acc=0.1650
176
+ Progress: 500/1000 acc=0.1600
177
+ Progress: 600/1000 acc=0.1517
178
+ Progress: 700/1000 acc=0.1514
179
+ Progress: 800/1000 acc=0.1450
180
+ Progress: 900/1000 acc=0.1444
181
+ Progress: 1000/1000 acc=0.1410
182
+ m1 / ood_dense: accuracy=0.1410 (386.2s)
183
+ M1 CoT check: 0/50 outputs contain CoT (0.00)
184
+ Parser check: 49/50 outputs have valid format (0.98)
185
+
186
+ ============================================================
187
+ Loading model: m3
188
+ ============================================================
189
+ The new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`
190
+ [load_model] Resized embeddings: 50260 -> 50260
191
+ [load_model] Coconut (continuous) loaded from /lambda/nfs/experiment/results/v9_meta_fork/prosqa-coconut/checkpoint_50
192
+ Missing keys: []
193
+ Unexpected keys: []
194
+ Type: coconut, feedback_mode: continuous
195
+ Thought token check: found 3, expected 3
196
+
197
+ Evaluating m3 on prosqa_test (500 samples)...
198
+ Sample 0: GT='Sally is a sterpus.' Pred='Sally is a sterpus.' Correct=True
199
+ Sample 1: GT='Bob is a shumpus.' Pred='Bob is a shumpus.' Correct=True
200
+ Sample 2: GT='Max is a boompus.' Pred='Max is a boompus.' Correct=True
201
+ Progress: 100/500 acc=0.9600
202
+ Progress: 200/500 acc=0.9700
203
+ Progress: 300/500 acc=0.9700
204
+ Progress: 400/500 acc=0.9675
205
+ Progress: 500/500 acc=0.9700
206
+ m3 / prosqa_test: accuracy=0.9700 (34.1s)
207
+
208
+ Evaluating m3 on ood_7hop (1000 samples)...
209
+ Sample 0: GT='Davis is a boompus.' Pred='Davis is a lempus.' Correct=False
210
+ Sample 1: GT='Sally is a yerpus.' Pred='Sally is a yerpus.' Correct=True
211
+ Sample 2: GT='Wren is a yumpus.' Pred='Wren is a yumpus.' Correct=True
212
+ Progress: 100/1000 acc=0.7500
213
+ Progress: 200/1000 acc=0.6950
214
+ Progress: 300/1000 acc=0.6833
215
+ Progress: 400/1000 acc=0.6675
216
+ Progress: 500/1000 acc=0.6760
217
+ Progress: 600/1000 acc=0.6717
218
+ Progress: 700/1000 acc=0.6643
219
+ Progress: 800/1000 acc=0.6625
220
+ Progress: 900/1000 acc=0.6622
221
+ Progress: 1000/1000 acc=0.6600
222
+ m3 / ood_7hop: accuracy=0.6600 (74.7s)
223
+
224
+ Evaluating m3 on ood_8hop (1000 samples)...
225
+ Sample 0: GT='Sally is a boompus.' Pred='Sally is a boompus.' Correct=True
226
+ Sample 1: GT='Bob is a lorpus.' Pred='Bob is a grimpus.' Correct=False
227
+ Sample 2: GT='Bob is a kerpus.' Pred='Bob is a kerpus.' Correct=True
228
+ Progress: 100/1000 acc=0.7100
229
+ Progress: 200/1000 acc=0.7300
230
+ Progress: 300/1000 acc=0.7300
231
+ Progress: 400/1000 acc=0.7100
232
+ Progress: 500/1000 acc=0.7040
233
+ Progress: 600/1000 acc=0.7117
234
+ Progress: 700/1000 acc=0.6886
235
+ Progress: 800/1000 acc=0.6775
236
+ Progress: 900/1000 acc=0.6756
237
+ Progress: 1000/1000 acc=0.6750
238
+ m3 / ood_8hop: accuracy=0.6750 (76.8s)
239
+
240
+ Evaluating m3 on ood_dag (1000 samples)...
241
+ Sample 0: GT='Stella is a vumpus.' Pred='Stella is a vumpus.' Correct=True
242
+ Sample 1: GT='Sam is a yimpus.' Pred='Sam is a yimpus.' Correct=True
243
+ Sample 2: GT='Wren is a zhorpus.' Pred='Wren is a sterpus.' Correct=False
244
+ Progress: 100/1000 acc=0.5900
245
+ Progress: 200/1000 acc=0.6250
246
+ Progress: 300/1000 acc=0.6100
247
+ Progress: 400/1000 acc=0.6125
248
+ Progress: 500/1000 acc=0.6080
249
+ Progress: 600/1000 acc=0.6083
250
+ Progress: 700/1000 acc=0.5957
251
+ Progress: 800/1000 acc=0.5950
252
+ Progress: 900/1000 acc=0.5922
253
+ Progress: 1000/1000 acc=0.5920
254
+ m3 / ood_dag: accuracy=0.5920 (65.4s)
255
+
256
+ Evaluating m3 on ood_dense (1000 samples)...
257
+ Sample 0: GT='Sally is a yerpus.' Pred='Sally is a yerpus.' Correct=True
258
+ Sample 1: GT='Fae is a rompus.' Pred='Fae is a rompus.' Correct=True
259
+ Sample 2: GT='Carol is a quimpus.' Pred='Carol is a quimpus.' Correct=True
260
+ Progress: 100/1000 acc=0.6100
261
+ Progress: 200/1000 acc=0.6050
262
+ Progress: 300/1000 acc=0.6200
263
+ Progress: 400/1000 acc=0.6075
264
+ Progress: 500/1000 acc=0.5860
265
+ Progress: 600/1000 acc=0.5983
266
+ Progress: 700/1000 acc=0.6057
267
+ Progress: 800/1000 acc=0.6050
268
+ Progress: 900/1000 acc=0.6144
269
+ Progress: 1000/1000 acc=0.6120
270
+ m3 / ood_dense: accuracy=0.6120 (73.9s)
271
+
272
+ ============================================================
273
+ Loading model: m5
274
+ ============================================================
275
+ [load_model] Resized embeddings: 50260 -> 50260
276
+ [load_model] Coconut (pause_curriculum) loaded from /lambda/nfs/experiment/results/v9_meta_fork/prosqa-m5-pause/checkpoint_50
277
+ Missing keys: []
278
+ Unexpected keys: []
279
+ Type: coconut, feedback_mode: pause_curriculum
280
+ Thought token check: found 3, expected 3
281
+
282
+ Evaluating m5 on prosqa_test (500 samples)...
283
+ Sample 0: GT='Sally is a sterpus.' Pred='Sally is a sterpus.' Correct=True
284
+ Sample 1: GT='Bob is a shumpus.' Pred='Bob is a shumpus.' Correct=True
285
+ Sample 2: GT='Max is a boompus.' Pred='Max is a boompus.' Correct=True
286
+ Progress: 100/500 acc=0.9600
287
+ Progress: 200/500 acc=0.9750
288
+ Progress: 300/500 acc=0.9633
289
+ Progress: 400/500 acc=0.9625
290
+ Progress: 500/500 acc=0.9660
291
+ m5 / prosqa_test: accuracy=0.9660 (24.7s)
292
+
293
+ Evaluating m5 on ood_7hop (1000 samples)...
294
+ Sample 0: GT='Davis is a boompus.' Pred='Davis is a boompus.' Correct=True
295
+ Sample 1: GT='Sally is a yerpus.' Pred='Sally is a rempus.' Correct=False
296
+ Sample 2: GT='Wren is a yumpus.' Pred='Wren is a yumpus.' Correct=True
297
+ Progress: 100/1000 acc=0.7600
298
+ Progress: 200/1000 acc=0.7250
299
+ Progress: 300/1000 acc=0.7367
300
+ Progress: 400/1000 acc=0.7450
301
+ Progress: 500/1000 acc=0.7360
302
+ Progress: 600/1000 acc=0.7317
303
+ Progress: 700/1000 acc=0.7314
304
+ Progress: 800/1000 acc=0.7325
305
+ Progress: 900/1000 acc=0.7478
306
+ Progress: 1000/1000 acc=0.7540
307
+ m5 / ood_7hop: accuracy=0.7540 (44.7s)
308
+
309
+ Evaluating m5 on ood_8hop (1000 samples)...
310
+ Sample 0: GT='Sally is a boompus.' Pred='Sally is a boompus.' Correct=True
311
+ Sample 1: GT='Bob is a lorpus.' Pred='Bob is a grimpus.' Correct=False
312
+ Sample 2: GT='Bob is a kerpus.' Pred='Bob is a kerpus.' Correct=True
313
+ Progress: 100/1000 acc=0.7600
314
+ Progress: 200/1000 acc=0.7500
315
+ Progress: 300/1000 acc=0.7500
316
+ Progress: 400/1000 acc=0.7425
317
+ Progress: 500/1000 acc=0.7480
318
+ Progress: 600/1000 acc=0.7433
319
+ Progress: 700/1000 acc=0.7386
320
+ Progress: 800/1000 acc=0.7462
321
+ Progress: 900/1000 acc=0.7511
322
+ Progress: 1000/1000 acc=0.7510
323
+ m5 / ood_8hop: accuracy=0.7510 (46.8s)
324
+
325
+ Evaluating m5 on ood_dag (1000 samples)...
326
+ Sample 0: GT='Stella is a vumpus.' Pred='Stella is a vumpus.' Correct=True
327
+ Sample 1: GT='Sam is a yimpus.' Pred='Sam is a yimpus.' Correct=True
328
+ Sample 2: GT='Wren is a zhorpus.' Pred='Wren is a sterpus.' Correct=False
329
+ Progress: 100/1000 acc=0.5600
330
+ Progress: 200/1000 acc=0.5200
331
+ Progress: 300/1000 acc=0.5500
332
+ Progress: 400/1000 acc=0.5475
333
+ Progress: 500/1000 acc=0.5380
334
+ Progress: 600/1000 acc=0.5317
335
+ Progress: 700/1000 acc=0.5229
336
+ Progress: 800/1000 acc=0.5288
337
+ Progress: 900/1000 acc=0.5200
338
+ Progress: 1000/1000 acc=0.5190
339
+ m5 / ood_dag: accuracy=0.5190 (40.8s)
340
+
341
+ Evaluating m5 on ood_dense (1000 samples)...
342
+ Sample 0: GT='Sally is a yerpus.' Pred='Sally is a yerpus.' Correct=True
343
+ Sample 1: GT='Fae is a rompus.' Pred='Fae is a lempus.' Correct=False
344
+ Sample 2: GT='Carol is a quimpus.' Pred='Carol is a rempus.' Correct=False
345
+ Progress: 100/1000 acc=0.7000
346
+ Progress: 200/1000 acc=0.6700
347
+ Progress: 300/1000 acc=0.6767
348
+ Progress: 400/1000 acc=0.6750
349
+ Progress: 500/1000 acc=0.6780
350
+ Progress: 600/1000 acc=0.6750
351
+ Progress: 700/1000 acc=0.6814
352
+ Progress: 800/1000 acc=0.6875
353
+ Progress: 900/1000 acc=0.6867
354
+ Progress: 1000/1000 acc=0.6840
355
+ m5 / ood_dense: accuracy=0.6840 (48.7s)
356
+
357
+ Results saved to /lambda/nfs/experiment/results/v9_meta_fork/experiments/ood/results.json
358
+ Detailed outputs saved to /lambda/nfs/experiment/results/v9_meta_fork/experiments/ood/detailed_outputs.json
359
+
360
+ ======================================================================
361
+ RESULTS SUMMARY
362
+ ======================================================================
363
+ Model prosqa_test ood_7hop ood_8hop ood_dag ood_dense
364
+ ----------------------------------------------------------------------------------------
365
+ m1 0.8300 0.1070 0.0820 0.2820 0.1410
366
+ m3 0.9700 0.6600 0.6750 0.5920 0.6120
367
+ m5 0.9660 0.7540 0.7510 0.5190 0.6840
368
+
369
+ Verification:
370
+ m1_generates_cot: False
371
+ parser_accuracy: 0.98
372
+ thought_token_checks: {'m3': True, 'm5': True}
373
+ thought_token_count_correct: True
374
+
375
+ Done.
experiments/ood/results.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "m1": {
3
+ "prosqa_test": 0.83,
4
+ "ood_7hop": 0.107,
5
+ "ood_8hop": 0.082,
6
+ "ood_dag": 0.282,
7
+ "ood_dense": 0.141
8
+ },
9
+ "m3": {
10
+ "prosqa_test": 0.97,
11
+ "ood_7hop": 0.66,
12
+ "ood_8hop": 0.675,
13
+ "ood_dag": 0.592,
14
+ "ood_dense": 0.612
15
+ },
16
+ "m5": {
17
+ "prosqa_test": 0.966,
18
+ "ood_7hop": 0.754,
19
+ "ood_8hop": 0.751,
20
+ "ood_dag": 0.519,
21
+ "ood_dense": 0.684
22
+ },
23
+ "verification": {
24
+ "m1_generates_cot": false,
25
+ "parser_accuracy": 0.98,
26
+ "thought_token_checks": {
27
+ "m3": true,
28
+ "m5": true
29
+ },
30
+ "thought_token_count_correct": true
31
+ }
32
+ }
experiments/per_sample_correctness.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"m3_prosqa_id": [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true], "m5_prosqa_id": [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true], "m3_7hop": [false, true, true, true, false, true, true, true, true, false, false, true, false, false, false, true, true, true, true, true, true, true, true, true, false, false, true, false, true, true, true, true, true, false, true, false, true, true, true, true, false, true, false, true, true, false, false, true, true, true, true, true, true, false, true, true, true, true, false, false, true, true, true, false, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, false, true, true, true, false, true, true, false, true, false, true, true, true, false, false, true, true, true, true, true, true, true, true, false, true, false, true, true, true, false, false, true, true, false, true, true, false, false, true, true, true, true, true, true, true, true, false, true, true, false, true, false, true, true, false, false, false, false, true, true, true, false, true, true, false, true, true, false, true, true, true, true, true, true, true, false, true, true, false, false, true, true, false, false, true, true, true, false, false, false, true, true, false, false, false, true, false, false, false, true, true, false, true, true, true, true, true, true, true, true, true, false, false, false, true, true, true, false, false, true, false, true, false, true, false, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, true, false, false, false, false, true, true, true, true, true, true, true, false, false, true, true, true, false, false, true, true, true, false, true, false, false, true, false, true, true, true, false, false, true, false, false, true, true, true, true, true, true, false, true, true, false, true, true, true, false, false, true, false, true, true, true, true, false, true, true, true, false, true, false, false, true, false, false, true, false, true, true, false, false, true, true, true, true, false, true, false, true, false, false, true, true, true, false, false, true, true, false, false, true, true, true, true, true, true, false, true, true, true, true, true, false, true, true, false, true, true, true, false, true, false, true, true, true, true, false, false, true, true, true, true, true, true, true, false, true, true, false, true, false, true, false, false, true, true, true, true, true, true, true, false, false, true, true, false, true, false, true, true, false, true, false, false, false, false, false, true, false, false, true, true, true, false, true, false, true, false, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, false, true, false, true, true, true, false, true, true, false, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, false, true, false, true, false, true, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, false, false, true, true, false, false, true, true, true, true, false, true, true, false, true, true, false, false, true, false, true, false, false, true, true, true, false, true, true, true, false, true, false, true, true, true, true, false, false, true, true, false, false, true, true, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, false, true, false, true, false, true, false, false, false, true, true, false, true, false, false, true, false, false, true, false, false, false, true, true, true, false, true, true, true, true, false, true, true, false, true, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, false, true, false, true, false, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, false, false, false, false, true, true, false, false, false, false, true, false, true, false, true, false, true, false, true, false, false, false, true, true, true, true, true, true, false, true, true, true, false, true, false, true, true, true, true, true, true, false, true, true, true, false, false, false, false, true, false, false, true, false, true, true, false, false, true, false, false, true, true, false, true, true, true, true, true, false, true, true, false, false, true, true, true, true, true, true, true, false, false, true, true, false, true, true, true, true, false, true, true, true, false, true, true, true, false, false, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, false, true, false, true, false, false, true, false, false, true, false, true, false, true, true, false, true, true, true, true, false, false, false, false, true, true, false, true, true, true, true, false, true, true, true, true, false, false, true, true, true, false, true, false, false, false, false, true, true, false, true, false, true, true, true, true, true, false, true, true, true, true, true, false, false, true, false, true, true, true, true, false, true, false, true, false, false, true, false, true, false, true, false, true, false, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, false, false, true, false, false, false, false, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, true, false, false, false, false, true, true, true, true, false, true, false, true, true, false, true, false, true, false, true, true, true, false, true, false, true, false, false, true, true, true, true, true, true, true, true, true, true, true, false, true, false, true, false, true, false, true, true, true, true, false, true, true, true, false, false, false, true, false, true, true, true, true, true, false, false, false, false, false, true, true, false, true, false, true, true, true, true, true, false, false, true, false, false, true, false, true, true, true, true, false, true, false, false, true, true, true, false, true, true, false, true, true, true, false, true, false, true, true, true, true, true, false, true, false, true, true, true, false, true, true, false, false, true, true, true, false, true, true, true, true, false, true, true, false, true, true], "m5_7hop": [true, false, true, true, true, true, true, true, true, false, true, true, false, false, true, true, false, true, false, true, true, true, false, true, true, false, false, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, true, false, false, false, true, false, true, false, true, false, true, true, true, true, false, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, false, true, false, false, false, true, true, true, true, true, true, false, true, false, true, false, true, true, true, true, true, false, false, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, false, true, false, true, false, false, true, true, true, true, false, true, true, false, false, true, true, true, true, true, false, false, true, true, true, true, true, true, false, false, true, true, true, false, false, true, false, true, false, false, true, true, true, true, false, true, false, true, true, true, false, false, true, true, true, true, false, false, true, true, true, true, true, true, false, true, true, true, false, false, true, true, true, true, false, true, true, true, true, true, true, false, false, true, true, false, true, true, true, false, true, false, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, false, true, true, false, false, true, true, true, false, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, true, false, true, false, false, true, true, true, true, false, true, false, false, true, true, true, true, false, true, true, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, false, false, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, false, true, true, true, false, true, false, false, true, false, true, false, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, false, true, false, true, false, true, false, false, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, false, true, true, true, true, false, true, true, false, false, true, true, true, true, true, false, true, true, true, false, false, true, true, true, true, false, true, true, true, true, true, false, false, true, true, true, true, true, true, false, false, true, true, false, true, true, true, true, true, true, true, true, true, true, false, true, false, false, false, true, true, false, false, true, true, true, true, false, false, true, true, true, true, false, true, false, false, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, true, false, true, true, false, true, true, true, true, true, true, true, true, false, false, true, true, false, true, true, false, true, false, true, false, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, false, true, false, false, true, false, true, true, false, true, true, true, false, true, false, true, true, true, true, false, false, true, true, false, true, false, false, true, false, false, true, true, true, true, true, true, true, false, true, false, true, false, true, true, true, true, true, true, true, false, true, true, true, true, false, true, false, false, true, true, true, true, true, true, false, false, true, true, false, false, true, true, true, true, true, true, true, true, false, true, true, true, false, true, false, true, true, true, true, true, true, true, true, false, true, false, true, true, false, false, false, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, false, false, true, true, false, true, true, false, true, true, false, false, true, true, false, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, false, true, true, true, true, true, false, true, false, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, false, false, true, true, true, false, true, true, false, true, true, true, true, true, false, true, false, false, false, true, true, false, true, false, true, true, false, true, true, true, false, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, false, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, false, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, false, false, true, true, true, true, true, true, true, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, false, true, true, true, false, true, true, true, false, true, false, true, false, true, true, true, true], "m3_8hop": [true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, false, true, true, false, true, false, true, true, true, false, true, false, false, true, false, true, true, true, true, false, false, true, true, false, false, true, true, true, true, false, true, true, true, false, true, true, true, false, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, false, false, false, true, false, true, true, true, true, false, false, false, true, true, false, true, false, false, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, false, true, true, false, true, false, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, false, true, true, false, false, true, true, true, false, false, true, true, true, true, true, true, false, false, false, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, false, true, true, false, true, true, true, false, true, true, true, true, false, false, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, false, true, true, true, true, false, false, true, true, true, true, true, false, false, false, true, true, true, false, false, false, true, true, true, false, true, true, false, true, true, true, false, true, false, false, true, true, true, true, false, false, true, false, true, true, true, false, true, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, false, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, false, false, false, true, false, true, true, true, true, false, false, true, true, false, true, false, true, false, true, true, false, false, true, true, true, false, true, true, false, true, false, true, false, true, true, true, false, false, false, true, true, true, false, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, false, false, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, false, false, true, false, false, false, false, true, true, false, true, true, false, true, false, false, true, false, true, false, true, true, false, true, true, false, true, true, true, true, false, false, false, true, false, false, false, true, true, true, true, true, true, true, true, false, false, true, true, true, true, false, true, true, true, true, true, false, true, false, true, true, true, true, true, false, false, true, true, true, true, true, false, true, false, true, true, false, true, false, true, true, true, false, true, true, false, true, true, true, true, true, false, false, true, false, true, true, false, true, false, true, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, false, true, true, true, true, true, true, false, true, false, false, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, false, true, true, true, false, true, true, false, true, true, true, true, true, false, false, true, true, false, true, true, true, false, true, false, true, true, false, true, true, true, true, true, true, true, true, true, true, false, true, false, false, true, true, true, true, true, true, true, false, true, false, false, true, true, true, true, false, true, true, true, false, false, true, false, true, true, true, true, true, true, false, true, false, false, true, true, false, true, false, false, true, true, true, false, true, true, false, false, false, true, true, true, false, true, true, true, false, false, false, false, true, false, false, true, false, true, true, false, false, false, true, true, false, true, false, false, true, true, true, true, false, true, true, true, true, false, false, false, false, false, false, true, true, false, true, false, false, false, false, true, true, true, true, true, true, false, false, true, true, false, false, false, true, true, true, true, true, false, true, true, false, true, true, true, true, false, true, false, false, true, true, true, true, true, false, false, false, true, false, true, false, true, false, true, false, false, false, true, true, false, false, true, true, false, true, false, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, false, true, false, true, true, true, false, false, false, false, false, false, true, true, false, true, true, false, true, false, true, true, false, true, false, false, true, false, true, true, false, false, false, true, true, true, true, true, true, true, true, true, true, false, false, false, true, true, true, true, true, true, true, true, true, true, false, false, false, false, true, true, false, false, true, true, true, true, false, false, true, true, true, true, true, true, true, false, true, true, false, true, true, true, false, false, false, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, false, true, false, true, false, true, true, false, false, true, false, true, false, true, true, true, true, false, true, false, true, true, false, false, false, true, true, true, true, false, true, true, true, false, false, false, true, false, true, true, true, false, false, true, true, false, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, true, true, false, true, true, false, true, false, true, false, true, false, false, true, false, false, true, true, true, false, false, true, true, true, false, true, false, false, true, false, true, true, true, true, true, false, true, true, false, true, true, false, true, true, true, true, true, true, true, false, false, false, true, false, false, true, false, true, false, true, true, true, true, true, true, false, true, false], "m5_8hop": [true, false, true, true, true, true, true, true, true, false, false, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, false, true, true, true, false, true, true, true, true, false, true, true, true, false, true, true, true, false, true, true, true, true, true, true, false, true, false, true, true, false, true, true, false, true, false, true, false, true, true, true, false, true, true, true, false, true, false, false, true, true, true, true, true, true, true, false, true, false, true, true, true, true, false, true, true, true, true, false, true, true, true, false, true, false, true, false, true, true, true, true, true, true, true, false, true, false, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, false, true, false, true, false, true, true, true, true, true, true, false, true, false, true, true, true, false, true, false, false, true, false, true, true, true, true, false, true, true, true, true, false, true, true, false, true, true, true, false, false, false, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, false, true, true, false, false, true, true, false, true, false, true, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, false, false, false, true, true, false, true, true, true, true, true, false, true, true, false, false, false, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, false, false, false, true, false, false, true, false, true, true, true, false, true, false, true, false, true, true, true, true, true, true, true, false, true, true, false, false, true, true, false, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, false, true, true, true, true, true, true, true, false, true, false, true, true, false, true, true, true, false, false, true, false, true, true, true, true, true, true, false, true, true, false, true, false, false, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, false, false, true, true, true, false, false, true, true, true, false, false, false, true, true, true, false, false, true, true, false, true, true, true, true, false, false, true, true, true, false, true, true, true, false, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, false, true, true, true, true, true, true, false, false, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, true, false, true, true, true, false, true, false, true, true, true, false, false, false, true, false, false, false, false, true, false, true, true, true, true, false, false, false, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, false, false, true, false, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, true, true, true, false, true, true, true, false, false, true, true, true, false, true, true, true, true, true, true, false, false, false, true, true, false, true, true, true, false, true, false, true, false, true, false, true, false, false, false, true, true, true, true, true, true, true, true, false, true, false, false, true, true, true, true, true, true, false, true, true, true, true, true, true, false, true, false, true, true, false, false, true, false, true, true, false, true, false, true, true, true, true, true, false, true, false, true, false, false, true, false, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, false, true, true, true, false, true, true, true, true, false, true, true, false, true, false, true, true, false, true, false, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, false, true, false, true, true, true, true, true, true, true, true, false, true, true, false, false, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, false, true, false, true, true, false, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, false, true, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, false, true, true, false, false, true, true, false, false, false, true, true, true, false, true, true, false, true, true, false, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, false, true, true, false, false, true, true, true, true, true, true, true, true, true, false, false, true, true, false, true, true, false, true, false, true, true, false, true, false, true, false, true, true, true, false, true, false, true, false, false, true, true, true, true, false, true, true, true, true, true, true, false, true], "m3_dag": [true, true, false, true, true, false, true, true, true, true, false, true, false, true, false, false, true, false, true, true, true, true, true, true, false, false, false, true, false, false, true, true, true, true, false, true, true, true, true, true, false, true, true, true, false, false, false, false, true, false, true, true, true, false, true, false, false, true, false, false, true, false, true, true, true, true, true, false, true, false, false, false, false, false, false, false, true, false, false, false, true, true, false, true, true, false, true, true, true, true, false, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, false, true, true, false, true, false, true, true, false, false, true, true, false, false, false, false, true, false, true, true, false, false, true, true, true, false, true, true, true, true, true, true, false, false, true, true, true, true, false, false, true, false, true, true, true, true, true, true, true, false, false, true, false, true, true, true, false, false, true, false, true, false, true, false, true, false, true, true, true, true, false, false, true, true, false, false, false, true, true, true, true, true, true, false, true, true, false, true, false, true, false, false, false, false, true, true, false, true, true, true, true, false, false, true, false, false, true, false, true, true, true, false, false, true, false, false, true, false, true, false, true, false, false, true, false, true, false, true, true, true, false, false, true, true, false, false, true, true, true, true, false, true, true, true, false, false, true, true, true, true, true, true, false, true, false, true, true, true, true, false, false, true, false, false, false, false, true, true, false, true, true, true, true, true, true, false, true, false, true, false, true, true, true, true, true, false, true, true, true, true, false, true, false, false, false, true, true, true, false, false, true, false, true, true, true, true, true, false, false, false, true, false, true, true, true, false, true, true, true, false, true, false, true, false, false, true, false, false, false, true, false, true, true, true, true, true, true, false, false, true, false, true, false, false, true, true, true, false, true, true, false, true, false, true, true, false, true, true, true, true, false, false, true, true, true, true, true, true, false, false, true, true, true, true, true, false, false, false, true, true, true, false, true, false, true, false, true, false, false, true, true, true, true, true, true, false, false, false, false, false, true, true, true, true, false, false, false, true, true, true, true, false, false, true, true, true, false, true, false, true, false, false, true, true, false, true, true, false, true, true, true, false, false, true, true, true, true, false, false, true, true, true, true, false, false, false, true, true, false, false, true, false, false, false, true, false, true, true, true, false, true, true, true, false, true, true, true, true, false, false, true, true, true, true, false, false, false, true, true, false, true, true, true, true, true, true, false, true, true, false, false, false, false, true, false, true, false, false, false, true, true, false, true, true, false, true, true, false, true, true, false, true, true, true, true, true, false, false, true, true, true, true, false, true, false, false, false, true, false, true, true, true, false, false, true, true, true, true, false, false, true, true, false, true, true, true, true, true, false, true, false, true, false, false, true, true, true, false, false, true, true, true, true, true, true, true, true, false, true, true, true, true, false, true, false, false, true, true, false, false, true, false, false, false, false, true, false, true, true, true, false, false, false, false, false, true, false, true, true, true, true, true, true, false, false, true, false, false, false, false, false, true, false, false, true, false, true, false, false, false, true, false, true, false, false, false, false, true, true, false, false, true, true, true, false, true, false, true, false, true, true, true, true, false, true, false, false, true, true, false, true, false, false, true, true, false, false, true, true, true, true, true, true, true, false, true, false, true, true, true, false, true, true, false, true, true, false, false, true, false, true, false, true, false, true, true, false, true, false, true, true, false, false, true, true, false, false, true, true, false, true, false, false, true, true, true, true, true, false, true, true, false, true, false, false, true, false, false, true, false, true, false, false, true, false, false, true, false, true, false, true, false, false, true, false, false, true, false, true, true, true, false, true, false, true, true, false, false, true, true, false, false, true, true, false, true, true, false, true, false, true, true, true, true, true, true, true, true, true, true, true, true, false, false, true, true, true, true, false, true, true, true, false, true, true, true, false, false, true, false, true, true, false, false, true, true, false, false, false, true, true, true, true, false, false, true, true, true, false, true, false, true, true, false, true, true, false, true, true, true, true, true, false, false, true, true, false, true, true, false, true, true, true, false, false, true, false, true, true, true, true, false, false, false, true, false, false, true, false, false, false, true, false, false, true, true, true, true, true, false, true, true, true, false, false, false, false, false, false, false, false, true, true, true, true, true, false, true, false, false, true, true, true, false, false, true, true, true, true, true, true, true, false, true, false, false, false, true, false, false, false, false, true, true, false, false, true, true, false, false, true, true, true, true, true, true, true, false, true, true, true, true, false, true, false, false, true, true, true, true, false, true, true, false, true, true, true, false, true, false, false, false, false, false, true, false, true, true, true, true, true, false, true, true, true, false, false, true, true, false, false, true, false, false, true, true, false, true, true, false, false, false, true, true, false, false, true, true, true, true, true, false, true], "m5_dag": [true, true, false, false, false, true, true, false, false, true, true, false, true, true, true, false, true, true, false, true, false, true, false, false, true, false, true, true, true, true, false, true, true, false, false, true, false, true, false, false, false, true, false, true, true, false, true, false, false, false, true, true, true, true, true, true, false, true, false, true, false, false, true, true, false, true, false, false, true, false, true, false, false, false, false, true, true, false, true, false, true, false, true, false, false, true, true, false, false, true, true, true, true, true, false, true, true, true, true, true, false, false, true, false, true, false, false, false, true, true, true, true, false, true, true, false, true, false, false, true, false, false, false, true, true, false, false, false, false, false, false, true, true, false, false, false, false, false, true, false, true, true, false, false, true, true, true, true, false, false, false, false, true, false, true, true, true, true, true, false, false, true, false, false, true, true, false, true, true, false, false, true, false, true, false, true, false, false, true, false, true, true, true, true, true, false, false, false, false, true, true, false, false, false, true, true, true, true, false, true, true, true, true, false, true, true, false, true, false, true, false, false, true, true, true, false, true, true, true, true, false, true, true, false, false, false, false, false, true, false, true, false, false, true, true, true, false, false, true, false, true, true, false, false, false, true, true, false, true, true, true, false, true, true, true, false, false, true, true, true, true, false, true, true, false, true, true, true, true, false, true, false, true, false, true, true, true, true, true, false, false, true, true, true, false, false, true, true, false, true, true, true, true, false, false, true, false, true, true, false, true, false, true, false, false, false, true, false, true, true, true, true, true, false, true, false, true, true, false, true, true, false, true, true, false, false, true, false, false, false, false, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, true, false, false, true, false, false, true, true, false, true, false, true, false, true, false, true, true, true, false, false, true, false, true, false, false, true, true, true, true, true, false, true, false, false, true, true, false, true, true, false, true, false, true, true, true, false, false, false, true, true, false, true, false, true, true, false, false, false, false, false, true, false, true, true, false, false, true, true, true, true, false, false, true, true, true, false, false, false, false, false, false, true, true, true, false, true, true, false, false, false, true, false, false, true, true, true, true, false, false, true, true, false, true, true, true, true, false, true, true, false, true, true, true, true, false, false, true, true, false, false, true, false, false, false, false, false, true, true, true, false, false, false, true, false, true, true, false, true, true, false, true, true, false, false, true, false, false, true, false, true, false, true, false, false, false, false, true, true, false, false, true, true, true, false, false, true, false, true, true, false, false, true, false, true, true, false, true, true, false, true, true, false, true, false, true, false, true, false, true, true, true, false, false, false, true, false, false, true, true, false, false, true, true, false, false, false, true, true, false, false, false, true, true, true, true, false, false, false, true, false, true, false, true, false, true, true, true, true, true, true, false, false, true, false, false, false, false, true, false, false, true, false, true, true, false, false, false, true, true, false, false, true, false, true, false, true, false, true, false, true, true, true, true, false, false, false, true, false, true, false, false, true, true, true, false, true, true, false, true, false, false, false, true, false, false, false, true, false, false, true, false, true, false, true, true, false, false, false, false, false, false, false, false, false, true, true, true, false, false, false, true, true, false, false, false, true, false, true, true, true, false, false, false, false, false, true, false, true, true, false, true, true, false, true, true, false, true, false, false, false, false, true, true, true, true, true, true, false, true, true, false, false, true, true, false, true, true, true, true, true, true, true, false, false, false, false, true, false, true, false, true, true, true, true, false, true, true, true, true, false, true, false, false, false, true, false, false, true, false, true, false, true, true, false, false, false, false, false, false, true, true, false, false, true, true, true, false, true, true, true, true, false, false, true, false, true, false, false, true, false, false, false, false, true, true, true, true, true, false, false, false, true, false, true, true, true, false, true, true, true, true, true, true, true, false, false, true, true, false, true, true, true, true, false, false, true, false, false, false, true, true, true, true, false, false, true, false, false, true, true, true, true, true, false, false, true, false, true, true, false, false, true, false, true, false, false, false, false, true, true, false, false, true, true, false, false, true, false, true, false, false, false, true, false, false, false, true, false, false, true, true, true, false, false, true, false, false, true, true, false, false, false, true, false, false, true, false, false, true, false, false, true, true, false, true, false, false, true, true, false, true, true, false, true, false, true, false, false, false, true, false, true, false, true, true, false, true, true, true, true, true, true, false, false, false, false, false, false, true, true, true, false, false, true, false, true, false, false, false, false, false, true, true, true, true, true, false, true, false, true, false, true, true, false, false, false, false, false, false, false, true, true, false, true, true, false, true, false, false, true, false, true, false, false, false, false, true, true, true, true, true, true, true, false, true, false, false, true, true, false, false, true, false, false, false, false, true, true, true, true, true, false, false, true, false, true, false, true, true, true, true, true, false, false], "m3_dense": [true, true, true, true, true, true, false, false, true, true, true, true, false, true, false, false, false, false, true, false, true, true, false, true, true, true, true, false, false, false, false, true, false, false, true, true, true, true, true, true, false, false, false, true, true, true, false, true, true, false, false, false, true, false, true, true, true, true, false, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, false, false, false, true, false, true, false, true, true, false, false, true, false, false, false, true, true, true, false, true, true, false, false, true, true, true, true, false, false, false, true, false, false, true, false, true, false, true, true, true, true, false, true, true, true, false, false, false, true, true, false, true, true, false, true, false, true, true, true, true, true, false, true, false, true, true, true, false, true, true, false, true, false, true, true, false, false, true, false, true, false, false, false, true, true, false, false, true, false, false, false, false, true, true, true, true, false, true, false, false, true, false, true, true, false, false, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, true, true, true, true, true, false, false, true, true, true, true, false, false, false, false, true, true, true, true, false, false, false, true, true, false, true, true, true, true, false, true, true, true, false, false, true, true, true, true, true, true, true, false, false, true, true, false, true, true, true, false, false, true, false, true, true, false, false, true, true, true, false, false, false, true, true, true, true, true, false, true, false, true, true, true, false, true, false, false, false, true, true, true, true, false, true, true, false, true, true, true, true, false, false, true, false, true, true, true, true, false, true, true, true, true, false, false, true, false, true, false, false, true, true, false, true, false, true, true, true, true, false, false, false, false, false, true, false, false, true, false, false, false, true, true, true, true, false, false, true, true, true, false, true, false, false, false, true, true, true, true, false, true, true, false, false, false, false, false, true, true, true, false, true, false, true, true, false, true, true, false, false, true, true, false, true, false, false, true, true, true, false, false, true, true, false, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, false, true, true, false, true, false, false, false, true, false, false, true, false, true, false, true, false, false, true, false, false, true, true, true, true, true, true, false, false, false, false, true, true, false, false, true, false, true, false, false, true, true, false, true, false, false, false, false, false, false, true, true, true, true, false, false, false, true, false, false, false, false, true, true, true, true, true, true, false, false, true, true, false, true, false, false, true, true, true, true, false, false, true, true, true, false, true, false, true, false, true, false, false, false, true, true, false, true, true, true, true, true, false, false, false, true, true, true, false, false, true, false, false, true, true, true, false, true, true, true, true, true, true, true, false, true, false, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, false, true, true, false, true, false, true, false, true, true, true, true, true, true, true, true, false, false, false, false, true, false, true, true, true, true, true, false, false, true, true, false, false, true, true, false, false, false, true, true, true, false, false, true, false, false, true, true, true, false, true, true, true, true, false, true, true, false, false, true, true, true, true, true, false, false, false, true, true, true, true, true, true, true, true, false, false, false, false, true, true, true, true, true, false, true, false, false, false, true, false, true, true, false, false, false, false, false, true, true, true, false, true, true, false, true, true, true, true, true, false, true, false, true, true, true, false, true, false, false, true, true, true, true, false, true, true, false, false, true, true, false, true, true, true, true, true, true, true, false, true, true, false, false, true, true, true, true, true, true, true, false, false, true, true, false, false, false, true, false, false, false, true, false, true, true, false, true, true, false, false, false, false, true, false, true, true, true, false, true, false, true, true, false, false, false, true, false, false, true, false, true, false, true, true, true, false, true, true, true, true, false, true, true, false, true, false, true, true, true, true, true, true, true, false, true, false, true, false, true, true, false, true, false, true, true, false, false, true, true, false, true, true, true, true, true, false, true, true, true, false, true, false, true, true, false, true, true, true, false, false, true, true, true, true, true, true, true, true, false, false, false, false, true, true, true, true, true, true, true, true, false, true, false, false, true, true, false, true, false, true, false, true, true, true, true, true, true, false, false, true, true, true, true, true, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, false, false, false, true, true, true, true, false, true, true, false, false, true, false, true, false, true, false, true, false, true, false, false, true, true, true, true, true, true, false, true, true, true, false, false, true, true, true, true, false, true, true, true, true, true, false, false, true, false, false, true, true, false, true, true, false, true, true, true, false, true, true, true, false, true, false, false, true, true, true, false, false, true, true, true, true, false, true, true, true, true, true, true, false, false, true, false, true, true, false, true, true, true, false, false, false, true, true, false, false, false, false, false, true, true, false, true, false, true, true, false, false, true, false, true, false, true, false, true, false, true, false, true, false, true, true, false, false, true, true, true, false, false, true, false, true], "m5_dense": [true, false, false, true, true, true, false, true, true, false, true, true, false, true, false, true, false, false, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, false, true, false, true, true, true, true, false, false, true, true, true, true, false, false, true, true, true, true, false, true, true, true, true, false, false, true, false, true, true, false, true, true, false, false, false, true, false, true, true, false, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, false, false, true, true, false, true, true, false, false, true, true, true, false, true, true, true, true, false, true, false, true, true, false, false, true, false, true, true, true, true, true, true, true, false, true, false, true, true, false, true, true, true, true, true, false, false, false, true, false, true, true, true, true, true, false, true, true, false, true, true, false, false, true, false, false, false, true, false, true, true, false, false, true, false, true, true, true, true, false, true, true, false, true, true, false, false, true, true, false, true, true, true, false, false, true, true, true, true, false, true, true, false, true, true, false, true, true, false, true, true, false, false, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, true, false, true, true, true, false, false, false, true, true, true, true, true, false, true, false, true, false, false, true, true, false, true, true, false, true, true, true, true, false, false, true, true, true, false, false, true, true, true, false, true, false, true, false, false, true, true, false, false, true, false, true, true, true, true, true, false, false, true, true, false, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, true, false, true, true, false, true, true, false, true, false, true, false, false, true, false, false, true, true, false, false, true, false, true, true, true, true, true, true, true, true, true, false, false, true, false, true, true, true, true, true, false, true, true, false, false, false, false, true, true, true, false, true, false, true, false, true, true, false, true, true, true, false, true, true, false, true, true, false, true, true, true, false, true, false, true, false, true, true, true, true, false, true, false, true, true, true, false, false, true, false, true, false, true, false, true, false, false, true, false, false, true, true, true, false, false, false, true, false, true, true, false, false, true, true, true, true, true, true, true, true, true, false, false, false, true, true, true, true, true, true, true, true, false, true, true, true, true, true, false, true, true, true, false, true, true, false, true, true, true, false, true, true, false, true, true, true, false, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, false, true, true, true, true, true, false, true, false, false, false, false, false, true, true, false, true, true, true, false, false, false, false, true, false, true, true, true, false, true, false, true, true, true, true, true, false, true, true, true, true, false, true, true, true, true, true, false, true, false, true, true, false, true, true, true, true, true, true, true, true, true, false, true, false, true, true, true, true, true, true, true, true, false, true, false, false, true, true, true, true, true, false, true, false, true, false, true, true, false, true, true, true, false, false, false, true, false, true, false, false, true, true, false, false, true, true, true, true, false, true, true, true, true, true, true, true, true, true, false, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, false, true, false, true, true, false, true, false, true, false, false, true, true, true, true, true, true, true, false, true, false, true, false, true, true, true, true, true, true, false, false, true, true, true, false, true, false, true, false, true, true, false, true, false, true, true, false, false, false, true, false, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, false, true, false, true, true, false, true, true, true, true, true, true, false, false, false, true, false, true, true, true, true, true, true, false, true, true, false, true, true, false, true, true, true, true, true, true, true, false, false, true, false, true, true, true, true, true, true, false, true, false, true, true, false, true, true, true, true, true, false, true, true, true, true, false, true, false, false, true, false, true, true, true, true, true, true, false, true, true, false, true, true, false, false, true, false, true, true, true, true, true, true, false, true, true, true, true, false, false, true, true, true, false, true, true, true, true, false, true, true, true, false, true, false, false, true, false, false, false, false, false, true, false, true, true, false, true, true, false, true, false, true, false, false, true, true, false, true, false, true, true, true, true, true, true, true, true, true, true, false, true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, false, true, true, true, false, false, true, false, false, true, true, true, true, true, true, true, true, false, true, false, false, true, false, true, true, true, false, true, true, true, true, false, true, false, true, true, true, true, true, false, true, false, false, true, true, true, false, true, true, false, true, true, true, false, true, true, false, true, false, true, true, true, false, false, true, false, true, true, true, true, true, false, true, false, true, true, true, true, true, false, true, true, false, false, false, true, false, false, false, true, true, false, false, false, true, false, true, true, false, true, true, true, true, false, true, true, true, true, true, true, true, false, true, false, false, true, true, false, true, false, true, true, true]}
experiments/per_sample_logprobs.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/per_sample_logprobs_v3.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"m3_prosqa_id": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m3_7hop": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.3112935448589269e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.145764938177308e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.015549961477518082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.5020257706055418e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m3_8hop": [0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.390948667307384e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.5497195136049413e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0], "m3_dag": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920922133867862e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -2.145764938177308e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m3_dense": [0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.9073468138230965e-06, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0020729261450469494, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.483612312673358e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.729855441430118e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.311301275563892e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m5_prosqa_id": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m5_7hop": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.311301275563892e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.7881377516459906e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -9.536738616588991e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -4.768370445162873e-07, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.145764938177308e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152531907195225e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -3.4570634852570947e-06, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0], "m5_8hop": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -9.536738616588991e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920922133867862e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.2186455882765586e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.4570634852570947e-06, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.5497195136049413e-06, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.9073468138230965e-06, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768360213347478e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.7881377516459906e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m5_dag": [0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -3.576278118089249e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -3.576278118089249e-07, 0.0, -2.0265558760002023e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -8.344646857949556e-07, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -2.3841830625315197e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920922133867862e-06, -2.3841855067985307e-07, -4.768370445162873e-07, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0728830375228426e-06, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.622600959512056e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0728830375228426e-06, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, -3.576278118089249e-07, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -7.152555099310121e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.4570634852570947e-06, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.311301275563892e-06, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, -9.536738616588991e-07, 0.0, -1.1920928244535389e-07, 0.0, -9.536738616588991e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "m5_dense": [0.0, -2.3841830625315197e-06, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -4.291525328881107e-06, -4.768370445162873e-07, -2.0265558760002023e-06, -2.3841855067985307e-07, 0.0, 0.0, -1.7881377516459906e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, -7.152555099310121e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -9.536738616588991e-07, 0.0, 0.0, -2.145764938177308e-06, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.3378546504536644e-06, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -2.3841855067985307e-07, -3.576278118089249e-07, -8.344646857949556e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.311301275563892e-06, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, -1.1920922133867862e-06, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.9073468138230965e-06, 0.0, -3.576278118089249e-07, 0.0, -2.3841855067985307e-07, 0.0, -2.7418097943154862e-06, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -8.4638240878121e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -1.0728830375228426e-06, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -4.768370445162873e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -5.960462772236497e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, -2.145764938177308e-06, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -1.168244216387393e-05, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.4305104514278355e-06, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -2.50339189733495e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841830625315197e-06, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, -2.3841855067985307e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -1.311301275563892e-06, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.311301275563892e-06, 0.0, -5.960462772236497e-07, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, -3.576278118089249e-07, -2.3841855067985307e-07, -2.3841855067985307e-07, -2.3841830625315197e-06, 0.0, -8.344646857949556e-07, 0.0, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, -4.768370445162873e-07, -1.1920928244535389e-07, 0.0, 0.0, -3.576278118089249e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -1.0728830375228426e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -4.768370445162873e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, -1.5497195136049413e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, -9.536738616588991e-07, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -5.960462772236497e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -8.344646857949556e-07, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.768370445162873e-07, 0.0, 0.0, -7.152555099310121e-07, -1.1920928244535389e-07, -3.576278118089249e-07, 0.0, -3.576278118089249e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -5.960462772236497e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -2.9802276912960224e-06, 0.0, 0.0, -1.1920928244535389e-07, -2.3841855067985307e-07, -3.576278118089249e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, -7.152555099310121e-07, 0.0, 0.0, -3.576278118089249e-07, -1.0728830375228426e-06, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -8.344646857949556e-07, -5.960462772236497e-07, -2.3841855067985307e-07, 0.0, -2.7418097943154862e-06, 0.0, -1.1920928244535389e-07, 0.0, -7.629365427419543e-06, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, -2.3841855067985307e-07, 0.0, -1.1920928244535389e-07, -1.5497195136049413e-06, 0.0, 0.0, 0.0, 0.0, -6.174850568640977e-05, -2.3841855067985307e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920922133867862e-06, -2.9802276912960224e-06, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.0728830375228426e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.0728830375228426e-06, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, 0.0, -1.7881377516459906e-06, 0.0, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, -2.3841855067985307e-07, -1.2159273865108844e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.3841855067985307e-07, -1.1920928244535389e-07, -1.1920928244535389e-07, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -2.3841855067985307e-07, 0.0, 0.0, -1.1920928244535389e-07, 0.0, 0.0, 0.0, -3.576278118089249e-07, 0.0, 0.0, 0.0, -1.1920928244535389e-07, -1.1920928244535389e-07, -7.152555099310121e-07, -1.1920928244535389e-07, 0.0]}
experiments/per_sample_species_logprobs_m3_vs_m5.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/per_sample_species_logprobs_m3_vs_m6.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/per_sample_species_logprobs_m5_vs_m6.json ADDED
The diff for this file is too large to render. See raw diff
 
experiments/probing/log.txt ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ======================================================================
2
+ REPRESENTATION PROBING EXPERIMENT
3
+ ======================================================================
4
+ Checkpoint dir: /lambda/nfs/experiment/results/v9_meta_fork
5
+ Data: data/prosqa_test.json
6
+ Num samples: 500
7
+ Output dir: /lambda/nfs/experiment/results/v9_meta_fork/experiments/probing/
8
+ Seed: 0
9
+ Num thoughts: 6
10
+ N permutations: 0
11
+
12
+ Loading data...
13
+ Loaded 500 samples
14
+
15
+ Running synthetic probe verification...
16
+ Synthetic probe test: accuracy = 1.0000 (expect > 0.99)
17
+ Bonferroni threshold: p < 0.000641
18
+
19
+ ============================================================
20
+ Processing model: m3
21
+ ============================================================
22
+ The new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`
23
+ [load_model] Resized embeddings: 50260 -> 50260
24
+ [load_model] Coconut (continuous) loaded from /lambda/nfs/experiment/results/v9_meta_fork/prosqa-coconut/checkpoint_50
25
+ Missing keys: []
26
+ Unexpected keys: []
27
+ Extracting hidden states from 500 samples...
28
+ Extracted: 100/500 (skipped 0)
29
+ Extracted: 200/500 (skipped 0)
30
+ Extracted: 300/500 (skipped 0)
31
+ Extracted: 400/500 (skipped 0)
32
+ Extracted: 500/500 (skipped 0)
33
+ Total extracted: 500, skipped: 0
34
+ Extraction done in 19.3s
35
+ Running linear + nonlinear probes (13 layers x 6 positions)...
36
+ Position 0: 500 samples, 38 classes
37
+ Linear: layer 8, acc=0.1300, sig=False
38
+ Nonlinear: layer 0, acc=0.0000
39
+ Position 1: 500 samples, 38 classes
40
+ Linear: layer 12, acc=0.1000, sig=False
41
+ Nonlinear: layer 0, acc=0.0000
42
+ Position 2: 500 samples, 38 classes
43
+ Linear: layer 12, acc=0.1900, sig=False
44
+ Nonlinear: layer 0, acc=0.0000
45
+ Position 3: 298 samples, 38 classes
46
+ Linear: layer 0, acc=0.5537, sig=False
47
+ Nonlinear: layer 0, acc=0.0000
48
+ Position 4: 81 samples, 32 classes
49
+ Linear: layer 0, acc=0.0000, sig=False
50
+ Nonlinear: layer 0, acc=0.0000
51
+ Position 5: 12 samples, 12 classes
52
+ Linear: layer 0, acc=0.0000, sig=False
53
+ Nonlinear: layer 0, acc=0.0000
54
+ Probing done in 12.4s
55
+ Computing selectivity metric...
56
+ Selectivity done in 0.0s
57
+ Mean selectivity (positive cells): 0.0000
58
+ Max cross-position probe acc: 0.0000
59
+ Front-to-back leakage (t=0->t=5): 0.0000
60
+ Running input-position control probes...
61
+ Input-position extraction: 500 samples, skipped 0
62
+ Input-position control done in 31.7s
63
+ Mean thought-over-input advantage: 0.1052
64
+ Max input-position probe acc: 0.0500
65
+ Saved intermediate results to /lambda/nfs/experiment/results/v9_meta_fork/experiments/probing/m3_probing.json
66
+
67
+ Probe accuracy heatmap for m3:
68
+ Layer t=0 t=1 t=2 t=3 t=4 t=5
69
+ 0 0.086 0.086 0.186 0.554 0.000 0.000
70
+ 1 0.088 0.088 0.160 0.547 0.000 0.000
71
+ 2 0.088 0.080 0.184 0.490 0.000 0.000
72
+ 3 0.080 0.072 0.146 0.466 0.000 0.000
73
+ 4 0.090 0.060 0.144 0.460 0.000 0.000
74
+ 5 0.076 0.074 0.140 0.430 0.000 0.000
75
+ 6 0.098 0.058 0.138 0.396 0.000 0.000
76
+ 7 0.072 0.064 0.122 0.379 0.000 0.000
77
+ 8 0.130 0.066 0.130 0.258 0.000 0.000
78
+ 9 0.090 0.076 0.118 0.279 0.000 0.000
79
+ 10 0.058 0.094 0.140 0.329 0.000 0.000
80
+ 11 0.076 0.094 0.174 0.527 0.000 0.000
81
+ 12 0.054 0.100 0.190 0.550 0.000 0.000
82
+
83
+ ============================================================
84
+ Processing model: m5
85
+ ============================================================
86
+ [load_model] Resized embeddings: 50260 -> 50260
87
+ [load_model] Coconut (pause_curriculum) loaded from /lambda/nfs/experiment/results/v9_meta_fork/prosqa-m5-pause/checkpoint_50
88
+ Missing keys: []
89
+ Unexpected keys: []
90
+ Extracting hidden states from 500 samples...
91
+ Extracted: 100/500 (skipped 0)
92
+ Extracted: 200/500 (skipped 0)
93
+ Extracted: 300/500 (skipped 0)
94
+ Extracted: 400/500 (skipped 0)
95
+ Extracted: 500/500 (skipped 0)
96
+ Total extracted: 500, skipped: 0
97
+ Extraction done in 9.6s
98
+ Running linear + nonlinear probes (13 layers x 6 positions)...
99
+ Position 0: 500 samples, 38 classes
100
+ Linear: layer 8, acc=0.2360, sig=False
101
+ Nonlinear: layer 0, acc=0.0000
102
+ Position 1: 500 samples, 38 classes
103
+ Linear: layer 11, acc=0.1040, sig=False
104
+ Nonlinear: layer 0, acc=0.0000
105
+ Position 2: 500 samples, 38 classes
106
+ Linear: layer 12, acc=0.2200, sig=False
107
+ Nonlinear: layer 0, acc=0.0000
108
+ Position 3: 298 samples, 38 classes
109
+ Linear: layer 12, acc=0.5705, sig=False
110
+ Nonlinear: layer 0, acc=0.0000
111
+ Position 4: 81 samples, 32 classes
112
+ Linear: layer 0, acc=0.0000, sig=False
113
+ Nonlinear: layer 0, acc=0.0000
114
+ Position 5: 12 samples, 12 classes
115
+ Linear: layer 0, acc=0.0000, sig=False
116
+ Nonlinear: layer 0, acc=0.0000
117
+ Probing done in 12.8s
118
+ Computing selectivity metric...
119
+ Selectivity done in 0.0s
120
+ Mean selectivity (positive cells): 0.0000
121
+ Max cross-position probe acc: 0.0000
122
+ Front-to-back leakage (t=0->t=5): 0.0000
123
+ Running input-position control probes...
124
+ Input-position extraction: 500 samples, skipped 0
125
+ Input-position control done in 23.0s
126
+ Mean thought-over-input advantage: 0.0396
127
+ Max input-position probe acc: 0.0620
128
+ Saved intermediate results to /lambda/nfs/experiment/results/v9_meta_fork/experiments/probing/m5_probing.json
129
+
130
+ Probe accuracy heatmap for m5:
131
+ Layer t=0 t=1 t=2 t=3 t=4 t=5
132
+ 0 0.032 0.032 0.044 0.044 0.000 0.000
133
+ 1 0.036 0.018 0.050 0.064 0.000 0.000
134
+ 2 0.058 0.030 0.044 0.050 0.000 0.000
135
+ 3 0.050 0.030 0.042 0.074 0.000 0.000
136
+ 4 0.052 0.036 0.044 0.084 0.000 0.000
137
+ 5 0.068 0.030 0.058 0.114 0.000 0.000
138
+ 6 0.074 0.058 0.046 0.084 0.000 0.000
139
+ 7 0.104 0.040 0.034 0.047 0.000 0.000
140
+ 8 0.236 0.054 0.038 0.027 0.000 0.000
141
+ 9 0.146 0.092 0.068 0.030 0.000 0.000
142
+ 10 0.110 0.090 0.066 0.047 0.000 0.000
143
+ 11 0.100 0.104 0.216 0.530 0.000 0.000
144
+ 12 0.102 0.078 0.220 0.570 0.000 0.000
145
+
146
+ Final results saved to /lambda/nfs/experiment/results/v9_meta_fork/experiments/probing/results.json
147
+
148
+ ======================================================================
149
+ PROBING RESULTS SUMMARY
150
+ ======================================================================
151
+
152
+ m3:
153
+ Linear peak: layer 0, pos 3, acc 0.5537
154
+ Nonlinear peak: layer 0, pos 0, acc 0.0000
155
+ NL>Linear by >10pp: 0 cells
156
+ Diagonal pattern: True
157
+ Significant cells: 0/78
158
+ Diagonal peak layers: [8, 12, 12, 0]
159
+ Mean selectivity: 0.0000
160
+ Max cross-pos acc: 0.0000
161
+ Front-to-back leakage: 0.0000
162
+ Mean thought-over-input advantage: 0.1052
163
+ Max input-position acc: 0.0500
164
+
165
+ m5:
166
+ Linear peak: layer 12, pos 3, acc 0.5705
167
+ Nonlinear peak: layer 0, pos 0, acc 0.0000
168
+ NL>Linear by >10pp: 0 cells
169
+ Diagonal pattern: True
170
+ Significant cells: 0/78
171
+ Diagonal peak layers: [8, 11, 12, 12]
172
+ Mean selectivity: 0.0000
173
+ Max cross-pos acc: 0.0000
174
+ Front-to-back leakage: 0.0000
175
+ Mean thought-over-input advantage: 0.0396
176
+ Max input-position acc: 0.0620
177
+
178
+ Verification:
179
+ synthetic_probe_accuracy: 1.0
180
+ hidden_state_shape: [768]
181
+ all_finite: True
182
+ label_alignment_check: PASSED
183
+
184
+ Done.
experiments/probing/m3_probing.json ADDED
@@ -0,0 +1,773 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linear_probe_accuracy": [
3
+ [
4
+ 0.086,
5
+ 0.08600000000000001,
6
+ 0.186,
7
+ 0.5536912751677852,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ [
12
+ 0.08800000000000001,
13
+ 0.08800000000000001,
14
+ 0.16,
15
+ 0.5469798657718121,
16
+ 0.0,
17
+ 0.0
18
+ ],
19
+ [
20
+ 0.088,
21
+ 0.08,
22
+ 0.184,
23
+ 0.4899328859060403,
24
+ 0.0,
25
+ 0.0
26
+ ],
27
+ [
28
+ 0.08,
29
+ 0.072,
30
+ 0.146,
31
+ 0.4664429530201342,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ [
36
+ 0.09,
37
+ 0.06000000000000001,
38
+ 0.144,
39
+ 0.4597315436241611,
40
+ 0.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.076,
45
+ 0.074,
46
+ 0.13999999999999999,
47
+ 0.42953020134228187,
48
+ 0.0,
49
+ 0.0
50
+ ],
51
+ [
52
+ 0.098,
53
+ 0.057999999999999996,
54
+ 0.138,
55
+ 0.3959731543624161,
56
+ 0.0,
57
+ 0.0
58
+ ],
59
+ [
60
+ 0.07200000000000001,
61
+ 0.06400000000000002,
62
+ 0.122,
63
+ 0.37919463087248323,
64
+ 0.0,
65
+ 0.0
66
+ ],
67
+ [
68
+ 0.13,
69
+ 0.06599999999999999,
70
+ 0.13,
71
+ 0.25838926174496646,
72
+ 0.0,
73
+ 0.0
74
+ ],
75
+ [
76
+ 0.09000000000000001,
77
+ 0.076,
78
+ 0.118,
79
+ 0.2785234899328859,
80
+ 0.0,
81
+ 0.0
82
+ ],
83
+ [
84
+ 0.05800000000000001,
85
+ 0.094,
86
+ 0.14,
87
+ 0.32885906040268453,
88
+ 0.0,
89
+ 0.0
90
+ ],
91
+ [
92
+ 0.076,
93
+ 0.09400000000000001,
94
+ 0.17400000000000002,
95
+ 0.5268456375838926,
96
+ 0.0,
97
+ 0.0
98
+ ],
99
+ [
100
+ 0.054000000000000006,
101
+ 0.1,
102
+ 0.19,
103
+ 0.5503355704697986,
104
+ 0.0,
105
+ 0.0
106
+ ]
107
+ ],
108
+ "nonlinear_probe_accuracy": [
109
+ [
110
+ 0.0,
111
+ 0.0,
112
+ 0.0,
113
+ 0.0,
114
+ 0.0,
115
+ 0.0
116
+ ],
117
+ [
118
+ 0.0,
119
+ 0.0,
120
+ 0.0,
121
+ 0.0,
122
+ 0.0,
123
+ 0.0
124
+ ],
125
+ [
126
+ 0.0,
127
+ 0.0,
128
+ 0.0,
129
+ 0.0,
130
+ 0.0,
131
+ 0.0
132
+ ],
133
+ [
134
+ 0.0,
135
+ 0.0,
136
+ 0.0,
137
+ 0.0,
138
+ 0.0,
139
+ 0.0
140
+ ],
141
+ [
142
+ 0.0,
143
+ 0.0,
144
+ 0.0,
145
+ 0.0,
146
+ 0.0,
147
+ 0.0
148
+ ],
149
+ [
150
+ 0.0,
151
+ 0.0,
152
+ 0.0,
153
+ 0.0,
154
+ 0.0,
155
+ 0.0
156
+ ],
157
+ [
158
+ 0.0,
159
+ 0.0,
160
+ 0.0,
161
+ 0.0,
162
+ 0.0,
163
+ 0.0
164
+ ],
165
+ [
166
+ 0.0,
167
+ 0.0,
168
+ 0.0,
169
+ 0.0,
170
+ 0.0,
171
+ 0.0
172
+ ],
173
+ [
174
+ 0.0,
175
+ 0.0,
176
+ 0.0,
177
+ 0.0,
178
+ 0.0,
179
+ 0.0
180
+ ],
181
+ [
182
+ 0.0,
183
+ 0.0,
184
+ 0.0,
185
+ 0.0,
186
+ 0.0,
187
+ 0.0
188
+ ],
189
+ [
190
+ 0.0,
191
+ 0.0,
192
+ 0.0,
193
+ 0.0,
194
+ 0.0,
195
+ 0.0
196
+ ],
197
+ [
198
+ 0.0,
199
+ 0.0,
200
+ 0.0,
201
+ 0.0,
202
+ 0.0,
203
+ 0.0
204
+ ],
205
+ [
206
+ 0.0,
207
+ 0.0,
208
+ 0.0,
209
+ 0.0,
210
+ 0.0,
211
+ 0.0
212
+ ]
213
+ ],
214
+ "permutation_p_values": [
215
+ [
216
+ 1.0,
217
+ 1.0,
218
+ 1.0,
219
+ 1.0,
220
+ 1.0,
221
+ 1.0
222
+ ],
223
+ [
224
+ 1.0,
225
+ 1.0,
226
+ 1.0,
227
+ 1.0,
228
+ 1.0,
229
+ 1.0
230
+ ],
231
+ [
232
+ 1.0,
233
+ 1.0,
234
+ 1.0,
235
+ 1.0,
236
+ 1.0,
237
+ 1.0
238
+ ],
239
+ [
240
+ 1.0,
241
+ 1.0,
242
+ 1.0,
243
+ 1.0,
244
+ 1.0,
245
+ 1.0
246
+ ],
247
+ [
248
+ 1.0,
249
+ 1.0,
250
+ 1.0,
251
+ 1.0,
252
+ 1.0,
253
+ 1.0
254
+ ],
255
+ [
256
+ 1.0,
257
+ 1.0,
258
+ 1.0,
259
+ 1.0,
260
+ 1.0,
261
+ 1.0
262
+ ],
263
+ [
264
+ 1.0,
265
+ 1.0,
266
+ 1.0,
267
+ 1.0,
268
+ 1.0,
269
+ 1.0
270
+ ],
271
+ [
272
+ 1.0,
273
+ 1.0,
274
+ 1.0,
275
+ 1.0,
276
+ 1.0,
277
+ 1.0
278
+ ],
279
+ [
280
+ 1.0,
281
+ 1.0,
282
+ 1.0,
283
+ 1.0,
284
+ 1.0,
285
+ 1.0
286
+ ],
287
+ [
288
+ 1.0,
289
+ 1.0,
290
+ 1.0,
291
+ 1.0,
292
+ 1.0,
293
+ 1.0
294
+ ],
295
+ [
296
+ 1.0,
297
+ 1.0,
298
+ 1.0,
299
+ 1.0,
300
+ 1.0,
301
+ 1.0
302
+ ],
303
+ [
304
+ 1.0,
305
+ 1.0,
306
+ 1.0,
307
+ 1.0,
308
+ 1.0,
309
+ 1.0
310
+ ],
311
+ [
312
+ 1.0,
313
+ 1.0,
314
+ 1.0,
315
+ 1.0,
316
+ 1.0,
317
+ 1.0
318
+ ]
319
+ ],
320
+ "significant_cells": [
321
+ [
322
+ false,
323
+ false,
324
+ false,
325
+ false,
326
+ false,
327
+ false
328
+ ],
329
+ [
330
+ false,
331
+ false,
332
+ false,
333
+ false,
334
+ false,
335
+ false
336
+ ],
337
+ [
338
+ false,
339
+ false,
340
+ false,
341
+ false,
342
+ false,
343
+ false
344
+ ],
345
+ [
346
+ false,
347
+ false,
348
+ false,
349
+ false,
350
+ false,
351
+ false
352
+ ],
353
+ [
354
+ false,
355
+ false,
356
+ false,
357
+ false,
358
+ false,
359
+ false
360
+ ],
361
+ [
362
+ false,
363
+ false,
364
+ false,
365
+ false,
366
+ false,
367
+ false
368
+ ],
369
+ [
370
+ false,
371
+ false,
372
+ false,
373
+ false,
374
+ false,
375
+ false
376
+ ],
377
+ [
378
+ false,
379
+ false,
380
+ false,
381
+ false,
382
+ false,
383
+ false
384
+ ],
385
+ [
386
+ false,
387
+ false,
388
+ false,
389
+ false,
390
+ false,
391
+ false
392
+ ],
393
+ [
394
+ false,
395
+ false,
396
+ false,
397
+ false,
398
+ false,
399
+ false
400
+ ],
401
+ [
402
+ false,
403
+ false,
404
+ false,
405
+ false,
406
+ false,
407
+ false
408
+ ],
409
+ [
410
+ false,
411
+ false,
412
+ false,
413
+ false,
414
+ false,
415
+ false
416
+ ],
417
+ [
418
+ false,
419
+ false,
420
+ false,
421
+ false,
422
+ false,
423
+ false
424
+ ]
425
+ ],
426
+ "peak_layer": 0,
427
+ "peak_position": 3,
428
+ "peak_accuracy": 0.5537,
429
+ "nonlinear_peak_layer": 0,
430
+ "nonlinear_peak_position": 0,
431
+ "nonlinear_peak_accuracy": 0.0,
432
+ "nonlinear_advantage_cells": 0,
433
+ "diagonal_pattern": true,
434
+ "diagonal_peak_layers": [
435
+ 8,
436
+ 12,
437
+ 12,
438
+ 0
439
+ ],
440
+ "n_samples_per_position": {
441
+ "0": 500,
442
+ "1": 500,
443
+ "2": 500,
444
+ "3": 298,
445
+ "4": 81,
446
+ "5": 12
447
+ },
448
+ "n_significant_cells": 0,
449
+ "total_cells": 78,
450
+ "selectivity": [
451
+ [
452
+ 0.0,
453
+ 0.0,
454
+ 0.0,
455
+ 0.0,
456
+ 0.0,
457
+ 0.0
458
+ ],
459
+ [
460
+ 0.0,
461
+ 0.0,
462
+ 0.0,
463
+ 0.0,
464
+ 0.0,
465
+ 0.0
466
+ ],
467
+ [
468
+ 0.0,
469
+ 0.0,
470
+ 0.0,
471
+ 0.0,
472
+ 0.0,
473
+ 0.0
474
+ ],
475
+ [
476
+ 0.0,
477
+ 0.0,
478
+ 0.0,
479
+ 0.0,
480
+ 0.0,
481
+ 0.0
482
+ ],
483
+ [
484
+ 0.0,
485
+ 0.0,
486
+ 0.0,
487
+ 0.0,
488
+ 0.0,
489
+ 0.0
490
+ ],
491
+ [
492
+ 0.0,
493
+ 0.0,
494
+ 0.0,
495
+ 0.0,
496
+ 0.0,
497
+ 0.0
498
+ ],
499
+ [
500
+ 0.0,
501
+ 0.0,
502
+ 0.0,
503
+ 0.0,
504
+ 0.0,
505
+ 0.0
506
+ ],
507
+ [
508
+ 0.0,
509
+ 0.0,
510
+ 0.0,
511
+ 0.0,
512
+ 0.0,
513
+ 0.0
514
+ ],
515
+ [
516
+ 0.0,
517
+ 0.0,
518
+ 0.0,
519
+ 0.0,
520
+ 0.0,
521
+ 0.0
522
+ ],
523
+ [
524
+ 0.0,
525
+ 0.0,
526
+ 0.0,
527
+ 0.0,
528
+ 0.0,
529
+ 0.0
530
+ ],
531
+ [
532
+ 0.0,
533
+ 0.0,
534
+ 0.0,
535
+ 0.0,
536
+ 0.0,
537
+ 0.0
538
+ ],
539
+ [
540
+ 0.0,
541
+ 0.0,
542
+ 0.0,
543
+ 0.0,
544
+ 0.0,
545
+ 0.0
546
+ ],
547
+ [
548
+ 0.0,
549
+ 0.0,
550
+ 0.0,
551
+ 0.0,
552
+ 0.0,
553
+ 0.0
554
+ ]
555
+ ],
556
+ "mean_selectivity": 0.0,
557
+ "max_cross_position_acc": 0.0,
558
+ "cross_leakage_front_to_back": 0.0,
559
+ "input_position_probe_acc": [
560
+ [
561
+ 0.042,
562
+ 0.014000000000000002,
563
+ 0.028000000000000004,
564
+ 0.020134228187919462,
565
+ 0.0,
566
+ 0.0
567
+ ],
568
+ [
569
+ 0.042,
570
+ 0.018000000000000002,
571
+ 0.027999999999999997,
572
+ 0.023489932885906038,
573
+ 0.0,
574
+ 0.0
575
+ ],
576
+ [
577
+ 0.04,
578
+ 0.022,
579
+ 0.03,
580
+ 0.013422818791946308,
581
+ 0.0,
582
+ 0.0
583
+ ],
584
+ [
585
+ 0.046,
586
+ 0.03,
587
+ 0.02,
588
+ 0.020134228187919462,
589
+ 0.0,
590
+ 0.0
591
+ ],
592
+ [
593
+ 0.042,
594
+ 0.042,
595
+ 0.032,
596
+ 0.02348993288590604,
597
+ 0.0,
598
+ 0.0
599
+ ],
600
+ [
601
+ 0.044000000000000004,
602
+ 0.04,
603
+ 0.028000000000000004,
604
+ 0.013422818791946308,
605
+ 0.0,
606
+ 0.0
607
+ ],
608
+ [
609
+ 0.046,
610
+ 0.04,
611
+ 0.026000000000000002,
612
+ 0.013422818791946308,
613
+ 0.0,
614
+ 0.0
615
+ ],
616
+ [
617
+ 0.05,
618
+ 0.04,
619
+ 0.028000000000000004,
620
+ 0.016778523489932886,
621
+ 0.0,
622
+ 0.0
623
+ ],
624
+ [
625
+ 0.041999999999999996,
626
+ 0.030000000000000006,
627
+ 0.032,
628
+ 0.010067114093959731,
629
+ 0.0,
630
+ 0.0
631
+ ],
632
+ [
633
+ 0.046,
634
+ 0.022,
635
+ 0.024,
636
+ 0.010067114093959731,
637
+ 0.0,
638
+ 0.0
639
+ ],
640
+ [
641
+ 0.044000000000000004,
642
+ 0.034,
643
+ 0.038,
644
+ 0.003355704697986577,
645
+ 0.0,
646
+ 0.0
647
+ ],
648
+ [
649
+ 0.044000000000000004,
650
+ 0.038,
651
+ 0.034,
652
+ 0.013422818791946308,
653
+ 0.0,
654
+ 0.0
655
+ ],
656
+ [
657
+ 0.036000000000000004,
658
+ 0.028000000000000004,
659
+ 0.026000000000000002,
660
+ 0.013422818791946308,
661
+ 0.0,
662
+ 0.0
663
+ ]
664
+ ],
665
+ "thought_minus_input_advantage": [
666
+ [
667
+ 0.04399999999999999,
668
+ 0.07200000000000001,
669
+ 0.158,
670
+ 0.5335570469798657,
671
+ 0.0,
672
+ 0.0
673
+ ],
674
+ [
675
+ 0.046000000000000006,
676
+ 0.07,
677
+ 0.132,
678
+ 0.5234899328859061,
679
+ 0.0,
680
+ 0.0
681
+ ],
682
+ [
683
+ 0.047999999999999994,
684
+ 0.058,
685
+ 0.154,
686
+ 0.476510067114094,
687
+ 0.0,
688
+ 0.0
689
+ ],
690
+ [
691
+ 0.034,
692
+ 0.041999999999999996,
693
+ 0.126,
694
+ 0.44630872483221473,
695
+ 0.0,
696
+ 0.0
697
+ ],
698
+ [
699
+ 0.047999999999999994,
700
+ 0.01800000000000001,
701
+ 0.11199999999999999,
702
+ 0.436241610738255,
703
+ 0.0,
704
+ 0.0
705
+ ],
706
+ [
707
+ 0.031999999999999994,
708
+ 0.033999999999999996,
709
+ 0.11199999999999999,
710
+ 0.4161073825503356,
711
+ 0.0,
712
+ 0.0
713
+ ],
714
+ [
715
+ 0.052000000000000005,
716
+ 0.017999999999999995,
717
+ 0.11200000000000002,
718
+ 0.3825503355704698,
719
+ 0.0,
720
+ 0.0
721
+ ],
722
+ [
723
+ 0.022000000000000006,
724
+ 0.024000000000000014,
725
+ 0.094,
726
+ 0.36241610738255037,
727
+ 0.0,
728
+ 0.0
729
+ ],
730
+ [
731
+ 0.08800000000000001,
732
+ 0.03599999999999998,
733
+ 0.098,
734
+ 0.24832214765100674,
735
+ 0.0,
736
+ 0.0
737
+ ],
738
+ [
739
+ 0.04400000000000001,
740
+ 0.054,
741
+ 0.094,
742
+ 0.2684563758389262,
743
+ 0.0,
744
+ 0.0
745
+ ],
746
+ [
747
+ 0.014000000000000005,
748
+ 0.06,
749
+ 0.10200000000000001,
750
+ 0.32550335570469796,
751
+ 0.0,
752
+ 0.0
753
+ ],
754
+ [
755
+ 0.031999999999999994,
756
+ 0.056000000000000015,
757
+ 0.14,
758
+ 0.5134228187919463,
759
+ 0.0,
760
+ 0.0
761
+ ],
762
+ [
763
+ 0.018000000000000002,
764
+ 0.07200000000000001,
765
+ 0.164,
766
+ 0.5369127516778524,
767
+ 0.0,
768
+ 0.0
769
+ ]
770
+ ],
771
+ "mean_thought_advantage": 0.1052,
772
+ "max_input_position_acc": 0.05
773
+ }
experiments/probing/m5_probing.json ADDED
@@ -0,0 +1,773 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linear_probe_accuracy": [
3
+ [
4
+ 0.031999999999999994,
5
+ 0.032,
6
+ 0.044000000000000004,
7
+ 0.0436241610738255,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ [
12
+ 0.036,
13
+ 0.018000000000000002,
14
+ 0.05,
15
+ 0.06375838926174497,
16
+ 0.0,
17
+ 0.0
18
+ ],
19
+ [
20
+ 0.057999999999999996,
21
+ 0.030000000000000006,
22
+ 0.044000000000000004,
23
+ 0.050335570469798654,
24
+ 0.0,
25
+ 0.0
26
+ ],
27
+ [
28
+ 0.049999999999999996,
29
+ 0.03,
30
+ 0.042,
31
+ 0.0738255033557047,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ [
36
+ 0.052000000000000005,
37
+ 0.036000000000000004,
38
+ 0.044,
39
+ 0.08389261744966443,
40
+ 0.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.06799999999999999,
45
+ 0.030000000000000006,
46
+ 0.05800000000000001,
47
+ 0.11409395973154363,
48
+ 0.0,
49
+ 0.0
50
+ ],
51
+ [
52
+ 0.07400000000000001,
53
+ 0.05800000000000001,
54
+ 0.046000000000000006,
55
+ 0.08389261744966442,
56
+ 0.0,
57
+ 0.0
58
+ ],
59
+ [
60
+ 0.10400000000000001,
61
+ 0.04,
62
+ 0.034,
63
+ 0.04697986577181208,
64
+ 0.0,
65
+ 0.0
66
+ ],
67
+ [
68
+ 0.236,
69
+ 0.054000000000000006,
70
+ 0.038,
71
+ 0.026845637583892617,
72
+ 0.0,
73
+ 0.0
74
+ ],
75
+ [
76
+ 0.14600000000000002,
77
+ 0.092,
78
+ 0.06799999999999999,
79
+ 0.030201342281879193,
80
+ 0.0,
81
+ 0.0
82
+ ],
83
+ [
84
+ 0.11000000000000001,
85
+ 0.09,
86
+ 0.06599999999999999,
87
+ 0.04697986577181208,
88
+ 0.0,
89
+ 0.0
90
+ ],
91
+ [
92
+ 0.09999999999999999,
93
+ 0.10400000000000001,
94
+ 0.21600000000000003,
95
+ 0.5302013422818792,
96
+ 0.0,
97
+ 0.0
98
+ ],
99
+ [
100
+ 0.10200000000000001,
101
+ 0.078,
102
+ 0.21999999999999997,
103
+ 0.5704697986577181,
104
+ 0.0,
105
+ 0.0
106
+ ]
107
+ ],
108
+ "nonlinear_probe_accuracy": [
109
+ [
110
+ 0.0,
111
+ 0.0,
112
+ 0.0,
113
+ 0.0,
114
+ 0.0,
115
+ 0.0
116
+ ],
117
+ [
118
+ 0.0,
119
+ 0.0,
120
+ 0.0,
121
+ 0.0,
122
+ 0.0,
123
+ 0.0
124
+ ],
125
+ [
126
+ 0.0,
127
+ 0.0,
128
+ 0.0,
129
+ 0.0,
130
+ 0.0,
131
+ 0.0
132
+ ],
133
+ [
134
+ 0.0,
135
+ 0.0,
136
+ 0.0,
137
+ 0.0,
138
+ 0.0,
139
+ 0.0
140
+ ],
141
+ [
142
+ 0.0,
143
+ 0.0,
144
+ 0.0,
145
+ 0.0,
146
+ 0.0,
147
+ 0.0
148
+ ],
149
+ [
150
+ 0.0,
151
+ 0.0,
152
+ 0.0,
153
+ 0.0,
154
+ 0.0,
155
+ 0.0
156
+ ],
157
+ [
158
+ 0.0,
159
+ 0.0,
160
+ 0.0,
161
+ 0.0,
162
+ 0.0,
163
+ 0.0
164
+ ],
165
+ [
166
+ 0.0,
167
+ 0.0,
168
+ 0.0,
169
+ 0.0,
170
+ 0.0,
171
+ 0.0
172
+ ],
173
+ [
174
+ 0.0,
175
+ 0.0,
176
+ 0.0,
177
+ 0.0,
178
+ 0.0,
179
+ 0.0
180
+ ],
181
+ [
182
+ 0.0,
183
+ 0.0,
184
+ 0.0,
185
+ 0.0,
186
+ 0.0,
187
+ 0.0
188
+ ],
189
+ [
190
+ 0.0,
191
+ 0.0,
192
+ 0.0,
193
+ 0.0,
194
+ 0.0,
195
+ 0.0
196
+ ],
197
+ [
198
+ 0.0,
199
+ 0.0,
200
+ 0.0,
201
+ 0.0,
202
+ 0.0,
203
+ 0.0
204
+ ],
205
+ [
206
+ 0.0,
207
+ 0.0,
208
+ 0.0,
209
+ 0.0,
210
+ 0.0,
211
+ 0.0
212
+ ]
213
+ ],
214
+ "permutation_p_values": [
215
+ [
216
+ 1.0,
217
+ 1.0,
218
+ 1.0,
219
+ 1.0,
220
+ 1.0,
221
+ 1.0
222
+ ],
223
+ [
224
+ 1.0,
225
+ 1.0,
226
+ 1.0,
227
+ 1.0,
228
+ 1.0,
229
+ 1.0
230
+ ],
231
+ [
232
+ 1.0,
233
+ 1.0,
234
+ 1.0,
235
+ 1.0,
236
+ 1.0,
237
+ 1.0
238
+ ],
239
+ [
240
+ 1.0,
241
+ 1.0,
242
+ 1.0,
243
+ 1.0,
244
+ 1.0,
245
+ 1.0
246
+ ],
247
+ [
248
+ 1.0,
249
+ 1.0,
250
+ 1.0,
251
+ 1.0,
252
+ 1.0,
253
+ 1.0
254
+ ],
255
+ [
256
+ 1.0,
257
+ 1.0,
258
+ 1.0,
259
+ 1.0,
260
+ 1.0,
261
+ 1.0
262
+ ],
263
+ [
264
+ 1.0,
265
+ 1.0,
266
+ 1.0,
267
+ 1.0,
268
+ 1.0,
269
+ 1.0
270
+ ],
271
+ [
272
+ 1.0,
273
+ 1.0,
274
+ 1.0,
275
+ 1.0,
276
+ 1.0,
277
+ 1.0
278
+ ],
279
+ [
280
+ 1.0,
281
+ 1.0,
282
+ 1.0,
283
+ 1.0,
284
+ 1.0,
285
+ 1.0
286
+ ],
287
+ [
288
+ 1.0,
289
+ 1.0,
290
+ 1.0,
291
+ 1.0,
292
+ 1.0,
293
+ 1.0
294
+ ],
295
+ [
296
+ 1.0,
297
+ 1.0,
298
+ 1.0,
299
+ 1.0,
300
+ 1.0,
301
+ 1.0
302
+ ],
303
+ [
304
+ 1.0,
305
+ 1.0,
306
+ 1.0,
307
+ 1.0,
308
+ 1.0,
309
+ 1.0
310
+ ],
311
+ [
312
+ 1.0,
313
+ 1.0,
314
+ 1.0,
315
+ 1.0,
316
+ 1.0,
317
+ 1.0
318
+ ]
319
+ ],
320
+ "significant_cells": [
321
+ [
322
+ false,
323
+ false,
324
+ false,
325
+ false,
326
+ false,
327
+ false
328
+ ],
329
+ [
330
+ false,
331
+ false,
332
+ false,
333
+ false,
334
+ false,
335
+ false
336
+ ],
337
+ [
338
+ false,
339
+ false,
340
+ false,
341
+ false,
342
+ false,
343
+ false
344
+ ],
345
+ [
346
+ false,
347
+ false,
348
+ false,
349
+ false,
350
+ false,
351
+ false
352
+ ],
353
+ [
354
+ false,
355
+ false,
356
+ false,
357
+ false,
358
+ false,
359
+ false
360
+ ],
361
+ [
362
+ false,
363
+ false,
364
+ false,
365
+ false,
366
+ false,
367
+ false
368
+ ],
369
+ [
370
+ false,
371
+ false,
372
+ false,
373
+ false,
374
+ false,
375
+ false
376
+ ],
377
+ [
378
+ false,
379
+ false,
380
+ false,
381
+ false,
382
+ false,
383
+ false
384
+ ],
385
+ [
386
+ false,
387
+ false,
388
+ false,
389
+ false,
390
+ false,
391
+ false
392
+ ],
393
+ [
394
+ false,
395
+ false,
396
+ false,
397
+ false,
398
+ false,
399
+ false
400
+ ],
401
+ [
402
+ false,
403
+ false,
404
+ false,
405
+ false,
406
+ false,
407
+ false
408
+ ],
409
+ [
410
+ false,
411
+ false,
412
+ false,
413
+ false,
414
+ false,
415
+ false
416
+ ],
417
+ [
418
+ false,
419
+ false,
420
+ false,
421
+ false,
422
+ false,
423
+ false
424
+ ]
425
+ ],
426
+ "peak_layer": 12,
427
+ "peak_position": 3,
428
+ "peak_accuracy": 0.5705,
429
+ "nonlinear_peak_layer": 0,
430
+ "nonlinear_peak_position": 0,
431
+ "nonlinear_peak_accuracy": 0.0,
432
+ "nonlinear_advantage_cells": 0,
433
+ "diagonal_pattern": true,
434
+ "diagonal_peak_layers": [
435
+ 8,
436
+ 11,
437
+ 12,
438
+ 12
439
+ ],
440
+ "n_samples_per_position": {
441
+ "0": 500,
442
+ "1": 500,
443
+ "2": 500,
444
+ "3": 298,
445
+ "4": 81,
446
+ "5": 12
447
+ },
448
+ "n_significant_cells": 0,
449
+ "total_cells": 78,
450
+ "selectivity": [
451
+ [
452
+ 0.0,
453
+ 0.0,
454
+ 0.0,
455
+ 0.0,
456
+ 0.0,
457
+ 0.0
458
+ ],
459
+ [
460
+ 0.0,
461
+ 0.0,
462
+ 0.0,
463
+ 0.0,
464
+ 0.0,
465
+ 0.0
466
+ ],
467
+ [
468
+ 0.0,
469
+ 0.0,
470
+ 0.0,
471
+ 0.0,
472
+ 0.0,
473
+ 0.0
474
+ ],
475
+ [
476
+ 0.0,
477
+ 0.0,
478
+ 0.0,
479
+ 0.0,
480
+ 0.0,
481
+ 0.0
482
+ ],
483
+ [
484
+ 0.0,
485
+ 0.0,
486
+ 0.0,
487
+ 0.0,
488
+ 0.0,
489
+ 0.0
490
+ ],
491
+ [
492
+ 0.0,
493
+ 0.0,
494
+ 0.0,
495
+ 0.0,
496
+ 0.0,
497
+ 0.0
498
+ ],
499
+ [
500
+ 0.0,
501
+ 0.0,
502
+ 0.0,
503
+ 0.0,
504
+ 0.0,
505
+ 0.0
506
+ ],
507
+ [
508
+ 0.0,
509
+ 0.0,
510
+ 0.0,
511
+ 0.0,
512
+ 0.0,
513
+ 0.0
514
+ ],
515
+ [
516
+ 0.0,
517
+ 0.0,
518
+ 0.0,
519
+ 0.0,
520
+ 0.0,
521
+ 0.0
522
+ ],
523
+ [
524
+ 0.0,
525
+ 0.0,
526
+ 0.0,
527
+ 0.0,
528
+ 0.0,
529
+ 0.0
530
+ ],
531
+ [
532
+ 0.0,
533
+ 0.0,
534
+ 0.0,
535
+ 0.0,
536
+ 0.0,
537
+ 0.0
538
+ ],
539
+ [
540
+ 0.0,
541
+ 0.0,
542
+ 0.0,
543
+ 0.0,
544
+ 0.0,
545
+ 0.0
546
+ ],
547
+ [
548
+ 0.0,
549
+ 0.0,
550
+ 0.0,
551
+ 0.0,
552
+ 0.0,
553
+ 0.0
554
+ ]
555
+ ],
556
+ "mean_selectivity": 0.0,
557
+ "max_cross_position_acc": 0.0,
558
+ "cross_leakage_front_to_back": 0.0,
559
+ "input_position_probe_acc": [
560
+ [
561
+ 0.05,
562
+ 0.018,
563
+ 0.028000000000000004,
564
+ 0.013422818791946308,
565
+ 0.0,
566
+ 0.0
567
+ ],
568
+ [
569
+ 0.041999999999999996,
570
+ 0.016,
571
+ 0.022,
572
+ 0.013422818791946308,
573
+ 0.0,
574
+ 0.0
575
+ ],
576
+ [
577
+ 0.038,
578
+ 0.016,
579
+ 0.038,
580
+ 0.016778523489932886,
581
+ 0.0,
582
+ 0.0
583
+ ],
584
+ [
585
+ 0.04,
586
+ 0.026000000000000002,
587
+ 0.026000000000000002,
588
+ 0.026845637583892617,
589
+ 0.0,
590
+ 0.0
591
+ ],
592
+ [
593
+ 0.052000000000000005,
594
+ 0.028000000000000004,
595
+ 0.028000000000000004,
596
+ 0.02348993288590604,
597
+ 0.0,
598
+ 0.0
599
+ ],
600
+ [
601
+ 0.05,
602
+ 0.033999999999999996,
603
+ 0.026000000000000002,
604
+ 0.020134228187919462,
605
+ 0.0,
606
+ 0.0
607
+ ],
608
+ [
609
+ 0.041999999999999996,
610
+ 0.032,
611
+ 0.016,
612
+ 0.020134228187919462,
613
+ 0.0,
614
+ 0.0
615
+ ],
616
+ [
617
+ 0.048,
618
+ 0.036,
619
+ 0.012,
620
+ 0.010067114093959731,
621
+ 0.0,
622
+ 0.0
623
+ ],
624
+ [
625
+ 0.042,
626
+ 0.036,
627
+ 0.024,
628
+ 0.010067114093959731,
629
+ 0.0,
630
+ 0.0
631
+ ],
632
+ [
633
+ 0.046,
634
+ 0.04,
635
+ 0.032,
636
+ 0.003355704697986577,
637
+ 0.0,
638
+ 0.0
639
+ ],
640
+ [
641
+ 0.046,
642
+ 0.028000000000000004,
643
+ 0.013999999999999999,
644
+ 0.010067114093959731,
645
+ 0.0,
646
+ 0.0
647
+ ],
648
+ [
649
+ 0.044,
650
+ 0.032,
651
+ 0.028000000000000004,
652
+ 0.016778523489932886,
653
+ 0.0,
654
+ 0.0
655
+ ],
656
+ [
657
+ 0.062,
658
+ 0.03,
659
+ 0.018000000000000002,
660
+ 0.03355704697986577,
661
+ 0.0,
662
+ 0.0
663
+ ]
664
+ ],
665
+ "thought_minus_input_advantage": [
666
+ [
667
+ -0.01800000000000001,
668
+ 0.014000000000000002,
669
+ 0.016,
670
+ 0.030201342281879193,
671
+ 0.0,
672
+ 0.0
673
+ ],
674
+ [
675
+ -0.005999999999999998,
676
+ 0.0020000000000000018,
677
+ 0.028000000000000004,
678
+ 0.05033557046979866,
679
+ 0.0,
680
+ 0.0
681
+ ],
682
+ [
683
+ 0.019999999999999997,
684
+ 0.014000000000000005,
685
+ 0.006000000000000005,
686
+ 0.033557046979865765,
687
+ 0.0,
688
+ 0.0
689
+ ],
690
+ [
691
+ 0.009999999999999995,
692
+ 0.003999999999999997,
693
+ 0.016,
694
+ 0.04697986577181208,
695
+ 0.0,
696
+ 0.0
697
+ ],
698
+ [
699
+ 0.0,
700
+ 0.008,
701
+ 0.015999999999999993,
702
+ 0.06040268456375839,
703
+ 0.0,
704
+ 0.0
705
+ ],
706
+ [
707
+ 0.017999999999999988,
708
+ -0.00399999999999999,
709
+ 0.03200000000000001,
710
+ 0.09395973154362416,
711
+ 0.0,
712
+ 0.0
713
+ ],
714
+ [
715
+ 0.032000000000000015,
716
+ 0.02600000000000001,
717
+ 0.030000000000000006,
718
+ 0.06375838926174496,
719
+ 0.0,
720
+ 0.0
721
+ ],
722
+ [
723
+ 0.05600000000000001,
724
+ 0.0040000000000000036,
725
+ 0.022000000000000002,
726
+ 0.03691275167785235,
727
+ 0.0,
728
+ 0.0
729
+ ],
730
+ [
731
+ 0.19399999999999998,
732
+ 0.01800000000000001,
733
+ 0.013999999999999999,
734
+ 0.016778523489932886,
735
+ 0.0,
736
+ 0.0
737
+ ],
738
+ [
739
+ 0.10000000000000002,
740
+ 0.052,
741
+ 0.03599999999999999,
742
+ 0.026845637583892617,
743
+ 0.0,
744
+ 0.0
745
+ ],
746
+ [
747
+ 0.06400000000000002,
748
+ 0.06199999999999999,
749
+ 0.05199999999999999,
750
+ 0.03691275167785235,
751
+ 0.0,
752
+ 0.0
753
+ ],
754
+ [
755
+ 0.055999999999999994,
756
+ 0.07200000000000001,
757
+ 0.18800000000000003,
758
+ 0.5134228187919463,
759
+ 0.0,
760
+ 0.0
761
+ ],
762
+ [
763
+ 0.04000000000000001,
764
+ 0.048,
765
+ 0.20199999999999996,
766
+ 0.5369127516778524,
767
+ 0.0,
768
+ 0.0
769
+ ]
770
+ ],
771
+ "mean_thought_advantage": 0.0396,
772
+ "max_input_position_acc": 0.062
773
+ }
experiments/probing/results.json ADDED
@@ -0,0 +1,1556 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "m3": {
3
+ "linear_probe_accuracy": [
4
+ [
5
+ 0.086,
6
+ 0.08600000000000001,
7
+ 0.186,
8
+ 0.5536912751677852,
9
+ 0.0,
10
+ 0.0
11
+ ],
12
+ [
13
+ 0.08800000000000001,
14
+ 0.08800000000000001,
15
+ 0.16,
16
+ 0.5469798657718121,
17
+ 0.0,
18
+ 0.0
19
+ ],
20
+ [
21
+ 0.088,
22
+ 0.08,
23
+ 0.184,
24
+ 0.4899328859060403,
25
+ 0.0,
26
+ 0.0
27
+ ],
28
+ [
29
+ 0.08,
30
+ 0.072,
31
+ 0.146,
32
+ 0.4664429530201342,
33
+ 0.0,
34
+ 0.0
35
+ ],
36
+ [
37
+ 0.09,
38
+ 0.06000000000000001,
39
+ 0.144,
40
+ 0.4597315436241611,
41
+ 0.0,
42
+ 0.0
43
+ ],
44
+ [
45
+ 0.076,
46
+ 0.074,
47
+ 0.13999999999999999,
48
+ 0.42953020134228187,
49
+ 0.0,
50
+ 0.0
51
+ ],
52
+ [
53
+ 0.098,
54
+ 0.057999999999999996,
55
+ 0.138,
56
+ 0.3959731543624161,
57
+ 0.0,
58
+ 0.0
59
+ ],
60
+ [
61
+ 0.07200000000000001,
62
+ 0.06400000000000002,
63
+ 0.122,
64
+ 0.37919463087248323,
65
+ 0.0,
66
+ 0.0
67
+ ],
68
+ [
69
+ 0.13,
70
+ 0.06599999999999999,
71
+ 0.13,
72
+ 0.25838926174496646,
73
+ 0.0,
74
+ 0.0
75
+ ],
76
+ [
77
+ 0.09000000000000001,
78
+ 0.076,
79
+ 0.118,
80
+ 0.2785234899328859,
81
+ 0.0,
82
+ 0.0
83
+ ],
84
+ [
85
+ 0.05800000000000001,
86
+ 0.094,
87
+ 0.14,
88
+ 0.32885906040268453,
89
+ 0.0,
90
+ 0.0
91
+ ],
92
+ [
93
+ 0.076,
94
+ 0.09400000000000001,
95
+ 0.17400000000000002,
96
+ 0.5268456375838926,
97
+ 0.0,
98
+ 0.0
99
+ ],
100
+ [
101
+ 0.054000000000000006,
102
+ 0.1,
103
+ 0.19,
104
+ 0.5503355704697986,
105
+ 0.0,
106
+ 0.0
107
+ ]
108
+ ],
109
+ "nonlinear_probe_accuracy": [
110
+ [
111
+ 0.0,
112
+ 0.0,
113
+ 0.0,
114
+ 0.0,
115
+ 0.0,
116
+ 0.0
117
+ ],
118
+ [
119
+ 0.0,
120
+ 0.0,
121
+ 0.0,
122
+ 0.0,
123
+ 0.0,
124
+ 0.0
125
+ ],
126
+ [
127
+ 0.0,
128
+ 0.0,
129
+ 0.0,
130
+ 0.0,
131
+ 0.0,
132
+ 0.0
133
+ ],
134
+ [
135
+ 0.0,
136
+ 0.0,
137
+ 0.0,
138
+ 0.0,
139
+ 0.0,
140
+ 0.0
141
+ ],
142
+ [
143
+ 0.0,
144
+ 0.0,
145
+ 0.0,
146
+ 0.0,
147
+ 0.0,
148
+ 0.0
149
+ ],
150
+ [
151
+ 0.0,
152
+ 0.0,
153
+ 0.0,
154
+ 0.0,
155
+ 0.0,
156
+ 0.0
157
+ ],
158
+ [
159
+ 0.0,
160
+ 0.0,
161
+ 0.0,
162
+ 0.0,
163
+ 0.0,
164
+ 0.0
165
+ ],
166
+ [
167
+ 0.0,
168
+ 0.0,
169
+ 0.0,
170
+ 0.0,
171
+ 0.0,
172
+ 0.0
173
+ ],
174
+ [
175
+ 0.0,
176
+ 0.0,
177
+ 0.0,
178
+ 0.0,
179
+ 0.0,
180
+ 0.0
181
+ ],
182
+ [
183
+ 0.0,
184
+ 0.0,
185
+ 0.0,
186
+ 0.0,
187
+ 0.0,
188
+ 0.0
189
+ ],
190
+ [
191
+ 0.0,
192
+ 0.0,
193
+ 0.0,
194
+ 0.0,
195
+ 0.0,
196
+ 0.0
197
+ ],
198
+ [
199
+ 0.0,
200
+ 0.0,
201
+ 0.0,
202
+ 0.0,
203
+ 0.0,
204
+ 0.0
205
+ ],
206
+ [
207
+ 0.0,
208
+ 0.0,
209
+ 0.0,
210
+ 0.0,
211
+ 0.0,
212
+ 0.0
213
+ ]
214
+ ],
215
+ "permutation_p_values": [
216
+ [
217
+ 1.0,
218
+ 1.0,
219
+ 1.0,
220
+ 1.0,
221
+ 1.0,
222
+ 1.0
223
+ ],
224
+ [
225
+ 1.0,
226
+ 1.0,
227
+ 1.0,
228
+ 1.0,
229
+ 1.0,
230
+ 1.0
231
+ ],
232
+ [
233
+ 1.0,
234
+ 1.0,
235
+ 1.0,
236
+ 1.0,
237
+ 1.0,
238
+ 1.0
239
+ ],
240
+ [
241
+ 1.0,
242
+ 1.0,
243
+ 1.0,
244
+ 1.0,
245
+ 1.0,
246
+ 1.0
247
+ ],
248
+ [
249
+ 1.0,
250
+ 1.0,
251
+ 1.0,
252
+ 1.0,
253
+ 1.0,
254
+ 1.0
255
+ ],
256
+ [
257
+ 1.0,
258
+ 1.0,
259
+ 1.0,
260
+ 1.0,
261
+ 1.0,
262
+ 1.0
263
+ ],
264
+ [
265
+ 1.0,
266
+ 1.0,
267
+ 1.0,
268
+ 1.0,
269
+ 1.0,
270
+ 1.0
271
+ ],
272
+ [
273
+ 1.0,
274
+ 1.0,
275
+ 1.0,
276
+ 1.0,
277
+ 1.0,
278
+ 1.0
279
+ ],
280
+ [
281
+ 1.0,
282
+ 1.0,
283
+ 1.0,
284
+ 1.0,
285
+ 1.0,
286
+ 1.0
287
+ ],
288
+ [
289
+ 1.0,
290
+ 1.0,
291
+ 1.0,
292
+ 1.0,
293
+ 1.0,
294
+ 1.0
295
+ ],
296
+ [
297
+ 1.0,
298
+ 1.0,
299
+ 1.0,
300
+ 1.0,
301
+ 1.0,
302
+ 1.0
303
+ ],
304
+ [
305
+ 1.0,
306
+ 1.0,
307
+ 1.0,
308
+ 1.0,
309
+ 1.0,
310
+ 1.0
311
+ ],
312
+ [
313
+ 1.0,
314
+ 1.0,
315
+ 1.0,
316
+ 1.0,
317
+ 1.0,
318
+ 1.0
319
+ ]
320
+ ],
321
+ "significant_cells": [
322
+ [
323
+ false,
324
+ false,
325
+ false,
326
+ false,
327
+ false,
328
+ false
329
+ ],
330
+ [
331
+ false,
332
+ false,
333
+ false,
334
+ false,
335
+ false,
336
+ false
337
+ ],
338
+ [
339
+ false,
340
+ false,
341
+ false,
342
+ false,
343
+ false,
344
+ false
345
+ ],
346
+ [
347
+ false,
348
+ false,
349
+ false,
350
+ false,
351
+ false,
352
+ false
353
+ ],
354
+ [
355
+ false,
356
+ false,
357
+ false,
358
+ false,
359
+ false,
360
+ false
361
+ ],
362
+ [
363
+ false,
364
+ false,
365
+ false,
366
+ false,
367
+ false,
368
+ false
369
+ ],
370
+ [
371
+ false,
372
+ false,
373
+ false,
374
+ false,
375
+ false,
376
+ false
377
+ ],
378
+ [
379
+ false,
380
+ false,
381
+ false,
382
+ false,
383
+ false,
384
+ false
385
+ ],
386
+ [
387
+ false,
388
+ false,
389
+ false,
390
+ false,
391
+ false,
392
+ false
393
+ ],
394
+ [
395
+ false,
396
+ false,
397
+ false,
398
+ false,
399
+ false,
400
+ false
401
+ ],
402
+ [
403
+ false,
404
+ false,
405
+ false,
406
+ false,
407
+ false,
408
+ false
409
+ ],
410
+ [
411
+ false,
412
+ false,
413
+ false,
414
+ false,
415
+ false,
416
+ false
417
+ ],
418
+ [
419
+ false,
420
+ false,
421
+ false,
422
+ false,
423
+ false,
424
+ false
425
+ ]
426
+ ],
427
+ "peak_layer": 0,
428
+ "peak_position": 3,
429
+ "peak_accuracy": 0.5537,
430
+ "nonlinear_peak_layer": 0,
431
+ "nonlinear_peak_position": 0,
432
+ "nonlinear_peak_accuracy": 0.0,
433
+ "nonlinear_advantage_cells": 0,
434
+ "diagonal_pattern": true,
435
+ "diagonal_peak_layers": [
436
+ 8,
437
+ 12,
438
+ 12,
439
+ 0
440
+ ],
441
+ "n_samples_per_position": {
442
+ "0": 500,
443
+ "1": 500,
444
+ "2": 500,
445
+ "3": 298,
446
+ "4": 81,
447
+ "5": 12
448
+ },
449
+ "n_significant_cells": 0,
450
+ "total_cells": 78,
451
+ "selectivity": [
452
+ [
453
+ 0.0,
454
+ 0.0,
455
+ 0.0,
456
+ 0.0,
457
+ 0.0,
458
+ 0.0
459
+ ],
460
+ [
461
+ 0.0,
462
+ 0.0,
463
+ 0.0,
464
+ 0.0,
465
+ 0.0,
466
+ 0.0
467
+ ],
468
+ [
469
+ 0.0,
470
+ 0.0,
471
+ 0.0,
472
+ 0.0,
473
+ 0.0,
474
+ 0.0
475
+ ],
476
+ [
477
+ 0.0,
478
+ 0.0,
479
+ 0.0,
480
+ 0.0,
481
+ 0.0,
482
+ 0.0
483
+ ],
484
+ [
485
+ 0.0,
486
+ 0.0,
487
+ 0.0,
488
+ 0.0,
489
+ 0.0,
490
+ 0.0
491
+ ],
492
+ [
493
+ 0.0,
494
+ 0.0,
495
+ 0.0,
496
+ 0.0,
497
+ 0.0,
498
+ 0.0
499
+ ],
500
+ [
501
+ 0.0,
502
+ 0.0,
503
+ 0.0,
504
+ 0.0,
505
+ 0.0,
506
+ 0.0
507
+ ],
508
+ [
509
+ 0.0,
510
+ 0.0,
511
+ 0.0,
512
+ 0.0,
513
+ 0.0,
514
+ 0.0
515
+ ],
516
+ [
517
+ 0.0,
518
+ 0.0,
519
+ 0.0,
520
+ 0.0,
521
+ 0.0,
522
+ 0.0
523
+ ],
524
+ [
525
+ 0.0,
526
+ 0.0,
527
+ 0.0,
528
+ 0.0,
529
+ 0.0,
530
+ 0.0
531
+ ],
532
+ [
533
+ 0.0,
534
+ 0.0,
535
+ 0.0,
536
+ 0.0,
537
+ 0.0,
538
+ 0.0
539
+ ],
540
+ [
541
+ 0.0,
542
+ 0.0,
543
+ 0.0,
544
+ 0.0,
545
+ 0.0,
546
+ 0.0
547
+ ],
548
+ [
549
+ 0.0,
550
+ 0.0,
551
+ 0.0,
552
+ 0.0,
553
+ 0.0,
554
+ 0.0
555
+ ]
556
+ ],
557
+ "mean_selectivity": 0.0,
558
+ "max_cross_position_acc": 0.0,
559
+ "cross_leakage_front_to_back": 0.0,
560
+ "input_position_probe_acc": [
561
+ [
562
+ 0.042,
563
+ 0.014000000000000002,
564
+ 0.028000000000000004,
565
+ 0.020134228187919462,
566
+ 0.0,
567
+ 0.0
568
+ ],
569
+ [
570
+ 0.042,
571
+ 0.018000000000000002,
572
+ 0.027999999999999997,
573
+ 0.023489932885906038,
574
+ 0.0,
575
+ 0.0
576
+ ],
577
+ [
578
+ 0.04,
579
+ 0.022,
580
+ 0.03,
581
+ 0.013422818791946308,
582
+ 0.0,
583
+ 0.0
584
+ ],
585
+ [
586
+ 0.046,
587
+ 0.03,
588
+ 0.02,
589
+ 0.020134228187919462,
590
+ 0.0,
591
+ 0.0
592
+ ],
593
+ [
594
+ 0.042,
595
+ 0.042,
596
+ 0.032,
597
+ 0.02348993288590604,
598
+ 0.0,
599
+ 0.0
600
+ ],
601
+ [
602
+ 0.044000000000000004,
603
+ 0.04,
604
+ 0.028000000000000004,
605
+ 0.013422818791946308,
606
+ 0.0,
607
+ 0.0
608
+ ],
609
+ [
610
+ 0.046,
611
+ 0.04,
612
+ 0.026000000000000002,
613
+ 0.013422818791946308,
614
+ 0.0,
615
+ 0.0
616
+ ],
617
+ [
618
+ 0.05,
619
+ 0.04,
620
+ 0.028000000000000004,
621
+ 0.016778523489932886,
622
+ 0.0,
623
+ 0.0
624
+ ],
625
+ [
626
+ 0.041999999999999996,
627
+ 0.030000000000000006,
628
+ 0.032,
629
+ 0.010067114093959731,
630
+ 0.0,
631
+ 0.0
632
+ ],
633
+ [
634
+ 0.046,
635
+ 0.022,
636
+ 0.024,
637
+ 0.010067114093959731,
638
+ 0.0,
639
+ 0.0
640
+ ],
641
+ [
642
+ 0.044000000000000004,
643
+ 0.034,
644
+ 0.038,
645
+ 0.003355704697986577,
646
+ 0.0,
647
+ 0.0
648
+ ],
649
+ [
650
+ 0.044000000000000004,
651
+ 0.038,
652
+ 0.034,
653
+ 0.013422818791946308,
654
+ 0.0,
655
+ 0.0
656
+ ],
657
+ [
658
+ 0.036000000000000004,
659
+ 0.028000000000000004,
660
+ 0.026000000000000002,
661
+ 0.013422818791946308,
662
+ 0.0,
663
+ 0.0
664
+ ]
665
+ ],
666
+ "thought_minus_input_advantage": [
667
+ [
668
+ 0.04399999999999999,
669
+ 0.07200000000000001,
670
+ 0.158,
671
+ 0.5335570469798657,
672
+ 0.0,
673
+ 0.0
674
+ ],
675
+ [
676
+ 0.046000000000000006,
677
+ 0.07,
678
+ 0.132,
679
+ 0.5234899328859061,
680
+ 0.0,
681
+ 0.0
682
+ ],
683
+ [
684
+ 0.047999999999999994,
685
+ 0.058,
686
+ 0.154,
687
+ 0.476510067114094,
688
+ 0.0,
689
+ 0.0
690
+ ],
691
+ [
692
+ 0.034,
693
+ 0.041999999999999996,
694
+ 0.126,
695
+ 0.44630872483221473,
696
+ 0.0,
697
+ 0.0
698
+ ],
699
+ [
700
+ 0.047999999999999994,
701
+ 0.01800000000000001,
702
+ 0.11199999999999999,
703
+ 0.436241610738255,
704
+ 0.0,
705
+ 0.0
706
+ ],
707
+ [
708
+ 0.031999999999999994,
709
+ 0.033999999999999996,
710
+ 0.11199999999999999,
711
+ 0.4161073825503356,
712
+ 0.0,
713
+ 0.0
714
+ ],
715
+ [
716
+ 0.052000000000000005,
717
+ 0.017999999999999995,
718
+ 0.11200000000000002,
719
+ 0.3825503355704698,
720
+ 0.0,
721
+ 0.0
722
+ ],
723
+ [
724
+ 0.022000000000000006,
725
+ 0.024000000000000014,
726
+ 0.094,
727
+ 0.36241610738255037,
728
+ 0.0,
729
+ 0.0
730
+ ],
731
+ [
732
+ 0.08800000000000001,
733
+ 0.03599999999999998,
734
+ 0.098,
735
+ 0.24832214765100674,
736
+ 0.0,
737
+ 0.0
738
+ ],
739
+ [
740
+ 0.04400000000000001,
741
+ 0.054,
742
+ 0.094,
743
+ 0.2684563758389262,
744
+ 0.0,
745
+ 0.0
746
+ ],
747
+ [
748
+ 0.014000000000000005,
749
+ 0.06,
750
+ 0.10200000000000001,
751
+ 0.32550335570469796,
752
+ 0.0,
753
+ 0.0
754
+ ],
755
+ [
756
+ 0.031999999999999994,
757
+ 0.056000000000000015,
758
+ 0.14,
759
+ 0.5134228187919463,
760
+ 0.0,
761
+ 0.0
762
+ ],
763
+ [
764
+ 0.018000000000000002,
765
+ 0.07200000000000001,
766
+ 0.164,
767
+ 0.5369127516778524,
768
+ 0.0,
769
+ 0.0
770
+ ]
771
+ ],
772
+ "mean_thought_advantage": 0.1052,
773
+ "max_input_position_acc": 0.05
774
+ },
775
+ "m5": {
776
+ "linear_probe_accuracy": [
777
+ [
778
+ 0.031999999999999994,
779
+ 0.032,
780
+ 0.044000000000000004,
781
+ 0.0436241610738255,
782
+ 0.0,
783
+ 0.0
784
+ ],
785
+ [
786
+ 0.036,
787
+ 0.018000000000000002,
788
+ 0.05,
789
+ 0.06375838926174497,
790
+ 0.0,
791
+ 0.0
792
+ ],
793
+ [
794
+ 0.057999999999999996,
795
+ 0.030000000000000006,
796
+ 0.044000000000000004,
797
+ 0.050335570469798654,
798
+ 0.0,
799
+ 0.0
800
+ ],
801
+ [
802
+ 0.049999999999999996,
803
+ 0.03,
804
+ 0.042,
805
+ 0.0738255033557047,
806
+ 0.0,
807
+ 0.0
808
+ ],
809
+ [
810
+ 0.052000000000000005,
811
+ 0.036000000000000004,
812
+ 0.044,
813
+ 0.08389261744966443,
814
+ 0.0,
815
+ 0.0
816
+ ],
817
+ [
818
+ 0.06799999999999999,
819
+ 0.030000000000000006,
820
+ 0.05800000000000001,
821
+ 0.11409395973154363,
822
+ 0.0,
823
+ 0.0
824
+ ],
825
+ [
826
+ 0.07400000000000001,
827
+ 0.05800000000000001,
828
+ 0.046000000000000006,
829
+ 0.08389261744966442,
830
+ 0.0,
831
+ 0.0
832
+ ],
833
+ [
834
+ 0.10400000000000001,
835
+ 0.04,
836
+ 0.034,
837
+ 0.04697986577181208,
838
+ 0.0,
839
+ 0.0
840
+ ],
841
+ [
842
+ 0.236,
843
+ 0.054000000000000006,
844
+ 0.038,
845
+ 0.026845637583892617,
846
+ 0.0,
847
+ 0.0
848
+ ],
849
+ [
850
+ 0.14600000000000002,
851
+ 0.092,
852
+ 0.06799999999999999,
853
+ 0.030201342281879193,
854
+ 0.0,
855
+ 0.0
856
+ ],
857
+ [
858
+ 0.11000000000000001,
859
+ 0.09,
860
+ 0.06599999999999999,
861
+ 0.04697986577181208,
862
+ 0.0,
863
+ 0.0
864
+ ],
865
+ [
866
+ 0.09999999999999999,
867
+ 0.10400000000000001,
868
+ 0.21600000000000003,
869
+ 0.5302013422818792,
870
+ 0.0,
871
+ 0.0
872
+ ],
873
+ [
874
+ 0.10200000000000001,
875
+ 0.078,
876
+ 0.21999999999999997,
877
+ 0.5704697986577181,
878
+ 0.0,
879
+ 0.0
880
+ ]
881
+ ],
882
+ "nonlinear_probe_accuracy": [
883
+ [
884
+ 0.0,
885
+ 0.0,
886
+ 0.0,
887
+ 0.0,
888
+ 0.0,
889
+ 0.0
890
+ ],
891
+ [
892
+ 0.0,
893
+ 0.0,
894
+ 0.0,
895
+ 0.0,
896
+ 0.0,
897
+ 0.0
898
+ ],
899
+ [
900
+ 0.0,
901
+ 0.0,
902
+ 0.0,
903
+ 0.0,
904
+ 0.0,
905
+ 0.0
906
+ ],
907
+ [
908
+ 0.0,
909
+ 0.0,
910
+ 0.0,
911
+ 0.0,
912
+ 0.0,
913
+ 0.0
914
+ ],
915
+ [
916
+ 0.0,
917
+ 0.0,
918
+ 0.0,
919
+ 0.0,
920
+ 0.0,
921
+ 0.0
922
+ ],
923
+ [
924
+ 0.0,
925
+ 0.0,
926
+ 0.0,
927
+ 0.0,
928
+ 0.0,
929
+ 0.0
930
+ ],
931
+ [
932
+ 0.0,
933
+ 0.0,
934
+ 0.0,
935
+ 0.0,
936
+ 0.0,
937
+ 0.0
938
+ ],
939
+ [
940
+ 0.0,
941
+ 0.0,
942
+ 0.0,
943
+ 0.0,
944
+ 0.0,
945
+ 0.0
946
+ ],
947
+ [
948
+ 0.0,
949
+ 0.0,
950
+ 0.0,
951
+ 0.0,
952
+ 0.0,
953
+ 0.0
954
+ ],
955
+ [
956
+ 0.0,
957
+ 0.0,
958
+ 0.0,
959
+ 0.0,
960
+ 0.0,
961
+ 0.0
962
+ ],
963
+ [
964
+ 0.0,
965
+ 0.0,
966
+ 0.0,
967
+ 0.0,
968
+ 0.0,
969
+ 0.0
970
+ ],
971
+ [
972
+ 0.0,
973
+ 0.0,
974
+ 0.0,
975
+ 0.0,
976
+ 0.0,
977
+ 0.0
978
+ ],
979
+ [
980
+ 0.0,
981
+ 0.0,
982
+ 0.0,
983
+ 0.0,
984
+ 0.0,
985
+ 0.0
986
+ ]
987
+ ],
988
+ "permutation_p_values": [
989
+ [
990
+ 1.0,
991
+ 1.0,
992
+ 1.0,
993
+ 1.0,
994
+ 1.0,
995
+ 1.0
996
+ ],
997
+ [
998
+ 1.0,
999
+ 1.0,
1000
+ 1.0,
1001
+ 1.0,
1002
+ 1.0,
1003
+ 1.0
1004
+ ],
1005
+ [
1006
+ 1.0,
1007
+ 1.0,
1008
+ 1.0,
1009
+ 1.0,
1010
+ 1.0,
1011
+ 1.0
1012
+ ],
1013
+ [
1014
+ 1.0,
1015
+ 1.0,
1016
+ 1.0,
1017
+ 1.0,
1018
+ 1.0,
1019
+ 1.0
1020
+ ],
1021
+ [
1022
+ 1.0,
1023
+ 1.0,
1024
+ 1.0,
1025
+ 1.0,
1026
+ 1.0,
1027
+ 1.0
1028
+ ],
1029
+ [
1030
+ 1.0,
1031
+ 1.0,
1032
+ 1.0,
1033
+ 1.0,
1034
+ 1.0,
1035
+ 1.0
1036
+ ],
1037
+ [
1038
+ 1.0,
1039
+ 1.0,
1040
+ 1.0,
1041
+ 1.0,
1042
+ 1.0,
1043
+ 1.0
1044
+ ],
1045
+ [
1046
+ 1.0,
1047
+ 1.0,
1048
+ 1.0,
1049
+ 1.0,
1050
+ 1.0,
1051
+ 1.0
1052
+ ],
1053
+ [
1054
+ 1.0,
1055
+ 1.0,
1056
+ 1.0,
1057
+ 1.0,
1058
+ 1.0,
1059
+ 1.0
1060
+ ],
1061
+ [
1062
+ 1.0,
1063
+ 1.0,
1064
+ 1.0,
1065
+ 1.0,
1066
+ 1.0,
1067
+ 1.0
1068
+ ],
1069
+ [
1070
+ 1.0,
1071
+ 1.0,
1072
+ 1.0,
1073
+ 1.0,
1074
+ 1.0,
1075
+ 1.0
1076
+ ],
1077
+ [
1078
+ 1.0,
1079
+ 1.0,
1080
+ 1.0,
1081
+ 1.0,
1082
+ 1.0,
1083
+ 1.0
1084
+ ],
1085
+ [
1086
+ 1.0,
1087
+ 1.0,
1088
+ 1.0,
1089
+ 1.0,
1090
+ 1.0,
1091
+ 1.0
1092
+ ]
1093
+ ],
1094
+ "significant_cells": [
1095
+ [
1096
+ false,
1097
+ false,
1098
+ false,
1099
+ false,
1100
+ false,
1101
+ false
1102
+ ],
1103
+ [
1104
+ false,
1105
+ false,
1106
+ false,
1107
+ false,
1108
+ false,
1109
+ false
1110
+ ],
1111
+ [
1112
+ false,
1113
+ false,
1114
+ false,
1115
+ false,
1116
+ false,
1117
+ false
1118
+ ],
1119
+ [
1120
+ false,
1121
+ false,
1122
+ false,
1123
+ false,
1124
+ false,
1125
+ false
1126
+ ],
1127
+ [
1128
+ false,
1129
+ false,
1130
+ false,
1131
+ false,
1132
+ false,
1133
+ false
1134
+ ],
1135
+ [
1136
+ false,
1137
+ false,
1138
+ false,
1139
+ false,
1140
+ false,
1141
+ false
1142
+ ],
1143
+ [
1144
+ false,
1145
+ false,
1146
+ false,
1147
+ false,
1148
+ false,
1149
+ false
1150
+ ],
1151
+ [
1152
+ false,
1153
+ false,
1154
+ false,
1155
+ false,
1156
+ false,
1157
+ false
1158
+ ],
1159
+ [
1160
+ false,
1161
+ false,
1162
+ false,
1163
+ false,
1164
+ false,
1165
+ false
1166
+ ],
1167
+ [
1168
+ false,
1169
+ false,
1170
+ false,
1171
+ false,
1172
+ false,
1173
+ false
1174
+ ],
1175
+ [
1176
+ false,
1177
+ false,
1178
+ false,
1179
+ false,
1180
+ false,
1181
+ false
1182
+ ],
1183
+ [
1184
+ false,
1185
+ false,
1186
+ false,
1187
+ false,
1188
+ false,
1189
+ false
1190
+ ],
1191
+ [
1192
+ false,
1193
+ false,
1194
+ false,
1195
+ false,
1196
+ false,
1197
+ false
1198
+ ]
1199
+ ],
1200
+ "peak_layer": 12,
1201
+ "peak_position": 3,
1202
+ "peak_accuracy": 0.5705,
1203
+ "nonlinear_peak_layer": 0,
1204
+ "nonlinear_peak_position": 0,
1205
+ "nonlinear_peak_accuracy": 0.0,
1206
+ "nonlinear_advantage_cells": 0,
1207
+ "diagonal_pattern": true,
1208
+ "diagonal_peak_layers": [
1209
+ 8,
1210
+ 11,
1211
+ 12,
1212
+ 12
1213
+ ],
1214
+ "n_samples_per_position": {
1215
+ "0": 500,
1216
+ "1": 500,
1217
+ "2": 500,
1218
+ "3": 298,
1219
+ "4": 81,
1220
+ "5": 12
1221
+ },
1222
+ "n_significant_cells": 0,
1223
+ "total_cells": 78,
1224
+ "selectivity": [
1225
+ [
1226
+ 0.0,
1227
+ 0.0,
1228
+ 0.0,
1229
+ 0.0,
1230
+ 0.0,
1231
+ 0.0
1232
+ ],
1233
+ [
1234
+ 0.0,
1235
+ 0.0,
1236
+ 0.0,
1237
+ 0.0,
1238
+ 0.0,
1239
+ 0.0
1240
+ ],
1241
+ [
1242
+ 0.0,
1243
+ 0.0,
1244
+ 0.0,
1245
+ 0.0,
1246
+ 0.0,
1247
+ 0.0
1248
+ ],
1249
+ [
1250
+ 0.0,
1251
+ 0.0,
1252
+ 0.0,
1253
+ 0.0,
1254
+ 0.0,
1255
+ 0.0
1256
+ ],
1257
+ [
1258
+ 0.0,
1259
+ 0.0,
1260
+ 0.0,
1261
+ 0.0,
1262
+ 0.0,
1263
+ 0.0
1264
+ ],
1265
+ [
1266
+ 0.0,
1267
+ 0.0,
1268
+ 0.0,
1269
+ 0.0,
1270
+ 0.0,
1271
+ 0.0
1272
+ ],
1273
+ [
1274
+ 0.0,
1275
+ 0.0,
1276
+ 0.0,
1277
+ 0.0,
1278
+ 0.0,
1279
+ 0.0
1280
+ ],
1281
+ [
1282
+ 0.0,
1283
+ 0.0,
1284
+ 0.0,
1285
+ 0.0,
1286
+ 0.0,
1287
+ 0.0
1288
+ ],
1289
+ [
1290
+ 0.0,
1291
+ 0.0,
1292
+ 0.0,
1293
+ 0.0,
1294
+ 0.0,
1295
+ 0.0
1296
+ ],
1297
+ [
1298
+ 0.0,
1299
+ 0.0,
1300
+ 0.0,
1301
+ 0.0,
1302
+ 0.0,
1303
+ 0.0
1304
+ ],
1305
+ [
1306
+ 0.0,
1307
+ 0.0,
1308
+ 0.0,
1309
+ 0.0,
1310
+ 0.0,
1311
+ 0.0
1312
+ ],
1313
+ [
1314
+ 0.0,
1315
+ 0.0,
1316
+ 0.0,
1317
+ 0.0,
1318
+ 0.0,
1319
+ 0.0
1320
+ ],
1321
+ [
1322
+ 0.0,
1323
+ 0.0,
1324
+ 0.0,
1325
+ 0.0,
1326
+ 0.0,
1327
+ 0.0
1328
+ ]
1329
+ ],
1330
+ "mean_selectivity": 0.0,
1331
+ "max_cross_position_acc": 0.0,
1332
+ "cross_leakage_front_to_back": 0.0,
1333
+ "input_position_probe_acc": [
1334
+ [
1335
+ 0.05,
1336
+ 0.018,
1337
+ 0.028000000000000004,
1338
+ 0.013422818791946308,
1339
+ 0.0,
1340
+ 0.0
1341
+ ],
1342
+ [
1343
+ 0.041999999999999996,
1344
+ 0.016,
1345
+ 0.022,
1346
+ 0.013422818791946308,
1347
+ 0.0,
1348
+ 0.0
1349
+ ],
1350
+ [
1351
+ 0.038,
1352
+ 0.016,
1353
+ 0.038,
1354
+ 0.016778523489932886,
1355
+ 0.0,
1356
+ 0.0
1357
+ ],
1358
+ [
1359
+ 0.04,
1360
+ 0.026000000000000002,
1361
+ 0.026000000000000002,
1362
+ 0.026845637583892617,
1363
+ 0.0,
1364
+ 0.0
1365
+ ],
1366
+ [
1367
+ 0.052000000000000005,
1368
+ 0.028000000000000004,
1369
+ 0.028000000000000004,
1370
+ 0.02348993288590604,
1371
+ 0.0,
1372
+ 0.0
1373
+ ],
1374
+ [
1375
+ 0.05,
1376
+ 0.033999999999999996,
1377
+ 0.026000000000000002,
1378
+ 0.020134228187919462,
1379
+ 0.0,
1380
+ 0.0
1381
+ ],
1382
+ [
1383
+ 0.041999999999999996,
1384
+ 0.032,
1385
+ 0.016,
1386
+ 0.020134228187919462,
1387
+ 0.0,
1388
+ 0.0
1389
+ ],
1390
+ [
1391
+ 0.048,
1392
+ 0.036,
1393
+ 0.012,
1394
+ 0.010067114093959731,
1395
+ 0.0,
1396
+ 0.0
1397
+ ],
1398
+ [
1399
+ 0.042,
1400
+ 0.036,
1401
+ 0.024,
1402
+ 0.010067114093959731,
1403
+ 0.0,
1404
+ 0.0
1405
+ ],
1406
+ [
1407
+ 0.046,
1408
+ 0.04,
1409
+ 0.032,
1410
+ 0.003355704697986577,
1411
+ 0.0,
1412
+ 0.0
1413
+ ],
1414
+ [
1415
+ 0.046,
1416
+ 0.028000000000000004,
1417
+ 0.013999999999999999,
1418
+ 0.010067114093959731,
1419
+ 0.0,
1420
+ 0.0
1421
+ ],
1422
+ [
1423
+ 0.044,
1424
+ 0.032,
1425
+ 0.028000000000000004,
1426
+ 0.016778523489932886,
1427
+ 0.0,
1428
+ 0.0
1429
+ ],
1430
+ [
1431
+ 0.062,
1432
+ 0.03,
1433
+ 0.018000000000000002,
1434
+ 0.03355704697986577,
1435
+ 0.0,
1436
+ 0.0
1437
+ ]
1438
+ ],
1439
+ "thought_minus_input_advantage": [
1440
+ [
1441
+ -0.01800000000000001,
1442
+ 0.014000000000000002,
1443
+ 0.016,
1444
+ 0.030201342281879193,
1445
+ 0.0,
1446
+ 0.0
1447
+ ],
1448
+ [
1449
+ -0.005999999999999998,
1450
+ 0.0020000000000000018,
1451
+ 0.028000000000000004,
1452
+ 0.05033557046979866,
1453
+ 0.0,
1454
+ 0.0
1455
+ ],
1456
+ [
1457
+ 0.019999999999999997,
1458
+ 0.014000000000000005,
1459
+ 0.006000000000000005,
1460
+ 0.033557046979865765,
1461
+ 0.0,
1462
+ 0.0
1463
+ ],
1464
+ [
1465
+ 0.009999999999999995,
1466
+ 0.003999999999999997,
1467
+ 0.016,
1468
+ 0.04697986577181208,
1469
+ 0.0,
1470
+ 0.0
1471
+ ],
1472
+ [
1473
+ 0.0,
1474
+ 0.008,
1475
+ 0.015999999999999993,
1476
+ 0.06040268456375839,
1477
+ 0.0,
1478
+ 0.0
1479
+ ],
1480
+ [
1481
+ 0.017999999999999988,
1482
+ -0.00399999999999999,
1483
+ 0.03200000000000001,
1484
+ 0.09395973154362416,
1485
+ 0.0,
1486
+ 0.0
1487
+ ],
1488
+ [
1489
+ 0.032000000000000015,
1490
+ 0.02600000000000001,
1491
+ 0.030000000000000006,
1492
+ 0.06375838926174496,
1493
+ 0.0,
1494
+ 0.0
1495
+ ],
1496
+ [
1497
+ 0.05600000000000001,
1498
+ 0.0040000000000000036,
1499
+ 0.022000000000000002,
1500
+ 0.03691275167785235,
1501
+ 0.0,
1502
+ 0.0
1503
+ ],
1504
+ [
1505
+ 0.19399999999999998,
1506
+ 0.01800000000000001,
1507
+ 0.013999999999999999,
1508
+ 0.016778523489932886,
1509
+ 0.0,
1510
+ 0.0
1511
+ ],
1512
+ [
1513
+ 0.10000000000000002,
1514
+ 0.052,
1515
+ 0.03599999999999999,
1516
+ 0.026845637583892617,
1517
+ 0.0,
1518
+ 0.0
1519
+ ],
1520
+ [
1521
+ 0.06400000000000002,
1522
+ 0.06199999999999999,
1523
+ 0.05199999999999999,
1524
+ 0.03691275167785235,
1525
+ 0.0,
1526
+ 0.0
1527
+ ],
1528
+ [
1529
+ 0.055999999999999994,
1530
+ 0.07200000000000001,
1531
+ 0.18800000000000003,
1532
+ 0.5134228187919463,
1533
+ 0.0,
1534
+ 0.0
1535
+ ],
1536
+ [
1537
+ 0.04000000000000001,
1538
+ 0.048,
1539
+ 0.20199999999999996,
1540
+ 0.5369127516778524,
1541
+ 0.0,
1542
+ 0.0
1543
+ ]
1544
+ ],
1545
+ "mean_thought_advantage": 0.0396,
1546
+ "max_input_position_acc": 0.062
1547
+ },
1548
+ "verification": {
1549
+ "synthetic_probe_accuracy": 1.0,
1550
+ "hidden_state_shape": [
1551
+ 768
1552
+ ],
1553
+ "all_finite": true,
1554
+ "label_alignment_check": "PASSED"
1555
+ }
1556
+ }
experiments/probing_corrected/m3_hidden_states.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8d68e45559b4d5fdbc5cf8896cdfd45c08420ed5f27ee26367d0870e4ee8631
3
+ size 70617413
experiments/probing_corrected/m3_labels.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"0": ["scrompus", "storpus", "sterpus", "vumpus", "jelpus", "bompus", "boompus", "brimpus", "sterpus", "bompus", "chorpus", "fompus", "rempus", "rompus", "sterpus", "rorpus", "numpus", "dumpus", "storpus", "brimpus", "gerpus", "rompus", "jompus", "numpus", "rompus", "terpus", "sterpus", "fompus", "storpus", "dumpus", "scrompus", "quimpus", "yerpus", "jelpus", "gorpus", "tumpus", "shumpus", "chorpus", "yerpus", "vumpus", "impus", "scrompus", "zumpus", "rorpus", "rempus", "fompus", "yumpus", "rorpus", "bompus", "dumpus", "gwompus", "felpus", "chorpus", "wumpus", "jompus", "felpus", "grimpus", "wumpus", "fompus", "wumpus", "gwompus", "gwompus", "bompus", "vumpus", "lorpus", "worpus", "yumpus", "fompus", "gorpus", "scrompus", "jompus", "brimpus", "yumpus", "worpus", "rempus", "jelpus", "fompus", "scrompus", "jompus", "rompus", "grimpus", "rompus", "terpus", "gerpus", "yumpus", "storpus", "gwompus", "lorpus", "lorpus", "rompus", "timpus", "grimpus", "tumpus", "rorpus", "gwompus", "gorpus", "numpus", "boompus", "jelpus", "grimpus", "dumpus", "lorpus", "rempus", "jompus", "bompus", "sterpus", "scrompus", "lempus", "grimpus", "quimpus", "hilpus", "jelpus", "worpus", "shumpus", "jompus", "zumpus", "quimpus", "gerpus", "grimpus", "yimpus", "zhorpus", "lempus", "zumpus", "zhorpus", "zumpus", "zumpus", "wumpus", "rompus", "shumpus", "quimpus", "chorpus", "grimpus", "sterpus", "sterpus", "lempus", "yimpus", "gerpus", "zumpus", "sterpus", "worpus", "rorpus", "yimpus", "jelpus", "storpus", "dumpus", "fompus", "brimpus", "terpus", "storpus", "gerpus", "timpus", "vumpus", "sterpus", "rorpus", "gerpus", "vumpus", "fompus", "bompus", "yerpus", "yerpus", "jelpus", "sterpus", "yumpus", "shumpus", "tumpus", "rompus", "dumpus", "storpus", "lempus", "rempus", "lempus", "gorpus", "lorpus", "lorpus", "bompus", "numpus", "felpus", "jompus", "rempus", "boompus", "brimpus", "sterpus", "dumpus", "wumpus", "vumpus", "vumpus", "felpus", "timpus", "boompus", "dumpus", "bompus", "boompus", "brimpus", "shumpus", "hilpus", "lempus", "impus", "rompus", "boompus", "terpus", "gorpus", "tumpus", "numpus", "kerpus", "gerpus", "scrompus", "grimpus", "lempus", "grimpus", "timpus", "chorpus", "hilpus", "vumpus", "dumpus", "gorpus", "gerpus", "lempus", "yimpus", "fompus", "yumpus", "shumpus", "lorpus", "fompus", "kerpus", "jompus", "storpus", "timpus", "kerpus", "sterpus", "rompus", "yimpus", "brimpus", "storpus", "boompus", "grimpus", "terpus", "kerpus", "rempus", "brimpus", "grimpus", "gwompus", "jelpus", "jelpus", "jelpus", "scrompus", "yumpus", "numpus", "grimpus", "yerpus", "jompus", "yumpus", "wumpus", "chorpus", "jelpus", "bompus", "shumpus", "brimpus", "hilpus", "jelpus", "zumpus", "sterpus", "fompus", "jompus", "dumpus", "zhorpus", "sterpus", "lorpus", "scrompus", "zhorpus", "yimpus", "jelpus", "scrompus", "rompus", "wumpus", "quimpus", "kerpus", "storpus", "bompus", "wumpus", "terpus", "quimpus", "impus", "quimpus", "boompus", "rorpus", "fompus", "storpus", "chorpus", "timpus", "zumpus", "wumpus", "grimpus", "kerpus", "lempus", "fompus", "quimpus", "zumpus", "grimpus", "yerpus", "yimpus", "numpus", "jompus", "jompus", "gorpus", "yimpus", "jelpus", "gorpus", "sterpus", "fompus", "scrompus", "zumpus", "tumpus", "rempus", "hilpus", "storpus", "rompus", "shumpus", "yumpus", "lempus", "jompus", "zumpus", "tumpus", "gerpus", "yerpus", "quimpus", "impus", "worpus", "zumpus", "yimpus", "terpus", "felpus", "tumpus", "wumpus", "bompus", "felpus", "vumpus", "quimpus", "yerpus", "scrompus", "worpus", "zumpus", "chorpus", "shumpus", "yimpus", "gwompus", "brimpus", "rorpus", "tumpus", "impus", "felpus", "boompus", "kerpus", "fompus", "gwompus", "rorpus", "vumpus", "yerpus", "impus", "quimpus", "yimpus", "tumpus", "quimpus", "quimpus", "bompus", "hilpus", "numpus", "numpus", "vumpus", "zumpus", "numpus", "scrompus", "tumpus", "fompus", "quimpus", "yerpus", "impus", "gerpus", "rempus", "jelpus", "chorpus", "felpus", "felpus", "numpus", "terpus", "sterpus", "yerpus", "terpus", "gorpus", "gorpus", "timpus", "zumpus", "shumpus", "hilpus", "scrompus", "vumpus", "jompus", "terpus", "jelpus", "hilpus", "bompus", "sterpus", "boompus", "lempus", "bompus", "scrompus", "jelpus", "rempus", "zumpus", "sterpus", "kerpus", "zumpus", "zhorpus", "vumpus", "dumpus", "storpus", "yimpus", "yimpus", "timpus", "storpus", "fompus", "dumpus", "gerpus", "bompus", "jelpus", "felpus", "rompus", "numpus", "zhorpus", "quimpus", "quimpus", "vumpus", "impus", "kerpus", "quimpus", "boompus", "yumpus", "hilpus", "gerpus", "terpus", "fompus", "shumpus", "kerpus", "lempus", "jelpus", "sterpus", "rempus", "boompus", "gorpus", "gorpus", "jompus", "grimpus", "sterpus", "scrompus", "zumpus", "rorpus", "rompus", "rompus", "scrompus", "dumpus", "zumpus", "grimpus", "jompus", "shumpus", "gorpus", "shumpus", "vumpus", "rempus", "yumpus", "zhorpus", "rempus", "bompus", "dumpus", "storpus", "scrompus", "boompus", "wumpus", "boompus", "grimpus", "storpus", "scrompus", "rempus", "timpus", "scrompus", "worpus", "kerpus", "gwompus", "zumpus", "yumpus", "jompus", "zhorpus", "timpus", "storpus", "fompus", "scrompus", "timpus", "yimpus", "gorpus", "wumpus", "storpus", "sterpus"], "1": ["rempus", "grimpus", "worpus", "rempus", "tumpus", "storpus", "rorpus", "storpus", "boompus", "rompus", "terpus", "gerpus", "lorpus", "storpus", "yerpus", "lorpus", "kerpus", "storpus", "gerpus", "numpus", "tumpus", "gerpus", "numpus", "jompus", "hilpus", "hilpus", "grimpus", "vumpus", "fompus", "jompus", "vumpus", "boompus", "storpus", "jompus", "grimpus", "impus", "lempus", "gerpus", "grimpus", "zhorpus", "fompus", "sterpus", "tumpus", "lempus", "tumpus", "wumpus", "storpus", "bompus", "gorpus", "sterpus", "numpus", "zhorpus", "gwompus", "scrompus", "boompus", "grimpus", "rorpus", "storpus", "zumpus", "tumpus", "rempus", "vumpus", "gorpus", "zumpus", "chorpus", "tumpus", "worpus", "chorpus", "storpus", "vumpus", "rorpus", "gwompus", "zhorpus", "yerpus", "shumpus", "hilpus", "gorpus", "rompus", "grimpus", "worpus", "gwompus", "brimpus", "worpus", "fompus", "storpus", "kerpus", "kerpus", "yimpus", "wumpus", "vumpus", "grimpus", "shumpus", "gorpus", "lorpus", "numpus", "tumpus", "yimpus", "wumpus", "rorpus", "yerpus", "zhorpus", "rorpus", "terpus", "lempus", "worpus", "rompus", "yimpus", "rempus", "brimpus", "storpus", "gorpus", "gwompus", "kerpus", "gerpus", "terpus", "impus", "jompus", "felpus", "impus", "dumpus", "vumpus", "jelpus", "gorpus", "lorpus", "numpus", "storpus", "hilpus", "storpus", "quimpus", "hilpus", "storpus", "gerpus", "chorpus", "fompus", "yerpus", "zumpus", "jelpus", "jompus", "terpus", "rorpus", "shumpus", "shumpus", "tumpus", "gorpus", "wumpus", "rempus", "chorpus", "rempus", "scrompus", "rempus", "jompus", "timpus", "bompus", "kerpus", "brimpus", "gwompus", "numpus", "jompus", "impus", "boompus", "bompus", "boompus", "tumpus", "numpus", "yerpus", "wumpus", "rompus", "terpus", "quimpus", "gerpus", "felpus", "quimpus", "quimpus", "numpus", "yumpus", "felpus", "brimpus", "quimpus", "sterpus", "lorpus", "jompus", "rorpus", "brimpus", "storpus", "worpus", "scrompus", "grimpus", "felpus", "worpus", "scrompus", "felpus", "gerpus", "yerpus", "zhorpus", "kerpus", "yumpus", "gerpus", "sterpus", "dumpus", "quimpus", "tumpus", "timpus", "quimpus", "boompus", "chorpus", "gerpus", "wumpus", "gwompus", "yerpus", "boompus", "hilpus", "felpus", "scrompus", "boompus", "rompus", "fompus", "quimpus", "felpus", "zumpus", "wumpus", "felpus", "chorpus", "yimpus", "worpus", "rempus", "jelpus", "gerpus", "yerpus", "gwompus", "jompus", "rorpus", "sterpus", "gwompus", "lorpus", "shumpus", "jompus", "fompus", "storpus", "rempus", "lorpus", "gorpus", "brimpus", "grimpus", "yimpus", "yerpus", "zumpus", "tumpus", "kerpus", "brimpus", "sterpus", "bompus", "numpus", "zumpus", "zhorpus", "tumpus", "numpus", "timpus", "rompus", "worpus", "rompus", "chorpus", "numpus", "hilpus", "timpus", "sterpus", "grimpus", "jompus", "terpus", "shumpus", "tumpus", "gwompus", "jelpus", "wumpus", "dumpus", "sterpus", "gorpus", "jompus", "dumpus", "zumpus", "brimpus", "jompus", "bompus", "sterpus", "rompus", "tumpus", "timpus", "gerpus", "hilpus", "wumpus", "rorpus", "rorpus", "quimpus", "brimpus", "zhorpus", "wumpus", "sterpus", "grimpus", "yimpus", "tumpus", "bompus", "jompus", "kerpus", "impus", "yerpus", "boompus", "yerpus", "bompus", "vumpus", "hilpus", "jelpus", "hilpus", "jelpus", "quimpus", "bompus", "yimpus", "wumpus", "boompus", "gwompus", "gorpus", "storpus", "rempus", "impus", "zhorpus", "vumpus", "numpus", "yumpus", "quimpus", "boompus", "gwompus", "timpus", "jelpus", "kerpus", "yimpus", "jelpus", "bompus", "scrompus", "rorpus", "zumpus", "yumpus", "dumpus", "zhorpus", "wumpus", "fompus", "jelpus", "rorpus", "numpus", "dumpus", "boompus", "worpus", "quimpus", "gerpus", "bompus", "brimpus", "yerpus", "scrompus", "quimpus", "yimpus", "sterpus", "tumpus", "rempus", "hilpus", "zhorpus", "bompus", "tumpus", "storpus", "quimpus", "terpus", "wumpus", "impus", "quimpus", "sterpus", "gorpus", "yerpus", "zhorpus", "worpus", "chorpus", "chorpus", "shumpus", "wumpus", "terpus", "yerpus", "tumpus", "shumpus", "tumpus", "zumpus", "hilpus", "wumpus", "rompus", "terpus", "fompus", "gerpus", "scrompus", "terpus", "chorpus", "lempus", "brimpus", "wumpus", "rompus", "rompus", "grimpus", "impus", "brimpus", "sterpus", "lempus", "gwompus", "kerpus", "felpus", "lempus", "hilpus", "zumpus", "rorpus", "quimpus", "boompus", "timpus", "boompus", "worpus", "shumpus", "yimpus", "yerpus", "worpus", "bompus", "zumpus", "tumpus", "impus", "numpus", "zumpus", "yimpus", "boompus", "rempus", "jelpus", "bompus", "rompus", "brimpus", "terpus", "dumpus", "felpus", "chorpus", "vumpus", "rorpus", "boompus", "yimpus", "bompus", "gwompus", "impus", "jompus", "dumpus", "tumpus", "fompus", "terpus", "worpus", "chorpus", "gwompus", "yerpus", "tumpus", "bompus", "tumpus", "yumpus", "yumpus", "yimpus", "gorpus", "fompus", "dumpus", "gerpus", "hilpus", "vumpus", "zhorpus", "tumpus", "brimpus", "jelpus", "yimpus", "scrompus", "sterpus", "dumpus", "brimpus", "rempus", "grimpus", "rompus", "tumpus", "sterpus", "yumpus", "scrompus", "impus", "jelpus", "storpus", "rompus", "rompus", "fompus", "timpus", "worpus", "yumpus", "zumpus", "wumpus", "felpus", "rompus", "terpus", "gorpus", "yerpus", "jompus", "bompus", "impus"], "2": ["sterpus", "jelpus", "boompus", "timpus", "zumpus", "wumpus", "gorpus", "dumpus", "lorpus", "yerpus", "felpus", "shumpus", "tumpus", "gorpus", "chorpus", "gwompus", "grimpus", "yumpus", "fompus", "boompus", "storpus", "felpus", "hilpus", "gwompus", "chorpus", "gerpus", "boompus", "yerpus", "gorpus", "brimpus", "worpus", "wumpus", "timpus", "gorpus", "sterpus", "rompus", "wumpus", "shumpus", "tumpus", "terpus", "yimpus", "fompus", "bompus", "boompus", "scrompus", "quimpus", "kerpus", "fompus", "wumpus", "gwompus", "tumpus", "kerpus", "tumpus", "rorpus", "chorpus", "rorpus", "bompus", "worpus", "terpus", "bompus", "vumpus", "boompus", "scrompus", "yumpus", "rorpus", "zhorpus", "dumpus", "numpus", "zhorpus", "zumpus", "felpus", "dumpus", "wumpus", "quimpus", "jompus", "storpus", "numpus", "worpus", "boompus", "fompus", "lempus", "boompus", "felpus", "wumpus", "yerpus", "quimpus", "gerpus", "rempus", "storpus", "bompus", "kerpus", "quimpus", "rempus", "timpus", "dumpus", "sterpus", "zumpus", "chorpus", "jompus", "impus", "rorpus", "numpus", "bompus", "felpus", "gerpus", "boompus", "brimpus", "storpus", "rompus", "yumpus", "felpus", "quimpus", "rempus", "fompus", "boompus", "gerpus", "numpus", "gwompus", "storpus", "chorpus", "timpus", "brimpus", "sterpus", "numpus", "timpus", "gerpus", "quimpus", "quimpus", "yerpus", "rempus", "fompus", "dumpus", "scrompus", "vumpus", "impus", "gwompus", "shumpus", "terpus", "impus", "gwompus", "dumpus", "tumpus", "shumpus", "lempus", "jompus", "yerpus", "jelpus", "wumpus", "tumpus", "yerpus", "lorpus", "gorpus", "fompus", "dumpus", "grimpus", "timpus", "timpus", "gorpus", "timpus", "jelpus", "scrompus", "bompus", "shumpus", "scrompus", "impus", "lempus", "jompus", "jelpus", "shumpus", "gorpus", "wumpus", "tumpus", "boompus", "tumpus", "vumpus", "rempus", "quimpus", "yerpus", "vumpus", "brimpus", "chorpus", "worpus", "gorpus", "zhorpus", "timpus", "impus", "impus", "yumpus", "lempus", "impus", "scrompus", "numpus", "wumpus", "wumpus", "yimpus", "impus", "numpus", "gorpus", "brimpus", "tumpus", "impus", "rompus", "terpus", "yumpus", "tumpus", "fompus", "gwompus", "yumpus", "quimpus", "storpus", "wumpus", "gwompus", "shumpus", "rompus", "boompus", "impus", "timpus", "wumpus", "yimpus", "hilpus", "gorpus", "zumpus", "grimpus", "scrompus", "gorpus", "boompus", "sterpus", "brimpus", "vumpus", "zumpus", "jelpus", "gorpus", "yimpus", "yumpus", "numpus", "jelpus", "bompus", "rompus", "felpus", "boompus", "gerpus", "rompus", "lorpus", "yumpus", "worpus", "boompus", "fompus", "rempus", "dumpus", "zhorpus", "brimpus", "terpus", "gwompus", "fompus", "rorpus", "chorpus", "rompus", "wumpus", "fompus", "rorpus", "zhorpus", "rompus", "zhorpus", "gwompus", "brimpus", "gerpus", "dumpus", "rempus", "wumpus", "storpus", "boompus", "hilpus", "kerpus", "yimpus", "rempus", "storpus", "jelpus", "storpus", "bompus", "fompus", "tumpus", "grimpus", "tumpus", "gorpus", "scrompus", "shumpus", "gorpus", "bompus", "numpus", "tumpus", "boompus", "yimpus", "felpus", "lorpus", "felpus", "zumpus", "shumpus", "hilpus", "yumpus", "storpus", "lempus", "yerpus", "lorpus", "dumpus", "brimpus", "kerpus", "fompus", "terpus", "vumpus", "shumpus", "dumpus", "lorpus", "rompus", "worpus", "tumpus", "gorpus", "fompus", "shumpus", "gwompus", "terpus", "fompus", "jompus", "rorpus", "tumpus", "rompus", "scrompus", "gerpus", "jelpus", "rempus", "fompus", "boompus", "timpus", "zhorpus", "lempus", "chorpus", "jelpus", "vumpus", "timpus", "chorpus", "brimpus", "hilpus", "shumpus", "gorpus", "jompus", "hilpus", "impus", "felpus", "zumpus", "yumpus", "gorpus", "brimpus", "hilpus", "tumpus", "numpus", "lempus", "lorpus", "quimpus", "bompus", "brimpus", "quimpus", "timpus", "worpus", "worpus", "dumpus", "brimpus", "impus", "jompus", "impus", "yimpus", "kerpus", "kerpus", "wumpus", "gerpus", "rempus", "gorpus", "felpus", "lorpus", "boompus", "lorpus", "yimpus", "yimpus", "quimpus", "gerpus", "vumpus", "grimpus", "quimpus", "quimpus", "dumpus", "bompus", "lempus", "timpus", "yumpus", "tumpus", "terpus", "numpus", "jelpus", "bompus", "chorpus", "lorpus", "felpus", "gwompus", "rompus", "jompus", "timpus", "lorpus", "dumpus", "sterpus", "yimpus", "fompus", "numpus", "kerpus", "sterpus", "shumpus", "boompus", "gerpus", "grimpus", "jompus", "zhorpus", "vumpus", "chorpus", "quimpus", "quimpus", "hilpus", "timpus", "scrompus", "terpus", "lorpus", "yerpus", "felpus", "grimpus", "yimpus", "worpus", "gerpus", "jelpus", "zumpus", "brimpus", "worpus", "terpus", "rompus", "rompus", "jompus", "lorpus", "shumpus", "grimpus", "hilpus", "yimpus", "rorpus", "boompus", "rorpus", "yumpus", "fompus", "terpus", "terpus", "numpus", "chorpus", "quimpus", "storpus", "rempus", "bompus", "rompus", "bompus", "terpus", "terpus", "sterpus", "rompus", "jompus", "impus", "terpus", "tumpus", "yumpus", "worpus", "rorpus", "fompus", "impus", "numpus", "dumpus", "zumpus", "yerpus", "shumpus", "vumpus", "lorpus", "rempus", "dumpus", "yumpus", "zumpus", "yerpus", "dumpus", "brimpus", "impus", "rorpus", "terpus", "bompus", "boompus", "rempus", "gorpus", "storpus", "dumpus", "gerpus", "jompus", "brimpus"], "3": ["shumpus", "tumpus", "rompus", "vumpus", "lempus", "quimpus", "rorpus", "vumpus", "felpus", "brimpus", "shumpus", "yerpus", "sterpus", "boompus", "kerpus", "shumpus", "gorpus", "chorpus", "gwompus", "sterpus", "fompus", "terpus", "terpus", "dumpus", "numpus", "yerpus", "vumpus", "shumpus", "scrompus", "grimpus", "brimpus", "gwompus", "lempus", "lorpus", "jompus", "jompus", "gwompus", "felpus", "chorpus", "yerpus", "hilpus", "kerpus", "rorpus", "yerpus", "felpus", "yerpus", "bompus", "scrompus", "lorpus", "yumpus", "zumpus", "scrompus", "gwompus", "gwompus", "fompus", "sterpus", "scrompus", "numpus", "yerpus", "bompus", "storpus", "timpus", "jompus", "wumpus", "bompus", "impus", "rorpus", "zumpus", "hilpus", "jompus", "gerpus", "gwompus", "storpus", "timpus", "chorpus", "numpus", "fompus", "chorpus", "vumpus", "yimpus", "numpus", "yumpus", "gorpus", "rompus", "quimpus", "zhorpus", "grimpus", "bompus", "zhorpus", "kerpus", "gerpus", "grimpus", "shumpus", "storpus", "yumpus", "worpus", "yumpus", "shumpus", "yumpus", "yumpus", "yerpus", "yimpus", "lempus", "yimpus", "fompus", "kerpus", "yimpus", "rorpus", "boompus", "gwompus", "hilpus", "yimpus", "dumpus", "zhorpus", "rompus", "lempus", "shumpus", "yimpus", "grimpus", "storpus", "impus", "boompus", "tumpus", "felpus", "shumpus", "felpus", "shumpus", "quimpus", "vumpus", "terpus", "gorpus", "boompus", "worpus", "scrompus", "sterpus", "lorpus", "lorpus", "felpus", "quimpus", "boompus", "shumpus", "rompus", "felpus", "rompus", "zhorpus", "rompus", "dumpus", "lempus", "yerpus", "zhorpus", "dumpus", "storpus", "chorpus", "dumpus", "dumpus", "chorpus", "timpus", "impus", "lempus", "rempus", "worpus", "chorpus", "numpus", "quimpus", "shumpus", "wumpus", "lorpus", "vumpus", "yerpus", "fompus", "timpus", "jompus", "gwompus", "impus", "lempus", "storpus", "quimpus", "zumpus", "terpus", "kerpus", "rompus", "hilpus", "zumpus", "chorpus", "rempus", "jompus", "numpus", "terpus", "rorpus", "felpus", "gorpus", "lorpus", "rompus", "felpus", "terpus", "tumpus", "quimpus", "wumpus", "chorpus", "vumpus", "jompus", "grimpus", "yerpus", "chorpus", "sterpus", "timpus", "rompus", "felpus", "kerpus", "lorpus", "worpus", "tumpus", "scrompus", "zhorpus", "terpus", "sterpus", "hilpus", "quimpus", "zumpus", "worpus", "tumpus", "gwompus", "jompus", "hilpus", "rempus", "rempus", "lorpus", "storpus", "zhorpus", "yerpus", "brimpus", "rompus", "quimpus", "storpus", "storpus", "impus", "fompus", "brimpus", "lempus", "chorpus", "impus", "zhorpus", "zhorpus", "gorpus", "bompus", "gorpus", "jelpus", "chorpus", "fompus", "storpus", "impus", "dumpus", "sterpus", "wumpus", "numpus", "rempus", "terpus", "numpus", "timpus", "rempus", "lorpus", "wumpus", "terpus", "zhorpus", "hilpus", "yumpus", "scrompus", "vumpus", "kerpus", "scrompus", "yimpus", "yimpus", "sterpus", "rempus", "yumpus", "grimpus", "rompus", "terpus", "rompus", "jompus", "brimpus", "gerpus", "numpus", "chorpus", "bompus", "boompus", "zhorpus", "yimpus", "gerpus", "terpus", "timpus", "felpus", "gorpus", "terpus", "terpus", "jompus", "lorpus", "jelpus"], "4": ["worpus", "jompus", "boompus", "hilpus", "yumpus", "terpus", "zhorpus", "wumpus", "sterpus", "brimpus", "hilpus", "tumpus", "rompus", "bompus", "yimpus", "yumpus", "bompus", "rempus", "gerpus", "vumpus", "felpus", "jompus", "quimpus", "scrompus", "brimpus", "wumpus", "tumpus", "lorpus", "yimpus", "brimpus", "storpus", "chorpus", "terpus", "jompus", "terpus", "jelpus", "brimpus", "chorpus", "scrompus", "lempus", "jelpus", "rempus", "wumpus", "boompus", "zumpus", "jompus", "rempus", "gwompus", "timpus", "yimpus", "bompus", "lorpus", "impus", "hilpus", "yimpus", "timpus", "scrompus", "vumpus", "gerpus", "dumpus", "terpus", "worpus", "lempus", "yerpus", "hilpus", "yerpus", "yumpus", "shumpus", "tumpus", "rempus", "worpus", "jompus", "storpus", "tumpus", "storpus", "gerpus", "yerpus", "dumpus", "hilpus", "jelpus", "zhorpus"], "5": ["hilpus", "shumpus", "lorpus", "dumpus", "sterpus", "scrompus", "felpus", "chorpus", "zumpus", "worpus", "lempus", "fompus"]}
experiments/probing_corrected/m3_linear_perm.json ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linear_probe_accuracy": [
3
+ [
4
+ 0.086,
5
+ 0.08600000000000001,
6
+ 0.186,
7
+ 0.5536912751677852,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ [
12
+ 0.08800000000000001,
13
+ 0.08800000000000001,
14
+ 0.16,
15
+ 0.5469798657718121,
16
+ 0.0,
17
+ 0.0
18
+ ],
19
+ [
20
+ 0.088,
21
+ 0.08,
22
+ 0.184,
23
+ 0.4899328859060403,
24
+ 0.0,
25
+ 0.0
26
+ ],
27
+ [
28
+ 0.08,
29
+ 0.072,
30
+ 0.146,
31
+ 0.4664429530201342,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ [
36
+ 0.09,
37
+ 0.06000000000000001,
38
+ 0.144,
39
+ 0.4597315436241611,
40
+ 0.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.076,
45
+ 0.074,
46
+ 0.13999999999999999,
47
+ 0.42953020134228187,
48
+ 0.0,
49
+ 0.0
50
+ ],
51
+ [
52
+ 0.098,
53
+ 0.057999999999999996,
54
+ 0.138,
55
+ 0.3959731543624161,
56
+ 0.0,
57
+ 0.0
58
+ ],
59
+ [
60
+ 0.07200000000000001,
61
+ 0.06400000000000002,
62
+ 0.122,
63
+ 0.37919463087248323,
64
+ 0.0,
65
+ 0.0
66
+ ],
67
+ [
68
+ 0.13,
69
+ 0.06599999999999999,
70
+ 0.13,
71
+ 0.25838926174496646,
72
+ 0.0,
73
+ 0.0
74
+ ],
75
+ [
76
+ 0.09000000000000001,
77
+ 0.076,
78
+ 0.118,
79
+ 0.2785234899328859,
80
+ 0.0,
81
+ 0.0
82
+ ],
83
+ [
84
+ 0.05800000000000001,
85
+ 0.094,
86
+ 0.14,
87
+ 0.32885906040268453,
88
+ 0.0,
89
+ 0.0
90
+ ],
91
+ [
92
+ 0.076,
93
+ 0.09400000000000001,
94
+ 0.17400000000000002,
95
+ 0.5268456375838926,
96
+ 0.0,
97
+ 0.0
98
+ ],
99
+ [
100
+ 0.054000000000000006,
101
+ 0.1,
102
+ 0.19,
103
+ 0.5503355704697986,
104
+ 0.0,
105
+ 0.0
106
+ ]
107
+ ],
108
+ "permutation_p_values": [
109
+ [
110
+ 0.0014992503748125937,
111
+ 0.0009995002498750624,
112
+ 0.0004997501249375312,
113
+ 0.0004997501249375312,
114
+ 1.0,
115
+ 1.0
116
+ ],
117
+ [
118
+ 0.001999000499750125,
119
+ 0.0024987506246876563,
120
+ 0.0004997501249375312,
121
+ 0.0004997501249375312,
122
+ 1.0,
123
+ 1.0
124
+ ],
125
+ [
126
+ 0.001999000499750125,
127
+ 0.005497251374312844,
128
+ 0.0004997501249375312,
129
+ 0.0004997501249375312,
130
+ 1.0,
131
+ 1.0
132
+ ],
133
+ [
134
+ 0.0069965017491254375,
135
+ 0.0024987506246876563,
136
+ 0.0004997501249375312,
137
+ 0.0004997501249375312,
138
+ 1.0,
139
+ 1.0
140
+ ],
141
+ [
142
+ 0.0014992503748125937,
143
+ 0.017491254372813594,
144
+ 0.0004997501249375312,
145
+ 0.0004997501249375312,
146
+ 1.0,
147
+ 1.0
148
+ ],
149
+ [
150
+ 0.0069965017491254375,
151
+ 0.00399800099950025,
152
+ 0.0004997501249375312,
153
+ 0.0004997501249375312,
154
+ 1.0,
155
+ 1.0
156
+ ],
157
+ [
158
+ 0.0014992503748125937,
159
+ 0.04697651174412794,
160
+ 0.0004997501249375312,
161
+ 0.0004997501249375312,
162
+ 1.0,
163
+ 1.0
164
+ ],
165
+ [
166
+ 0.0034982508745627187,
167
+ 0.013493253373313344,
168
+ 0.0004997501249375312,
169
+ 0.0004997501249375312,
170
+ 1.0,
171
+ 1.0
172
+ ],
173
+ [
174
+ 0.0004997501249375312,
175
+ 0.01699150424787606,
176
+ 0.0004997501249375312,
177
+ 0.0004997501249375312,
178
+ 1.0,
179
+ 1.0
180
+ ],
181
+ [
182
+ 0.0004997501249375312,
183
+ 0.0069965017491254375,
184
+ 0.0004997501249375312,
185
+ 0.0004997501249375312,
186
+ 1.0,
187
+ 1.0
188
+ ],
189
+ [
190
+ 0.05747126436781609,
191
+ 0.0009995002498750624,
192
+ 0.0004997501249375312,
193
+ 0.0004997501249375312,
194
+ 1.0,
195
+ 1.0
196
+ ],
197
+ [
198
+ 0.0074962518740629685,
199
+ 0.0004997501249375312,
200
+ 0.0004997501249375312,
201
+ 0.0004997501249375312,
202
+ 1.0,
203
+ 1.0
204
+ ],
205
+ [
206
+ 0.054972513743128434,
207
+ 0.0009995002498750624,
208
+ 0.0004997501249375312,
209
+ 0.0004997501249375312,
210
+ 1.0,
211
+ 1.0
212
+ ]
213
+ ],
214
+ "significant_cells": [
215
+ [
216
+ false,
217
+ false,
218
+ true,
219
+ true,
220
+ false,
221
+ false
222
+ ],
223
+ [
224
+ false,
225
+ false,
226
+ true,
227
+ true,
228
+ false,
229
+ false
230
+ ],
231
+ [
232
+ false,
233
+ false,
234
+ true,
235
+ true,
236
+ false,
237
+ false
238
+ ],
239
+ [
240
+ false,
241
+ false,
242
+ true,
243
+ true,
244
+ false,
245
+ false
246
+ ],
247
+ [
248
+ false,
249
+ false,
250
+ true,
251
+ true,
252
+ false,
253
+ false
254
+ ],
255
+ [
256
+ false,
257
+ false,
258
+ true,
259
+ true,
260
+ false,
261
+ false
262
+ ],
263
+ [
264
+ false,
265
+ false,
266
+ true,
267
+ true,
268
+ false,
269
+ false
270
+ ],
271
+ [
272
+ false,
273
+ false,
274
+ true,
275
+ true,
276
+ false,
277
+ false
278
+ ],
279
+ [
280
+ true,
281
+ false,
282
+ true,
283
+ true,
284
+ false,
285
+ false
286
+ ],
287
+ [
288
+ true,
289
+ false,
290
+ true,
291
+ true,
292
+ false,
293
+ false
294
+ ],
295
+ [
296
+ false,
297
+ false,
298
+ true,
299
+ true,
300
+ false,
301
+ false
302
+ ],
303
+ [
304
+ false,
305
+ true,
306
+ true,
307
+ true,
308
+ false,
309
+ false
310
+ ],
311
+ [
312
+ false,
313
+ false,
314
+ true,
315
+ true,
316
+ false,
317
+ false
318
+ ]
319
+ ],
320
+ "n_significant_cells": 29,
321
+ "bonferroni_threshold": 0.000641025641025641,
322
+ "n_samples_per_position": {
323
+ "0": 500,
324
+ "1": 500,
325
+ "2": 500,
326
+ "3": 298,
327
+ "4": 81,
328
+ "5": 12
329
+ },
330
+ "n_permutations": 2000,
331
+ "peak_layer": 0,
332
+ "peak_position": 3,
333
+ "peak_accuracy": 0.5537
334
+ }
experiments/probing_corrected/m5_hidden_states.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c64081234820427d6ae8aeb2b62f2304d9f24d44aa82542f4041c78c8f4da1ad
3
+ size 70677014
experiments/probing_corrected/m5_labels.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"0": ["scrompus", "storpus", "sterpus", "vumpus", "jelpus", "bompus", "boompus", "brimpus", "sterpus", "bompus", "chorpus", "fompus", "rempus", "rompus", "sterpus", "rorpus", "numpus", "dumpus", "storpus", "brimpus", "gerpus", "rompus", "jompus", "numpus", "rompus", "terpus", "sterpus", "fompus", "storpus", "dumpus", "scrompus", "quimpus", "yerpus", "jelpus", "gorpus", "tumpus", "shumpus", "chorpus", "yerpus", "vumpus", "impus", "scrompus", "zumpus", "rorpus", "rempus", "fompus", "yumpus", "rorpus", "bompus", "dumpus", "gwompus", "felpus", "chorpus", "wumpus", "jompus", "felpus", "grimpus", "wumpus", "fompus", "wumpus", "gwompus", "gwompus", "bompus", "vumpus", "lorpus", "worpus", "yumpus", "fompus", "gorpus", "scrompus", "jompus", "brimpus", "yumpus", "worpus", "rempus", "jelpus", "fompus", "scrompus", "jompus", "rompus", "grimpus", "rompus", "terpus", "gerpus", "yumpus", "storpus", "gwompus", "lorpus", "lorpus", "rompus", "timpus", "grimpus", "tumpus", "rorpus", "gwompus", "gorpus", "numpus", "boompus", "jelpus", "grimpus", "dumpus", "lorpus", "rempus", "jompus", "bompus", "sterpus", "scrompus", "lempus", "grimpus", "quimpus", "hilpus", "jelpus", "worpus", "shumpus", "jompus", "zumpus", "quimpus", "gerpus", "grimpus", "yimpus", "zhorpus", "lempus", "zumpus", "zhorpus", "zumpus", "zumpus", "wumpus", "rompus", "shumpus", "quimpus", "chorpus", "grimpus", "sterpus", "sterpus", "lempus", "yimpus", "gerpus", "zumpus", "sterpus", "worpus", "rorpus", "yimpus", "jelpus", "storpus", "dumpus", "fompus", "brimpus", "terpus", "storpus", "gerpus", "timpus", "vumpus", "sterpus", "rorpus", "gerpus", "vumpus", "fompus", "bompus", "yerpus", "yerpus", "jelpus", "sterpus", "yumpus", "shumpus", "tumpus", "rompus", "dumpus", "storpus", "lempus", "rempus", "lempus", "gorpus", "lorpus", "lorpus", "bompus", "numpus", "felpus", "jompus", "rempus", "boompus", "brimpus", "sterpus", "dumpus", "wumpus", "vumpus", "vumpus", "felpus", "timpus", "boompus", "dumpus", "bompus", "boompus", "brimpus", "shumpus", "hilpus", "lempus", "impus", "rompus", "boompus", "terpus", "gorpus", "tumpus", "numpus", "kerpus", "gerpus", "scrompus", "grimpus", "lempus", "grimpus", "timpus", "chorpus", "hilpus", "vumpus", "dumpus", "gorpus", "gerpus", "lempus", "yimpus", "fompus", "yumpus", "shumpus", "lorpus", "fompus", "kerpus", "jompus", "storpus", "timpus", "kerpus", "sterpus", "rompus", "yimpus", "brimpus", "storpus", "boompus", "grimpus", "terpus", "kerpus", "rempus", "brimpus", "grimpus", "gwompus", "jelpus", "jelpus", "jelpus", "scrompus", "yumpus", "numpus", "grimpus", "yerpus", "jompus", "yumpus", "wumpus", "chorpus", "jelpus", "bompus", "shumpus", "brimpus", "hilpus", "jelpus", "zumpus", "sterpus", "fompus", "jompus", "dumpus", "zhorpus", "sterpus", "lorpus", "scrompus", "zhorpus", "yimpus", "jelpus", "scrompus", "rompus", "wumpus", "quimpus", "kerpus", "storpus", "bompus", "wumpus", "terpus", "quimpus", "impus", "quimpus", "boompus", "rorpus", "fompus", "storpus", "chorpus", "timpus", "zumpus", "wumpus", "grimpus", "kerpus", "lempus", "fompus", "quimpus", "zumpus", "grimpus", "yerpus", "yimpus", "numpus", "jompus", "jompus", "gorpus", "yimpus", "jelpus", "gorpus", "sterpus", "fompus", "scrompus", "zumpus", "tumpus", "rempus", "hilpus", "storpus", "rompus", "shumpus", "yumpus", "lempus", "jompus", "zumpus", "tumpus", "gerpus", "yerpus", "quimpus", "impus", "worpus", "zumpus", "yimpus", "terpus", "felpus", "tumpus", "wumpus", "bompus", "felpus", "vumpus", "quimpus", "yerpus", "scrompus", "worpus", "zumpus", "chorpus", "shumpus", "yimpus", "gwompus", "brimpus", "rorpus", "tumpus", "impus", "felpus", "boompus", "kerpus", "fompus", "gwompus", "rorpus", "vumpus", "yerpus", "impus", "quimpus", "yimpus", "tumpus", "quimpus", "quimpus", "bompus", "hilpus", "numpus", "numpus", "vumpus", "zumpus", "numpus", "scrompus", "tumpus", "fompus", "quimpus", "yerpus", "impus", "gerpus", "rempus", "jelpus", "chorpus", "felpus", "felpus", "numpus", "terpus", "sterpus", "yerpus", "terpus", "gorpus", "gorpus", "timpus", "zumpus", "shumpus", "hilpus", "scrompus", "vumpus", "jompus", "terpus", "jelpus", "hilpus", "bompus", "sterpus", "boompus", "lempus", "bompus", "scrompus", "jelpus", "rempus", "zumpus", "sterpus", "kerpus", "zumpus", "zhorpus", "vumpus", "dumpus", "storpus", "yimpus", "yimpus", "timpus", "storpus", "fompus", "dumpus", "gerpus", "bompus", "jelpus", "felpus", "rompus", "numpus", "zhorpus", "quimpus", "quimpus", "vumpus", "impus", "kerpus", "quimpus", "boompus", "yumpus", "hilpus", "gerpus", "terpus", "fompus", "shumpus", "kerpus", "lempus", "jelpus", "sterpus", "rempus", "boompus", "gorpus", "gorpus", "jompus", "grimpus", "sterpus", "scrompus", "zumpus", "rorpus", "rompus", "rompus", "scrompus", "dumpus", "zumpus", "grimpus", "jompus", "shumpus", "gorpus", "shumpus", "vumpus", "rempus", "yumpus", "zhorpus", "rempus", "bompus", "dumpus", "storpus", "scrompus", "boompus", "wumpus", "boompus", "grimpus", "storpus", "scrompus", "rempus", "timpus", "scrompus", "worpus", "kerpus", "gwompus", "zumpus", "yumpus", "jompus", "zhorpus", "timpus", "storpus", "fompus", "scrompus", "timpus", "yimpus", "gorpus", "wumpus", "storpus", "sterpus"], "1": ["rempus", "grimpus", "worpus", "rempus", "tumpus", "storpus", "rorpus", "storpus", "boompus", "rompus", "terpus", "gerpus", "lorpus", "storpus", "yerpus", "lorpus", "kerpus", "storpus", "gerpus", "numpus", "tumpus", "gerpus", "numpus", "jompus", "hilpus", "hilpus", "grimpus", "vumpus", "fompus", "jompus", "vumpus", "boompus", "storpus", "jompus", "grimpus", "impus", "lempus", "gerpus", "grimpus", "zhorpus", "fompus", "sterpus", "tumpus", "lempus", "tumpus", "wumpus", "storpus", "bompus", "gorpus", "sterpus", "numpus", "zhorpus", "gwompus", "scrompus", "boompus", "grimpus", "rorpus", "storpus", "zumpus", "tumpus", "rempus", "vumpus", "gorpus", "zumpus", "chorpus", "tumpus", "worpus", "chorpus", "storpus", "vumpus", "rorpus", "gwompus", "zhorpus", "yerpus", "shumpus", "hilpus", "gorpus", "rompus", "grimpus", "worpus", "gwompus", "brimpus", "worpus", "fompus", "storpus", "kerpus", "kerpus", "yimpus", "wumpus", "vumpus", "grimpus", "shumpus", "gorpus", "lorpus", "numpus", "tumpus", "yimpus", "wumpus", "rorpus", "yerpus", "zhorpus", "rorpus", "terpus", "lempus", "worpus", "rompus", "yimpus", "rempus", "brimpus", "storpus", "gorpus", "gwompus", "kerpus", "gerpus", "terpus", "impus", "jompus", "felpus", "impus", "dumpus", "vumpus", "jelpus", "gorpus", "lorpus", "numpus", "storpus", "hilpus", "storpus", "quimpus", "hilpus", "storpus", "gerpus", "chorpus", "fompus", "yerpus", "zumpus", "jelpus", "jompus", "terpus", "rorpus", "shumpus", "shumpus", "tumpus", "gorpus", "wumpus", "rempus", "chorpus", "rempus", "scrompus", "rempus", "jompus", "timpus", "bompus", "kerpus", "brimpus", "gwompus", "numpus", "jompus", "impus", "boompus", "bompus", "boompus", "tumpus", "numpus", "yerpus", "wumpus", "rompus", "terpus", "quimpus", "gerpus", "felpus", "quimpus", "quimpus", "numpus", "yumpus", "felpus", "brimpus", "quimpus", "sterpus", "lorpus", "jompus", "rorpus", "brimpus", "storpus", "worpus", "scrompus", "grimpus", "felpus", "worpus", "scrompus", "felpus", "gerpus", "yerpus", "zhorpus", "kerpus", "yumpus", "gerpus", "sterpus", "dumpus", "quimpus", "tumpus", "timpus", "quimpus", "boompus", "chorpus", "gerpus", "wumpus", "gwompus", "yerpus", "boompus", "hilpus", "felpus", "scrompus", "boompus", "rompus", "fompus", "quimpus", "felpus", "zumpus", "wumpus", "felpus", "chorpus", "yimpus", "worpus", "rempus", "jelpus", "gerpus", "yerpus", "gwompus", "jompus", "rorpus", "sterpus", "gwompus", "lorpus", "shumpus", "jompus", "fompus", "storpus", "rempus", "lorpus", "gorpus", "brimpus", "grimpus", "yimpus", "yerpus", "zumpus", "tumpus", "kerpus", "brimpus", "sterpus", "bompus", "numpus", "zumpus", "zhorpus", "tumpus", "numpus", "timpus", "rompus", "worpus", "rompus", "chorpus", "numpus", "hilpus", "timpus", "sterpus", "grimpus", "jompus", "terpus", "shumpus", "tumpus", "gwompus", "jelpus", "wumpus", "dumpus", "sterpus", "gorpus", "jompus", "dumpus", "zumpus", "brimpus", "jompus", "bompus", "sterpus", "rompus", "tumpus", "timpus", "gerpus", "hilpus", "wumpus", "rorpus", "rorpus", "quimpus", "brimpus", "zhorpus", "wumpus", "sterpus", "grimpus", "yimpus", "tumpus", "bompus", "jompus", "kerpus", "impus", "yerpus", "boompus", "yerpus", "bompus", "vumpus", "hilpus", "jelpus", "hilpus", "jelpus", "quimpus", "bompus", "yimpus", "wumpus", "boompus", "gwompus", "gorpus", "storpus", "rempus", "impus", "zhorpus", "vumpus", "numpus", "yumpus", "quimpus", "boompus", "gwompus", "timpus", "jelpus", "kerpus", "yimpus", "jelpus", "bompus", "scrompus", "rorpus", "zumpus", "yumpus", "dumpus", "zhorpus", "wumpus", "fompus", "jelpus", "rorpus", "numpus", "dumpus", "boompus", "worpus", "quimpus", "gerpus", "bompus", "brimpus", "yerpus", "scrompus", "quimpus", "yimpus", "sterpus", "tumpus", "rempus", "hilpus", "zhorpus", "bompus", "tumpus", "storpus", "quimpus", "terpus", "wumpus", "impus", "quimpus", "sterpus", "gorpus", "yerpus", "zhorpus", "worpus", "chorpus", "chorpus", "shumpus", "wumpus", "terpus", "yerpus", "tumpus", "shumpus", "tumpus", "zumpus", "hilpus", "wumpus", "rompus", "terpus", "fompus", "gerpus", "scrompus", "terpus", "chorpus", "lempus", "brimpus", "wumpus", "rompus", "rompus", "grimpus", "impus", "brimpus", "sterpus", "lempus", "gwompus", "kerpus", "felpus", "lempus", "hilpus", "zumpus", "rorpus", "quimpus", "boompus", "timpus", "boompus", "worpus", "shumpus", "yimpus", "yerpus", "worpus", "bompus", "zumpus", "tumpus", "impus", "numpus", "zumpus", "yimpus", "boompus", "rempus", "jelpus", "bompus", "rompus", "brimpus", "terpus", "dumpus", "felpus", "chorpus", "vumpus", "rorpus", "boompus", "yimpus", "bompus", "gwompus", "impus", "jompus", "dumpus", "tumpus", "fompus", "terpus", "worpus", "chorpus", "gwompus", "yerpus", "tumpus", "bompus", "tumpus", "yumpus", "yumpus", "yimpus", "gorpus", "fompus", "dumpus", "gerpus", "hilpus", "vumpus", "zhorpus", "tumpus", "brimpus", "jelpus", "yimpus", "scrompus", "sterpus", "dumpus", "brimpus", "rempus", "grimpus", "rompus", "tumpus", "sterpus", "yumpus", "scrompus", "impus", "jelpus", "storpus", "rompus", "rompus", "fompus", "timpus", "worpus", "yumpus", "zumpus", "wumpus", "felpus", "rompus", "terpus", "gorpus", "yerpus", "jompus", "bompus", "impus"], "2": ["sterpus", "jelpus", "boompus", "timpus", "zumpus", "wumpus", "gorpus", "dumpus", "lorpus", "yerpus", "felpus", "shumpus", "tumpus", "gorpus", "chorpus", "gwompus", "grimpus", "yumpus", "fompus", "boompus", "storpus", "felpus", "hilpus", "gwompus", "chorpus", "gerpus", "boompus", "yerpus", "gorpus", "brimpus", "worpus", "wumpus", "timpus", "gorpus", "sterpus", "rompus", "wumpus", "shumpus", "tumpus", "terpus", "yimpus", "fompus", "bompus", "boompus", "scrompus", "quimpus", "kerpus", "fompus", "wumpus", "gwompus", "tumpus", "kerpus", "tumpus", "rorpus", "chorpus", "rorpus", "bompus", "worpus", "terpus", "bompus", "vumpus", "boompus", "scrompus", "yumpus", "rorpus", "zhorpus", "dumpus", "numpus", "zhorpus", "zumpus", "felpus", "dumpus", "wumpus", "quimpus", "jompus", "storpus", "numpus", "worpus", "boompus", "fompus", "lempus", "boompus", "felpus", "wumpus", "yerpus", "quimpus", "gerpus", "rempus", "storpus", "bompus", "kerpus", "quimpus", "rempus", "timpus", "dumpus", "sterpus", "zumpus", "chorpus", "jompus", "impus", "rorpus", "numpus", "bompus", "felpus", "gerpus", "boompus", "brimpus", "storpus", "rompus", "yumpus", "felpus", "quimpus", "rempus", "fompus", "boompus", "gerpus", "numpus", "gwompus", "storpus", "chorpus", "timpus", "brimpus", "sterpus", "numpus", "timpus", "gerpus", "quimpus", "quimpus", "yerpus", "rempus", "fompus", "dumpus", "scrompus", "vumpus", "impus", "gwompus", "shumpus", "terpus", "impus", "gwompus", "dumpus", "tumpus", "shumpus", "lempus", "jompus", "yerpus", "jelpus", "wumpus", "tumpus", "yerpus", "lorpus", "gorpus", "fompus", "dumpus", "grimpus", "timpus", "timpus", "gorpus", "timpus", "jelpus", "scrompus", "bompus", "shumpus", "scrompus", "impus", "lempus", "jompus", "jelpus", "shumpus", "gorpus", "wumpus", "tumpus", "boompus", "tumpus", "vumpus", "rempus", "quimpus", "yerpus", "vumpus", "brimpus", "chorpus", "worpus", "gorpus", "zhorpus", "timpus", "impus", "impus", "yumpus", "lempus", "impus", "scrompus", "numpus", "wumpus", "wumpus", "yimpus", "impus", "numpus", "gorpus", "brimpus", "tumpus", "impus", "rompus", "terpus", "yumpus", "tumpus", "fompus", "gwompus", "yumpus", "quimpus", "storpus", "wumpus", "gwompus", "shumpus", "rompus", "boompus", "impus", "timpus", "wumpus", "yimpus", "hilpus", "gorpus", "zumpus", "grimpus", "scrompus", "gorpus", "boompus", "sterpus", "brimpus", "vumpus", "zumpus", "jelpus", "gorpus", "yimpus", "yumpus", "numpus", "jelpus", "bompus", "rompus", "felpus", "boompus", "gerpus", "rompus", "lorpus", "yumpus", "worpus", "boompus", "fompus", "rempus", "dumpus", "zhorpus", "brimpus", "terpus", "gwompus", "fompus", "rorpus", "chorpus", "rompus", "wumpus", "fompus", "rorpus", "zhorpus", "rompus", "zhorpus", "gwompus", "brimpus", "gerpus", "dumpus", "rempus", "wumpus", "storpus", "boompus", "hilpus", "kerpus", "yimpus", "rempus", "storpus", "jelpus", "storpus", "bompus", "fompus", "tumpus", "grimpus", "tumpus", "gorpus", "scrompus", "shumpus", "gorpus", "bompus", "numpus", "tumpus", "boompus", "yimpus", "felpus", "lorpus", "felpus", "zumpus", "shumpus", "hilpus", "yumpus", "storpus", "lempus", "yerpus", "lorpus", "dumpus", "brimpus", "kerpus", "fompus", "terpus", "vumpus", "shumpus", "dumpus", "lorpus", "rompus", "worpus", "tumpus", "gorpus", "fompus", "shumpus", "gwompus", "terpus", "fompus", "jompus", "rorpus", "tumpus", "rompus", "scrompus", "gerpus", "jelpus", "rempus", "fompus", "boompus", "timpus", "zhorpus", "lempus", "chorpus", "jelpus", "vumpus", "timpus", "chorpus", "brimpus", "hilpus", "shumpus", "gorpus", "jompus", "hilpus", "impus", "felpus", "zumpus", "yumpus", "gorpus", "brimpus", "hilpus", "tumpus", "numpus", "lempus", "lorpus", "quimpus", "bompus", "brimpus", "quimpus", "timpus", "worpus", "worpus", "dumpus", "brimpus", "impus", "jompus", "impus", "yimpus", "kerpus", "kerpus", "wumpus", "gerpus", "rempus", "gorpus", "felpus", "lorpus", "boompus", "lorpus", "yimpus", "yimpus", "quimpus", "gerpus", "vumpus", "grimpus", "quimpus", "quimpus", "dumpus", "bompus", "lempus", "timpus", "yumpus", "tumpus", "terpus", "numpus", "jelpus", "bompus", "chorpus", "lorpus", "felpus", "gwompus", "rompus", "jompus", "timpus", "lorpus", "dumpus", "sterpus", "yimpus", "fompus", "numpus", "kerpus", "sterpus", "shumpus", "boompus", "gerpus", "grimpus", "jompus", "zhorpus", "vumpus", "chorpus", "quimpus", "quimpus", "hilpus", "timpus", "scrompus", "terpus", "lorpus", "yerpus", "felpus", "grimpus", "yimpus", "worpus", "gerpus", "jelpus", "zumpus", "brimpus", "worpus", "terpus", "rompus", "rompus", "jompus", "lorpus", "shumpus", "grimpus", "hilpus", "yimpus", "rorpus", "boompus", "rorpus", "yumpus", "fompus", "terpus", "terpus", "numpus", "chorpus", "quimpus", "storpus", "rempus", "bompus", "rompus", "bompus", "terpus", "terpus", "sterpus", "rompus", "jompus", "impus", "terpus", "tumpus", "yumpus", "worpus", "rorpus", "fompus", "impus", "numpus", "dumpus", "zumpus", "yerpus", "shumpus", "vumpus", "lorpus", "rempus", "dumpus", "yumpus", "zumpus", "yerpus", "dumpus", "brimpus", "impus", "rorpus", "terpus", "bompus", "boompus", "rempus", "gorpus", "storpus", "dumpus", "gerpus", "jompus", "brimpus"], "3": ["shumpus", "tumpus", "rompus", "vumpus", "lempus", "quimpus", "rorpus", "vumpus", "felpus", "brimpus", "shumpus", "yerpus", "sterpus", "boompus", "kerpus", "shumpus", "gorpus", "chorpus", "gwompus", "sterpus", "fompus", "terpus", "terpus", "dumpus", "numpus", "yerpus", "vumpus", "shumpus", "scrompus", "grimpus", "brimpus", "gwompus", "lempus", "lorpus", "jompus", "jompus", "gwompus", "felpus", "chorpus", "yerpus", "hilpus", "kerpus", "rorpus", "yerpus", "felpus", "yerpus", "bompus", "scrompus", "lorpus", "yumpus", "zumpus", "scrompus", "gwompus", "gwompus", "fompus", "sterpus", "scrompus", "numpus", "yerpus", "bompus", "storpus", "timpus", "jompus", "wumpus", "bompus", "impus", "rorpus", "zumpus", "hilpus", "jompus", "gerpus", "gwompus", "storpus", "timpus", "chorpus", "numpus", "fompus", "chorpus", "vumpus", "yimpus", "numpus", "yumpus", "gorpus", "rompus", "quimpus", "zhorpus", "grimpus", "bompus", "zhorpus", "kerpus", "gerpus", "grimpus", "shumpus", "storpus", "yumpus", "worpus", "yumpus", "shumpus", "yumpus", "yumpus", "yerpus", "yimpus", "lempus", "yimpus", "fompus", "kerpus", "yimpus", "rorpus", "boompus", "gwompus", "hilpus", "yimpus", "dumpus", "zhorpus", "rompus", "lempus", "shumpus", "yimpus", "grimpus", "storpus", "impus", "boompus", "tumpus", "felpus", "shumpus", "felpus", "shumpus", "quimpus", "vumpus", "terpus", "gorpus", "boompus", "worpus", "scrompus", "sterpus", "lorpus", "lorpus", "felpus", "quimpus", "boompus", "shumpus", "rompus", "felpus", "rompus", "zhorpus", "rompus", "dumpus", "lempus", "yerpus", "zhorpus", "dumpus", "storpus", "chorpus", "dumpus", "dumpus", "chorpus", "timpus", "impus", "lempus", "rempus", "worpus", "chorpus", "numpus", "quimpus", "shumpus", "wumpus", "lorpus", "vumpus", "yerpus", "fompus", "timpus", "jompus", "gwompus", "impus", "lempus", "storpus", "quimpus", "zumpus", "terpus", "kerpus", "rompus", "hilpus", "zumpus", "chorpus", "rempus", "jompus", "numpus", "terpus", "rorpus", "felpus", "gorpus", "lorpus", "rompus", "felpus", "terpus", "tumpus", "quimpus", "wumpus", "chorpus", "vumpus", "jompus", "grimpus", "yerpus", "chorpus", "sterpus", "timpus", "rompus", "felpus", "kerpus", "lorpus", "worpus", "tumpus", "scrompus", "zhorpus", "terpus", "sterpus", "hilpus", "quimpus", "zumpus", "worpus", "tumpus", "gwompus", "jompus", "hilpus", "rempus", "rempus", "lorpus", "storpus", "zhorpus", "yerpus", "brimpus", "rompus", "quimpus", "storpus", "storpus", "impus", "fompus", "brimpus", "lempus", "chorpus", "impus", "zhorpus", "zhorpus", "gorpus", "bompus", "gorpus", "jelpus", "chorpus", "fompus", "storpus", "impus", "dumpus", "sterpus", "wumpus", "numpus", "rempus", "terpus", "numpus", "timpus", "rempus", "lorpus", "wumpus", "terpus", "zhorpus", "hilpus", "yumpus", "scrompus", "vumpus", "kerpus", "scrompus", "yimpus", "yimpus", "sterpus", "rempus", "yumpus", "grimpus", "rompus", "terpus", "rompus", "jompus", "brimpus", "gerpus", "numpus", "chorpus", "bompus", "boompus", "zhorpus", "yimpus", "gerpus", "terpus", "timpus", "felpus", "gorpus", "terpus", "terpus", "jompus", "lorpus", "jelpus"], "4": ["worpus", "jompus", "boompus", "hilpus", "yumpus", "terpus", "zhorpus", "wumpus", "sterpus", "brimpus", "hilpus", "tumpus", "rompus", "bompus", "yimpus", "yumpus", "bompus", "rempus", "gerpus", "vumpus", "felpus", "jompus", "quimpus", "scrompus", "brimpus", "wumpus", "tumpus", "lorpus", "yimpus", "brimpus", "storpus", "chorpus", "terpus", "jompus", "terpus", "jelpus", "brimpus", "chorpus", "scrompus", "lempus", "jelpus", "rempus", "wumpus", "boompus", "zumpus", "jompus", "rempus", "gwompus", "timpus", "yimpus", "bompus", "lorpus", "impus", "hilpus", "yimpus", "timpus", "scrompus", "vumpus", "gerpus", "dumpus", "terpus", "worpus", "lempus", "yerpus", "hilpus", "yerpus", "yumpus", "shumpus", "tumpus", "rempus", "worpus", "jompus", "storpus", "tumpus", "storpus", "gerpus", "yerpus", "dumpus", "hilpus", "jelpus", "zhorpus"], "5": ["hilpus", "shumpus", "lorpus", "dumpus", "sterpus", "scrompus", "felpus", "chorpus", "zumpus", "worpus", "lempus", "fompus"]}
experiments/probing_corrected/m5_linear_perm.json ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "linear_probe_accuracy": [
3
+ [
4
+ 0.031999999999999994,
5
+ 0.032,
6
+ 0.044000000000000004,
7
+ 0.0436241610738255,
8
+ 0.0,
9
+ 0.0
10
+ ],
11
+ [
12
+ 0.036,
13
+ 0.018000000000000002,
14
+ 0.05,
15
+ 0.06375838926174497,
16
+ 0.0,
17
+ 0.0
18
+ ],
19
+ [
20
+ 0.057999999999999996,
21
+ 0.030000000000000006,
22
+ 0.044000000000000004,
23
+ 0.050335570469798654,
24
+ 0.0,
25
+ 0.0
26
+ ],
27
+ [
28
+ 0.049999999999999996,
29
+ 0.03,
30
+ 0.042,
31
+ 0.0738255033557047,
32
+ 0.0,
33
+ 0.0
34
+ ],
35
+ [
36
+ 0.052000000000000005,
37
+ 0.036000000000000004,
38
+ 0.044,
39
+ 0.08389261744966443,
40
+ 0.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.06799999999999999,
45
+ 0.030000000000000006,
46
+ 0.05800000000000001,
47
+ 0.11409395973154363,
48
+ 0.0,
49
+ 0.0
50
+ ],
51
+ [
52
+ 0.07400000000000001,
53
+ 0.05800000000000001,
54
+ 0.046000000000000006,
55
+ 0.08389261744966442,
56
+ 0.0,
57
+ 0.0
58
+ ],
59
+ [
60
+ 0.10400000000000001,
61
+ 0.04,
62
+ 0.034,
63
+ 0.04697986577181208,
64
+ 0.0,
65
+ 0.0
66
+ ],
67
+ [
68
+ 0.236,
69
+ 0.054000000000000006,
70
+ 0.038,
71
+ 0.026845637583892617,
72
+ 0.0,
73
+ 0.0
74
+ ],
75
+ [
76
+ 0.14600000000000002,
77
+ 0.092,
78
+ 0.06799999999999999,
79
+ 0.030201342281879193,
80
+ 0.0,
81
+ 0.0
82
+ ],
83
+ [
84
+ 0.11000000000000001,
85
+ 0.09,
86
+ 0.06599999999999999,
87
+ 0.04697986577181208,
88
+ 0.0,
89
+ 0.0
90
+ ],
91
+ [
92
+ 0.09999999999999999,
93
+ 0.10400000000000001,
94
+ 0.21600000000000003,
95
+ 0.5302013422818792,
96
+ 0.0,
97
+ 0.0
98
+ ],
99
+ [
100
+ 0.10200000000000001,
101
+ 0.078,
102
+ 0.21999999999999997,
103
+ 0.5704697986577181,
104
+ 0.0,
105
+ 0.0
106
+ ]
107
+ ],
108
+ "permutation_p_values": [
109
+ [
110
+ 0.2463768115942029,
111
+ 0.27636181909045476,
112
+ 0.11544227886056972,
113
+ 0.19790104947526238,
114
+ 1.0,
115
+ 1.0
116
+ ],
117
+ [
118
+ 0.27786106946526734,
119
+ 0.7331334332833583,
120
+ 0.11494252873563218,
121
+ 0.053973013493253376,
122
+ 1.0,
123
+ 1.0
124
+ ],
125
+ [
126
+ 0.05347326336831584,
127
+ 0.26036981509245377,
128
+ 0.11394302848575712,
129
+ 0.059470264867566215,
130
+ 1.0,
131
+ 1.0
132
+ ],
133
+ [
134
+ 0.12893553223388307,
135
+ 0.4847576211894053,
136
+ 0.12543728135932034,
137
+ 0.020989505247376312,
138
+ 1.0,
139
+ 1.0
140
+ ],
141
+ [
142
+ 0.05547226386806597,
143
+ 0.2543728135932034,
144
+ 0.13193403298350825,
145
+ 0.0029985007496251873,
146
+ 1.0,
147
+ 1.0
148
+ ],
149
+ [
150
+ 0.019490254872563718,
151
+ 0.27636181909045476,
152
+ 0.054972513743128434,
153
+ 0.0009995002498750624,
154
+ 1.0,
155
+ 1.0
156
+ ],
157
+ [
158
+ 0.004497751124437781,
159
+ 0.04597701149425287,
160
+ 0.11594202898550725,
161
+ 0.00399800099950025,
162
+ 1.0,
163
+ 1.0
164
+ ],
165
+ [
166
+ 0.0004997501249375312,
167
+ 0.25837081459270367,
168
+ 0.26936531734132935,
169
+ 0.21039480259870064,
170
+ 1.0,
171
+ 1.0
172
+ ],
173
+ [
174
+ 0.0004997501249375312,
175
+ 0.05247376311844078,
176
+ 0.2608695652173913,
177
+ 0.45827086456771615,
178
+ 1.0,
179
+ 1.0
180
+ ],
181
+ [
182
+ 0.0004997501249375312,
183
+ 0.0004997501249375312,
184
+ 0.013993003498250875,
185
+ 0.46926536731634183,
186
+ 1.0,
187
+ 1.0
188
+ ],
189
+ [
190
+ 0.0004997501249375312,
191
+ 0.0009995002498750624,
192
+ 0.01649175412293853,
193
+ 0.18140929535232383,
194
+ 1.0,
195
+ 1.0
196
+ ],
197
+ [
198
+ 0.0009995002498750624,
199
+ 0.0004997501249375312,
200
+ 0.0004997501249375312,
201
+ 0.0004997501249375312,
202
+ 1.0,
203
+ 1.0
204
+ ],
205
+ [
206
+ 0.0004997501249375312,
207
+ 0.0029985007496251873,
208
+ 0.0004997501249375312,
209
+ 0.0004997501249375312,
210
+ 1.0,
211
+ 1.0
212
+ ]
213
+ ],
214
+ "significant_cells": [
215
+ [
216
+ false,
217
+ false,
218
+ false,
219
+ false,
220
+ false,
221
+ false
222
+ ],
223
+ [
224
+ false,
225
+ false,
226
+ false,
227
+ false,
228
+ false,
229
+ false
230
+ ],
231
+ [
232
+ false,
233
+ false,
234
+ false,
235
+ false,
236
+ false,
237
+ false
238
+ ],
239
+ [
240
+ false,
241
+ false,
242
+ false,
243
+ false,
244
+ false,
245
+ false
246
+ ],
247
+ [
248
+ false,
249
+ false,
250
+ false,
251
+ false,
252
+ false,
253
+ false
254
+ ],
255
+ [
256
+ false,
257
+ false,
258
+ false,
259
+ false,
260
+ false,
261
+ false
262
+ ],
263
+ [
264
+ false,
265
+ false,
266
+ false,
267
+ false,
268
+ false,
269
+ false
270
+ ],
271
+ [
272
+ true,
273
+ false,
274
+ false,
275
+ false,
276
+ false,
277
+ false
278
+ ],
279
+ [
280
+ true,
281
+ false,
282
+ false,
283
+ false,
284
+ false,
285
+ false
286
+ ],
287
+ [
288
+ true,
289
+ true,
290
+ false,
291
+ false,
292
+ false,
293
+ false
294
+ ],
295
+ [
296
+ true,
297
+ false,
298
+ false,
299
+ false,
300
+ false,
301
+ false
302
+ ],
303
+ [
304
+ false,
305
+ true,
306
+ true,
307
+ true,
308
+ false,
309
+ false
310
+ ],
311
+ [
312
+ true,
313
+ false,
314
+ true,
315
+ true,
316
+ false,
317
+ false
318
+ ]
319
+ ],
320
+ "n_significant_cells": 11,
321
+ "bonferroni_threshold": 0.000641025641025641,
322
+ "n_samples_per_position": {
323
+ "0": 500,
324
+ "1": 500,
325
+ "2": 500,
326
+ "3": 298,
327
+ "4": 81,
328
+ "5": 12
329
+ },
330
+ "n_permutations": 2000,
331
+ "peak_layer": 12,
332
+ "peak_position": 3,
333
+ "peak_accuracy": 0.5705
334
+ }
experiments/probing_corrected/results_linear_perm.json ADDED
@@ -0,0 +1,336 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "m5": {
3
+ "linear_probe_accuracy": [
4
+ [
5
+ 0.031999999999999994,
6
+ 0.032,
7
+ 0.044000000000000004,
8
+ 0.0436241610738255,
9
+ 0.0,
10
+ 0.0
11
+ ],
12
+ [
13
+ 0.036,
14
+ 0.018000000000000002,
15
+ 0.05,
16
+ 0.06375838926174497,
17
+ 0.0,
18
+ 0.0
19
+ ],
20
+ [
21
+ 0.057999999999999996,
22
+ 0.030000000000000006,
23
+ 0.044000000000000004,
24
+ 0.050335570469798654,
25
+ 0.0,
26
+ 0.0
27
+ ],
28
+ [
29
+ 0.049999999999999996,
30
+ 0.03,
31
+ 0.042,
32
+ 0.0738255033557047,
33
+ 0.0,
34
+ 0.0
35
+ ],
36
+ [
37
+ 0.052000000000000005,
38
+ 0.036000000000000004,
39
+ 0.044,
40
+ 0.08389261744966443,
41
+ 0.0,
42
+ 0.0
43
+ ],
44
+ [
45
+ 0.06799999999999999,
46
+ 0.030000000000000006,
47
+ 0.05800000000000001,
48
+ 0.11409395973154363,
49
+ 0.0,
50
+ 0.0
51
+ ],
52
+ [
53
+ 0.07400000000000001,
54
+ 0.05800000000000001,
55
+ 0.046000000000000006,
56
+ 0.08389261744966442,
57
+ 0.0,
58
+ 0.0
59
+ ],
60
+ [
61
+ 0.10400000000000001,
62
+ 0.04,
63
+ 0.034,
64
+ 0.04697986577181208,
65
+ 0.0,
66
+ 0.0
67
+ ],
68
+ [
69
+ 0.236,
70
+ 0.054000000000000006,
71
+ 0.038,
72
+ 0.026845637583892617,
73
+ 0.0,
74
+ 0.0
75
+ ],
76
+ [
77
+ 0.14600000000000002,
78
+ 0.092,
79
+ 0.06799999999999999,
80
+ 0.030201342281879193,
81
+ 0.0,
82
+ 0.0
83
+ ],
84
+ [
85
+ 0.11000000000000001,
86
+ 0.09,
87
+ 0.06599999999999999,
88
+ 0.04697986577181208,
89
+ 0.0,
90
+ 0.0
91
+ ],
92
+ [
93
+ 0.09999999999999999,
94
+ 0.10400000000000001,
95
+ 0.21600000000000003,
96
+ 0.5302013422818792,
97
+ 0.0,
98
+ 0.0
99
+ ],
100
+ [
101
+ 0.10200000000000001,
102
+ 0.078,
103
+ 0.21999999999999997,
104
+ 0.5704697986577181,
105
+ 0.0,
106
+ 0.0
107
+ ]
108
+ ],
109
+ "permutation_p_values": [
110
+ [
111
+ 0.2463768115942029,
112
+ 0.27636181909045476,
113
+ 0.11544227886056972,
114
+ 0.19790104947526238,
115
+ 1.0,
116
+ 1.0
117
+ ],
118
+ [
119
+ 0.27786106946526734,
120
+ 0.7331334332833583,
121
+ 0.11494252873563218,
122
+ 0.053973013493253376,
123
+ 1.0,
124
+ 1.0
125
+ ],
126
+ [
127
+ 0.05347326336831584,
128
+ 0.26036981509245377,
129
+ 0.11394302848575712,
130
+ 0.059470264867566215,
131
+ 1.0,
132
+ 1.0
133
+ ],
134
+ [
135
+ 0.12893553223388307,
136
+ 0.4847576211894053,
137
+ 0.12543728135932034,
138
+ 0.020989505247376312,
139
+ 1.0,
140
+ 1.0
141
+ ],
142
+ [
143
+ 0.05547226386806597,
144
+ 0.2543728135932034,
145
+ 0.13193403298350825,
146
+ 0.0029985007496251873,
147
+ 1.0,
148
+ 1.0
149
+ ],
150
+ [
151
+ 0.019490254872563718,
152
+ 0.27636181909045476,
153
+ 0.054972513743128434,
154
+ 0.0009995002498750624,
155
+ 1.0,
156
+ 1.0
157
+ ],
158
+ [
159
+ 0.004497751124437781,
160
+ 0.04597701149425287,
161
+ 0.11594202898550725,
162
+ 0.00399800099950025,
163
+ 1.0,
164
+ 1.0
165
+ ],
166
+ [
167
+ 0.0004997501249375312,
168
+ 0.25837081459270367,
169
+ 0.26936531734132935,
170
+ 0.21039480259870064,
171
+ 1.0,
172
+ 1.0
173
+ ],
174
+ [
175
+ 0.0004997501249375312,
176
+ 0.05247376311844078,
177
+ 0.2608695652173913,
178
+ 0.45827086456771615,
179
+ 1.0,
180
+ 1.0
181
+ ],
182
+ [
183
+ 0.0004997501249375312,
184
+ 0.0004997501249375312,
185
+ 0.013993003498250875,
186
+ 0.46926536731634183,
187
+ 1.0,
188
+ 1.0
189
+ ],
190
+ [
191
+ 0.0004997501249375312,
192
+ 0.0009995002498750624,
193
+ 0.01649175412293853,
194
+ 0.18140929535232383,
195
+ 1.0,
196
+ 1.0
197
+ ],
198
+ [
199
+ 0.0009995002498750624,
200
+ 0.0004997501249375312,
201
+ 0.0004997501249375312,
202
+ 0.0004997501249375312,
203
+ 1.0,
204
+ 1.0
205
+ ],
206
+ [
207
+ 0.0004997501249375312,
208
+ 0.0029985007496251873,
209
+ 0.0004997501249375312,
210
+ 0.0004997501249375312,
211
+ 1.0,
212
+ 1.0
213
+ ]
214
+ ],
215
+ "significant_cells": [
216
+ [
217
+ false,
218
+ false,
219
+ false,
220
+ false,
221
+ false,
222
+ false
223
+ ],
224
+ [
225
+ false,
226
+ false,
227
+ false,
228
+ false,
229
+ false,
230
+ false
231
+ ],
232
+ [
233
+ false,
234
+ false,
235
+ false,
236
+ false,
237
+ false,
238
+ false
239
+ ],
240
+ [
241
+ false,
242
+ false,
243
+ false,
244
+ false,
245
+ false,
246
+ false
247
+ ],
248
+ [
249
+ false,
250
+ false,
251
+ false,
252
+ false,
253
+ false,
254
+ false
255
+ ],
256
+ [
257
+ false,
258
+ false,
259
+ false,
260
+ false,
261
+ false,
262
+ false
263
+ ],
264
+ [
265
+ false,
266
+ false,
267
+ false,
268
+ false,
269
+ false,
270
+ false
271
+ ],
272
+ [
273
+ true,
274
+ false,
275
+ false,
276
+ false,
277
+ false,
278
+ false
279
+ ],
280
+ [
281
+ true,
282
+ false,
283
+ false,
284
+ false,
285
+ false,
286
+ false
287
+ ],
288
+ [
289
+ true,
290
+ true,
291
+ false,
292
+ false,
293
+ false,
294
+ false
295
+ ],
296
+ [
297
+ true,
298
+ false,
299
+ false,
300
+ false,
301
+ false,
302
+ false
303
+ ],
304
+ [
305
+ false,
306
+ true,
307
+ true,
308
+ true,
309
+ false,
310
+ false
311
+ ],
312
+ [
313
+ true,
314
+ false,
315
+ true,
316
+ true,
317
+ false,
318
+ false
319
+ ]
320
+ ],
321
+ "n_significant_cells": 11,
322
+ "bonferroni_threshold": 0.000641025641025641,
323
+ "n_samples_per_position": {
324
+ "0": 500,
325
+ "1": 500,
326
+ "2": 500,
327
+ "3": 298,
328
+ "4": 81,
329
+ "5": 12
330
+ },
331
+ "n_permutations": 2000,
332
+ "peak_layer": 12,
333
+ "peak_position": 3,
334
+ "peak_accuracy": 0.5705
335
+ }
336
+ }
experiments/wilcoxon_diagnostics.json ADDED
@@ -0,0 +1,508 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "diagnostic_1_probability_scale": {
3
+ "prosqa_id": {
4
+ "m3_median_prob": 1.8015017595499927e-16,
5
+ "m5_median_prob": 7.91679494328894e-14,
6
+ "ratio_m5_over_m3": 439.4552989649325
7
+ },
8
+ "7hop": {
9
+ "m3_median_prob": 1.124004303514157e-15,
10
+ "m5_median_prob": 8.064306124379313e-14,
11
+ "ratio_m5_over_m3": 71.74622107020912
12
+ },
13
+ "8hop": {
14
+ "m3_median_prob": 1.374099505807729e-15,
15
+ "m5_median_prob": 1.1024880913607591e-13,
16
+ "ratio_m5_over_m3": 80.23349740692105
17
+ },
18
+ "dag": {
19
+ "m3_median_prob": 2.452494473518025e-16,
20
+ "m5_median_prob": 6.406020952062677e-14,
21
+ "ratio_m5_over_m3": 261.20429714459027
22
+ },
23
+ "dense": {
24
+ "m3_median_prob": 9.287058992349747e-16,
25
+ "m5_median_prob": 6.880599064425863e-14,
26
+ "ratio_m5_over_m3": 74.08803012981595
27
+ }
28
+ },
29
+ "diagnostic_2_entropy": {
30
+ "prosqa_id": {
31
+ "n": 500,
32
+ "m3_mean_entropy": 8.245684373986754e-08,
33
+ "m5_mean_entropy": 7.790865615004972e-07,
34
+ "m3_std_entropy": 5.380577548978713e-07,
35
+ "m5_std_entropy": 2.5222992223835106e-06,
36
+ "m3_median_entropy": 3.2982319098096013e-09,
37
+ "m5_median_entropy": 9.583473570273782e-08,
38
+ "entropy_diff_mean": -6.966297177606296e-07,
39
+ "entropy_diff_p_value": 3.2357157290647014e-09,
40
+ "direction": "M5 > M3"
41
+ },
42
+ "dag": {
43
+ "n": 1000,
44
+ "m3_mean_entropy": 5.54877669094885e-08,
45
+ "m5_mean_entropy": 9.562996519757449e-07,
46
+ "m3_std_entropy": 1.9521541945357018e-07,
47
+ "m5_std_entropy": 4.271332058753898e-06,
48
+ "m3_median_entropy": 8.63850679877487e-09,
49
+ "m5_median_entropy": 2.2466630866802006e-07,
50
+ "entropy_diff_mean": -9.008118850662564e-07,
51
+ "entropy_diff_p_value": 3.5962255165486594e-11,
52
+ "direction": "M5 > M3"
53
+ }
54
+ },
55
+ "diagnostic_3_subsampling": {
56
+ "prosqa_id": {
57
+ "50": {
58
+ "median_r": 1.0111228967991008,
59
+ "mean_r": 1.0051491497447864,
60
+ "std_r": 0.06029258401254942,
61
+ "min_r": 0.8318166676389841,
62
+ "max_r": 1.1251537019813398,
63
+ "n_successful": 200
64
+ },
65
+ "100": {
66
+ "median_r": 0.8386074293556046,
67
+ "mean_r": 0.836016651585947,
68
+ "std_r": 0.015510156365912298,
69
+ "min_r": 0.7880640541545902,
70
+ "max_r": 0.8616442058077676,
71
+ "n_successful": 200
72
+ },
73
+ "200": {
74
+ "median_r": 0.8358284039529764,
75
+ "mean_r": 0.8359841376581776,
76
+ "std_r": 0.009534859818316218,
77
+ "min_r": 0.8060189883590921,
78
+ "max_r": 0.8577864249910184,
79
+ "n_successful": 200
80
+ },
81
+ "500": {
82
+ "median_r": 0.8347188840290514,
83
+ "mean_r": 0.8347188840290514,
84
+ "std_r": 0.0,
85
+ "min_r": 0.8347188840290514,
86
+ "max_r": 0.8347188840290514,
87
+ "n_successful": 200
88
+ }
89
+ },
90
+ "7hop": {
91
+ "50": {
92
+ "median_r": 0.9481486999740725,
93
+ "mean_r": 0.9531135622917283,
94
+ "std_r": 0.06293261512730695,
95
+ "min_r": 0.7928139676366417,
96
+ "max_r": 1.1251537019813398,
97
+ "n_successful": 200
98
+ },
99
+ "100": {
100
+ "median_r": 0.819008977747048,
101
+ "mean_r": 0.8156927134353896,
102
+ "std_r": 0.02111324353944751,
103
+ "min_r": 0.7382083439222973,
104
+ "max_r": 0.8575182159954398,
105
+ "n_successful": 200
106
+ },
107
+ "200": {
108
+ "median_r": 0.8189177079865471,
109
+ "mean_r": 0.8169341523729338,
110
+ "std_r": 0.015910137227400267,
111
+ "min_r": 0.7601185278787841,
112
+ "max_r": 0.8519194488394003,
113
+ "n_successful": 200
114
+ },
115
+ "500": {
116
+ "median_r": 0.8164834911273876,
117
+ "mean_r": 0.8167422178703777,
118
+ "std_r": 0.007052610683216399,
119
+ "min_r": 0.8009183795838579,
120
+ "max_r": 0.8364206695578106,
121
+ "n_successful": 200
122
+ },
123
+ "1000": {
124
+ "median_r": 0.8170330113820354,
125
+ "mean_r": 0.8170330113820353,
126
+ "std_r": 1.1102230246251565e-16,
127
+ "min_r": 0.8170330113820354,
128
+ "max_r": 0.8170330113820354,
129
+ "n_successful": 200
130
+ }
131
+ },
132
+ "8hop": {
133
+ "50": {
134
+ "median_r": 0.9739360145177666,
135
+ "mean_r": 0.9693214129779848,
136
+ "std_r": 0.06744026838119704,
137
+ "min_r": 0.75397404039132,
138
+ "max_r": 1.1251537019813398,
139
+ "n_successful": 200
140
+ },
141
+ "100": {
142
+ "median_r": 0.8214158051375724,
143
+ "mean_r": 0.8201642548944997,
144
+ "std_r": 0.019890168055637775,
145
+ "min_r": 0.7581506280152144,
146
+ "max_r": 0.8633633682295706,
147
+ "n_successful": 200
148
+ },
149
+ "200": {
150
+ "median_r": 0.8193922428223398,
151
+ "mean_r": 0.8200169032243652,
152
+ "std_r": 0.0133925371240747,
153
+ "min_r": 0.7714210848767548,
154
+ "max_r": 0.8509703791678149,
155
+ "n_successful": 200
156
+ },
157
+ "500": {
158
+ "median_r": 0.8197694591199104,
159
+ "mean_r": 0.819961774720315,
160
+ "std_r": 0.007195415498732053,
161
+ "min_r": 0.801264270951492,
162
+ "max_r": 0.839699719722981,
163
+ "n_successful": 200
164
+ },
165
+ "1000": {
166
+ "median_r": 0.8202902881460207,
167
+ "mean_r": 0.8202902881460207,
168
+ "std_r": 0.0,
169
+ "min_r": 0.8202902881460207,
170
+ "max_r": 0.8202902881460207,
171
+ "n_successful": 200
172
+ }
173
+ },
174
+ "dag": {
175
+ "50": {
176
+ "median_r": 1.0673612216706767,
177
+ "mean_r": 1.0618530277163836,
178
+ "std_r": 0.04394317499378165,
179
+ "min_r": 0.9294377393890321,
180
+ "max_r": 1.1251537019813398,
181
+ "n_successful": 200
182
+ },
183
+ "100": {
184
+ "median_r": 0.8544237236361942,
185
+ "mean_r": 0.8535108483902165,
186
+ "std_r": 0.00847521599972869,
187
+ "min_r": 0.8245102974968181,
188
+ "max_r": 0.8678331905262591,
189
+ "n_successful": 200
190
+ },
191
+ "200": {
192
+ "median_r": 0.8526959603888792,
193
+ "mean_r": 0.8522434267136549,
194
+ "std_r": 0.006530786353488246,
195
+ "min_r": 0.8320752647971617,
196
+ "max_r": 0.8655515404858075,
197
+ "n_successful": 200
198
+ },
199
+ "500": {
200
+ "median_r": 0.8517021501798804,
201
+ "mean_r": 0.8516264691486423,
202
+ "std_r": 0.003518625131681763,
203
+ "min_r": 0.8409449286464634,
204
+ "max_r": 0.862016630762726,
205
+ "n_successful": 200
206
+ },
207
+ "1000": {
208
+ "median_r": 0.8518384501300393,
209
+ "mean_r": 0.851838450130039,
210
+ "std_r": 3.3306690738754696e-16,
211
+ "min_r": 0.8518384501300393,
212
+ "max_r": 0.8518384501300393,
213
+ "n_successful": 200
214
+ }
215
+ },
216
+ "dense": {
217
+ "50": {
218
+ "median_r": 0.9893792786083057,
219
+ "mean_r": 0.9841720531880661,
220
+ "std_r": 0.06501513630435339,
221
+ "min_r": 0.8275968732060146,
222
+ "max_r": 1.1251537019813398,
223
+ "n_successful": 200
224
+ },
225
+ "100": {
226
+ "median_r": 0.8289801197935066,
227
+ "mean_r": 0.8266678463361812,
228
+ "std_r": 0.020971271348192252,
229
+ "min_r": 0.7578067955308537,
230
+ "max_r": 0.8661140281044558,
231
+ "n_successful": 200
232
+ },
233
+ "200": {
234
+ "median_r": 0.8226708471423619,
235
+ "mean_r": 0.8233761784664719,
236
+ "std_r": 0.013990142267202568,
237
+ "min_r": 0.780135270043129,
238
+ "max_r": 0.8511429372899214,
239
+ "n_successful": 200
240
+ },
241
+ "500": {
242
+ "median_r": 0.823643442437411,
243
+ "mean_r": 0.8231771116955668,
244
+ "std_r": 0.007503303921795524,
245
+ "min_r": 0.8019007110679384,
246
+ "max_r": 0.8400041041264988,
247
+ "n_successful": 200
248
+ },
249
+ "1000": {
250
+ "median_r": 0.82315641477788,
251
+ "mean_r": 0.8231564147778804,
252
+ "std_r": 3.3306690738754696e-16,
253
+ "min_r": 0.82315641477788,
254
+ "max_r": 0.82315641477788,
255
+ "n_successful": 200
256
+ }
257
+ }
258
+ },
259
+ "diagnostic_4_species_token": {
260
+ "dag": {
261
+ "first_token_wilcoxon": {
262
+ "n": 1000,
263
+ "direction": "M5 > M3",
264
+ "p": 7.992937269512403e-160,
265
+ "r": 0.8518384501300393,
266
+ "m3_median_lp": -35.94425582885742,
267
+ "m5_median_lp": -30.378952980041504
268
+ },
269
+ "species_token_wilcoxon": {
270
+ "n": 1000,
271
+ "direction": "M5 > M3",
272
+ "p": 1.2486659436314225e-164,
273
+ "r": 0.8647152509398429,
274
+ "m3_median_lp": -51.22725296020508,
275
+ "m5_median_lp": -38.59332847595215
276
+ },
277
+ "species_same_as_first": false,
278
+ "n_species_valid": 1000
279
+ }
280
+ },
281
+ "diagnostic_5_quartile_accuracy": {
282
+ "prosqa_id": {
283
+ "by_m3_confidence": {
284
+ "Q1": {
285
+ "n": 125,
286
+ "m3_acc": 0.984,
287
+ "m5_acc": 0.976,
288
+ "diff_m3_minus_m5": 0.008000000000000007,
289
+ "m3_lp_range": [
290
+ -50.72520446777344,
291
+ -38.92578887939453
292
+ ]
293
+ },
294
+ "Q2": {
295
+ "n": 125,
296
+ "m3_acc": 0.984,
297
+ "m5_acc": 0.944,
298
+ "diff_m3_minus_m5": 0.040000000000000036,
299
+ "m3_lp_range": [
300
+ -38.88484191894531,
301
+ -36.26152801513672
302
+ ]
303
+ },
304
+ "Q3": {
305
+ "n": 125,
306
+ "m3_acc": 0.96,
307
+ "m5_acc": 0.984,
308
+ "diff_m3_minus_m5": -0.02400000000000002,
309
+ "m3_lp_range": [
310
+ -36.243953704833984,
311
+ -33.15000915527344
312
+ ]
313
+ },
314
+ "Q4": {
315
+ "n": 125,
316
+ "m3_acc": 0.952,
317
+ "m5_acc": 0.96,
318
+ "diff_m3_minus_m5": -0.008000000000000007,
319
+ "m3_lp_range": [
320
+ -33.142913818359375,
321
+ -25.003509521484375
322
+ ]
323
+ }
324
+ },
325
+ "by_m5_confidence": {}
326
+ },
327
+ "7hop": {
328
+ "by_m3_confidence": {
329
+ "Q1": {
330
+ "n": 250,
331
+ "m3_acc": 0.66,
332
+ "m5_acc": 0.772,
333
+ "diff_m3_minus_m5": -0.11199999999999999,
334
+ "m3_lp_range": [
335
+ -47.91731262207031,
336
+ -38.02136993408203
337
+ ]
338
+ },
339
+ "Q2": {
340
+ "n": 250,
341
+ "m3_acc": 0.712,
342
+ "m5_acc": 0.74,
343
+ "diff_m3_minus_m5": -0.028000000000000025,
344
+ "m3_lp_range": [
345
+ -38.012367248535156,
346
+ -34.42694091796875
347
+ ]
348
+ },
349
+ "Q3": {
350
+ "n": 250,
351
+ "m3_acc": 0.608,
352
+ "m5_acc": 0.756,
353
+ "diff_m3_minus_m5": -0.14800000000000002,
354
+ "m3_lp_range": [
355
+ -34.41681671142578,
356
+ -31.62476348876953
357
+ ]
358
+ },
359
+ "Q4": {
360
+ "n": 250,
361
+ "m3_acc": 0.66,
362
+ "m5_acc": 0.748,
363
+ "diff_m3_minus_m5": -0.08799999999999997,
364
+ "m3_lp_range": [
365
+ -31.606525421142578,
366
+ -21.095178604125977
367
+ ]
368
+ }
369
+ },
370
+ "by_m5_confidence": {}
371
+ },
372
+ "8hop": {
373
+ "by_m3_confidence": {
374
+ "Q1": {
375
+ "n": 250,
376
+ "m3_acc": 0.676,
377
+ "m5_acc": 0.776,
378
+ "diff_m3_minus_m5": -0.09999999999999998,
379
+ "m3_lp_range": [
380
+ -47.47686767578125,
381
+ -37.30986022949219
382
+ ]
383
+ },
384
+ "Q2": {
385
+ "n": 250,
386
+ "m3_acc": 0.652,
387
+ "m5_acc": 0.744,
388
+ "diff_m3_minus_m5": -0.09199999999999997,
389
+ "m3_lp_range": [
390
+ -37.29325866699219,
391
+ -34.222503662109375
392
+ ]
393
+ },
394
+ "Q3": {
395
+ "n": 250,
396
+ "m3_acc": 0.676,
397
+ "m5_acc": 0.732,
398
+ "diff_m3_minus_m5": -0.05599999999999994,
399
+ "m3_lp_range": [
400
+ -34.219451904296875,
401
+ -31.44131088256836
402
+ ]
403
+ },
404
+ "Q4": {
405
+ "n": 250,
406
+ "m3_acc": 0.696,
407
+ "m5_acc": 0.752,
408
+ "diff_m3_minus_m5": -0.05600000000000005,
409
+ "m3_lp_range": [
410
+ -31.43999481201172,
411
+ -23.230289459228516
412
+ ]
413
+ }
414
+ },
415
+ "by_m5_confidence": {}
416
+ },
417
+ "dag": {
418
+ "by_m3_confidence": {
419
+ "Q1": {
420
+ "n": 250,
421
+ "m3_acc": 0.568,
422
+ "m5_acc": 0.516,
423
+ "diff_m3_minus_m5": 0.051999999999999935,
424
+ "m3_lp_range": [
425
+ -50.762939453125,
426
+ -39.221099853515625
427
+ ]
428
+ },
429
+ "Q2": {
430
+ "n": 250,
431
+ "m3_acc": 0.556,
432
+ "m5_acc": 0.5,
433
+ "diff_m3_minus_m5": 0.05600000000000005,
434
+ "m3_lp_range": [
435
+ -39.2105598449707,
436
+ -35.946693420410156
437
+ ]
438
+ },
439
+ "Q3": {
440
+ "n": 250,
441
+ "m3_acc": 0.636,
442
+ "m5_acc": 0.56,
443
+ "diff_m3_minus_m5": 0.07599999999999996,
444
+ "m3_lp_range": [
445
+ -35.94181823730469,
446
+ -32.98561096191406
447
+ ]
448
+ },
449
+ "Q4": {
450
+ "n": 250,
451
+ "m3_acc": 0.608,
452
+ "m5_acc": 0.5,
453
+ "diff_m3_minus_m5": 0.10799999999999998,
454
+ "m3_lp_range": [
455
+ -32.979331970214844,
456
+ -25.686195373535156
457
+ ]
458
+ }
459
+ },
460
+ "by_m5_confidence": {}
461
+ },
462
+ "dense": {
463
+ "by_m3_confidence": {
464
+ "Q1": {
465
+ "n": 250,
466
+ "m3_acc": 0.648,
467
+ "m5_acc": 0.692,
468
+ "diff_m3_minus_m5": -0.04399999999999993,
469
+ "m3_lp_range": [
470
+ -48.519752502441406,
471
+ -37.903221130371094
472
+ ]
473
+ },
474
+ "Q2": {
475
+ "n": 250,
476
+ "m3_acc": 0.632,
477
+ "m5_acc": 0.696,
478
+ "diff_m3_minus_m5": -0.06399999999999995,
479
+ "m3_lp_range": [
480
+ -37.88905334472656,
481
+ -34.615211486816406
482
+ ]
483
+ },
484
+ "Q3": {
485
+ "n": 250,
486
+ "m3_acc": 0.568,
487
+ "m5_acc": 0.648,
488
+ "diff_m3_minus_m5": -0.08000000000000007,
489
+ "m3_lp_range": [
490
+ -34.610267639160156,
491
+ -31.857315063476562
492
+ ]
493
+ },
494
+ "Q4": {
495
+ "n": 250,
496
+ "m3_acc": 0.6,
497
+ "m5_acc": 0.7,
498
+ "diff_m3_minus_m5": -0.09999999999999998,
499
+ "m3_lp_range": [
500
+ -31.839698791503906,
501
+ -22.86867332458496
502
+ ]
503
+ }
504
+ },
505
+ "by_m5_confidence": {}
506
+ }
507
+ }
508
+ }
experiments/wilcoxon_sensitivity.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "Wilcoxon signed-rank test on paired per-sample first-answer-token log-probabilities",
3
+ "description": "Log P(first correct answer token | input + thoughts) under each model. No answer tokens appended; logits from last position predict first answer token.",
4
+ "bonferroni_k": 5,
5
+ "adjusted_alpha": 0.01,
6
+ "results": {
7
+ "prosqa_id": {
8
+ "n": 500,
9
+ "median_diff": 5.5448455810546875,
10
+ "mean_diff": 5.900060585021973,
11
+ "std_diff": 4.304266040095551,
12
+ "wilcoxon_stat": 2294.0,
13
+ "p_value": 9.558665789472875e-78,
14
+ "p_bonferroni": 4.779332894736438e-77,
15
+ "effect_size_r": 0.8347188840290514,
16
+ "z_score": 18.664881667917225,
17
+ "m3_median_logprob": -36.25274085998535,
18
+ "m5_median_logprob": -30.16720485687256,
19
+ "m3_mean_logprob": -36.290785385131834,
20
+ "m5_mean_logprob": -30.39072480010986,
21
+ "direction": "M5 > M3"
22
+ },
23
+ "7hop": {
24
+ "n": 1000,
25
+ "median_diff": 4.609404563903809,
26
+ "mean_diff": 4.781453802108764,
27
+ "std_diff": 3.692269596981685,
28
+ "wilcoxon_stat": 14216.0,
29
+ "p_value": 3.419355734386437e-147,
30
+ "p_bonferroni": 1.7096778671932184e-146,
31
+ "effect_size_r": 0.8170330113820354,
32
+ "z_score": 25.83685239513508,
33
+ "m3_median_logprob": -34.421878814697266,
34
+ "m5_median_logprob": -30.148743629455566,
35
+ "m3_mean_logprob": -34.93062486076355,
36
+ "m5_mean_logprob": -30.149171058654787,
37
+ "direction": "M5 > M3"
38
+ },
39
+ "8hop": {
40
+ "n": 1000,
41
+ "median_diff": 4.223997116088867,
42
+ "mean_diff": 4.518639762878418,
43
+ "std_diff": 3.4717507040890796,
44
+ "wilcoxon_stat": 13275.0,
45
+ "p_value": 2.3666197544095806e-148,
46
+ "p_bonferroni": 1.1833098772047903e-147,
47
+ "effect_size_r": 0.8202902881460207,
48
+ "z_score": 25.93985653057244,
49
+ "m3_median_logprob": -34.220977783203125,
50
+ "m5_median_logprob": -29.836036682128906,
51
+ "m3_mean_logprob": -34.49567288208008,
52
+ "m5_mean_logprob": -29.97703311920166,
53
+ "direction": "M5 > M3"
54
+ },
55
+ "dag": {
56
+ "n": 1000,
57
+ "median_diff": 5.726862907409668,
58
+ "mean_diff": 5.837504280090332,
59
+ "std_diff": 3.5533476769526535,
60
+ "wilcoxon_stat": 4161.0,
61
+ "p_value": 7.992937269512403e-160,
62
+ "p_bonferroni": 3.9964686347562014e-159,
63
+ "effect_size_r": 0.8518384501300393,
64
+ "z_score": 26.937497009186796,
65
+ "m3_median_logprob": -35.94425582885742,
66
+ "m5_median_logprob": -30.378952980041504,
67
+ "m3_mean_logprob": -36.28265309143067,
68
+ "m5_mean_logprob": -30.44514881134033,
69
+ "direction": "M5 > M3"
70
+ },
71
+ "dense": {
72
+ "n": 1000,
73
+ "median_diff": 4.454345703125,
74
+ "mean_diff": 4.632727506637573,
75
+ "std_diff": 3.4062677335293263,
76
+ "wilcoxon_stat": 12447.0,
77
+ "p_value": 2.237606774559982e-149,
78
+ "p_bonferroni": 1.118803387279991e-148,
79
+ "effect_size_r": 0.82315641477788,
80
+ "z_score": 26.030491412763865,
81
+ "m3_median_logprob": -34.61273956298828,
82
+ "m5_median_logprob": -30.307485580444336,
83
+ "m3_mean_logprob": -35.10507657432556,
84
+ "m5_mean_logprob": -30.47234906768799,
85
+ "direction": "M5 > M3"
86
+ }
87
+ }
88
+ }
experiments/wilcoxon_sensitivity_v3.json ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "Wilcoxon signed-rank test on paired per-sample first-answer-token log-probabilities",
3
+ "description": "Log P(first correct answer token | input + thoughts + ### separator). v3: Fixed extraction point (### position) AND tokenization (leading space).",
4
+ "fixes": [
5
+ "Appended ### separator token so logits[-1] is at the answer-predicting position",
6
+ "Tokenized answer with leading space to match model generation format (### Sally...)"
7
+ ],
8
+ "bonferroni_k": 5,
9
+ "adjusted_alpha": 0.01,
10
+ "results": {
11
+ "prosqa_id": {
12
+ "n": 500,
13
+ "median_diff": 0.0,
14
+ "mean_diff": 1.9073484480713887e-09,
15
+ "std_diff": 3.2808333491232266e-08,
16
+ "wilcoxon_stat": 59.0,
17
+ "p_value": 0.22669185385013768,
18
+ "p_bonferroni": 1.0,
19
+ "effect_size_r": 0.05406474709320131,
20
+ "z_score": 1.208924496867323,
21
+ "m3_median_logprob": 0.0,
22
+ "m5_median_logprob": 0.0,
23
+ "m3_mean_logprob": -3.814696910353632e-09,
24
+ "m5_mean_logprob": -1.9073484622822432e-09,
25
+ "m3_prob_median": 1.0,
26
+ "m5_prob_median": 1.0,
27
+ "direction": "equal"
28
+ },
29
+ "7hop": {
30
+ "n": 1000,
31
+ "median_diff": 0.0,
32
+ "mean_diff": 1.5545431362859574e-05,
33
+ "std_diff": 0.000491480095991523,
34
+ "wilcoxon_stat": 817.5,
35
+ "p_value": 2.825225828337809e-10,
36
+ "p_bonferroni": 1.4126129141689046e-09,
37
+ "effect_size_r": 0.19947892842275394,
38
+ "z_score": 6.308077590256019,
39
+ "m3_median_logprob": 0.0,
40
+ "m5_median_logprob": 0.0,
41
+ "m3_mean_logprob": -1.5582147784044765e-05,
42
+ "m5_mean_logprob": -3.671642118519003e-08,
43
+ "m3_prob_median": 1.0,
44
+ "m5_prob_median": 1.0,
45
+ "direction": "equal"
46
+ },
47
+ "8hop": {
48
+ "n": 1000,
49
+ "median_diff": 0.0,
50
+ "mean_diff": -2.5510785491178468e-08,
51
+ "std_diff": 3.456691393519963e-07,
52
+ "wilcoxon_stat": 556.0,
53
+ "p_value": 6.337349739272441e-11,
54
+ "p_bonferroni": 3.1686748696362206e-10,
55
+ "effect_size_r": 0.20667243975845923,
56
+ "z_score": 6.535556392206708,
57
+ "m3_median_logprob": 0.0,
58
+ "m5_median_logprob": 0.0,
59
+ "m3_mean_logprob": -1.1801690376955776e-08,
60
+ "m5_mean_logprob": -3.7312475868134245e-08,
61
+ "m3_prob_median": 1.0,
62
+ "m5_prob_median": 1.0,
63
+ "direction": "equal"
64
+ },
65
+ "dag": {
66
+ "n": 1000,
67
+ "median_diff": 0.0,
68
+ "mean_diff": -3.8504580892606554e-08,
69
+ "std_diff": 2.2670660050335698e-07,
70
+ "wilcoxon_stat": 1072.5,
71
+ "p_value": 7.740160468120871e-16,
72
+ "p_bonferroni": 3.870080234060436e-15,
73
+ "effect_size_r": 0.25482388020266233,
74
+ "z_score": 8.058238636423024,
75
+ "m3_median_logprob": 0.0,
76
+ "m5_median_logprob": 0.0,
77
+ "m3_mean_logprob": -6.675716875292892e-09,
78
+ "m5_mean_logprob": -4.518029776789945e-08,
79
+ "m3_prob_median": 1.0,
80
+ "m5_prob_median": 1.0,
81
+ "direction": "equal"
82
+ },
83
+ "dense": {
84
+ "n": 1000,
85
+ "median_diff": 0.0,
86
+ "mean_diff": 1.8954253092857698e-06,
87
+ "std_diff": 6.556349770050734e-05,
88
+ "wilcoxon_stat": 3283.5,
89
+ "p_value": 4.2083786008929025e-43,
90
+ "p_bonferroni": 2.104189300446451e-42,
91
+ "effect_size_r": 0.4352494406715319,
92
+ "z_score": 13.763795828363676,
93
+ "m3_median_logprob": 0.0,
94
+ "m5_median_logprob": 0.0,
95
+ "m3_mean_logprob": -2.1150066327138006e-06,
96
+ "m5_mean_logprob": -2.1958132342803083e-07,
97
+ "m3_prob_median": 1.0,
98
+ "m5_prob_median": 1.0,
99
+ "direction": "equal"
100
+ }
101
+ }
102
+ }
experiments/wilcoxon_teacher_forced_m3_vs_m5.json ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "Wilcoxon signed-rank on teacher-forced species token log-probabilities",
3
+ "description": "Teacher-force through '### [Name] is a' prefix, then extract log P(species_token). Species token is the discriminative one requiring multi-hop reasoning. Comparison: m3 vs m5.",
4
+ "comparison": "m3_vs_m5",
5
+ "bonferroni_k": 5,
6
+ "adjusted_alpha": 0.01,
7
+ "results": {
8
+ "prosqa_id": {
9
+ "n": 500,
10
+ "n_ties": 0,
11
+ "n_nonzero": 500,
12
+ "median_diff": -0.0001227500616778343,
13
+ "mean_diff": -0.06978289037453805,
14
+ "std_diff": 1.0628266559166086,
15
+ "wilcoxon_stat": 19908.0,
16
+ "p_value": 7.135644023141254e-40,
17
+ "p_bonferroni": 3.567822011570627e-39,
18
+ "effect_size_r": 0.5910176620488471,
19
+ "z_score": 13.215556682442198,
20
+ "m3_median_lp": -1.9490528757160064e-05,
21
+ "m5_median_lp": -0.0002181292074965313,
22
+ "m3_mean_lp": -0.11297737371596042,
23
+ "m5_mean_lp": -0.18276026409049848,
24
+ "m3_prob_median": 0.999980509661182,
25
+ "m5_prob_median": 0.9997818945809493,
26
+ "direction": "m3>m5"
27
+ },
28
+ "7hop": {
29
+ "n": 1000,
30
+ "n_ties": 0,
31
+ "n_nonzero": 1000,
32
+ "median_diff": -0.00032740284859755775,
33
+ "mean_diff": 0.2813422611065431,
34
+ "std_diff": 2.906805478757986,
35
+ "wilcoxon_stat": 248577.0,
36
+ "p_value": 0.8546955264303868,
37
+ "p_bonferroni": 1.0,
38
+ "effect_size_r": 0.005791098858817567,
39
+ "z_score": 0.18313062549065387,
40
+ "m3_median_lp": -0.026303524151444435,
41
+ "m5_median_lp": -0.037403758615255356,
42
+ "m3_mean_lp": -1.3909491490889976,
43
+ "m5_mean_lp": -1.1096068879824543,
44
+ "m3_prob_median": 0.9740394002541536,
45
+ "m5_prob_median": 0.9632871213462526,
46
+ "direction": "m3>m5"
47
+ },
48
+ "8hop": {
49
+ "n": 1000,
50
+ "n_ties": 0,
51
+ "n_nonzero": 1000,
52
+ "median_diff": -0.00027873104806985793,
53
+ "mean_diff": 0.21474395308568736,
54
+ "std_diff": 2.776021865502376,
55
+ "wilcoxon_stat": 248628.0,
56
+ "p_value": 0.8590779324092401,
57
+ "p_bonferroni": 1.0,
58
+ "effect_size_r": 0.005614562073521871,
59
+ "z_score": 0.17754804216726866,
60
+ "m3_median_lp": -0.02322522271424532,
61
+ "m5_median_lp": -0.04081491939723492,
62
+ "m3_mean_lp": -1.3103880740790188,
63
+ "m5_mean_lp": -1.0956441209933316,
64
+ "m3_prob_median": 0.9770424068482431,
65
+ "m5_prob_median": 0.960006792142127,
66
+ "direction": "m3>m5"
67
+ },
68
+ "dag": {
69
+ "n": 1000,
70
+ "n_ties": 0,
71
+ "n_nonzero": 1000,
72
+ "median_diff": -0.0010396945101547317,
73
+ "mean_diff": 0.16994009311438624,
74
+ "std_diff": 2.963397551627641,
75
+ "wilcoxon_stat": 249483.0,
76
+ "p_value": 0.9330900836102644,
77
+ "p_bonferroni": 1.0,
78
+ "effect_size_r": 0.0026549747906234756,
79
+ "z_score": 0.08395767468698836,
80
+ "m3_median_lp": -0.22533510625362396,
81
+ "m5_median_lp": -0.34515246748924255,
82
+ "m3_mean_lp": -1.8029823147184267,
83
+ "m5_mean_lp": -1.6330422216040406,
84
+ "m3_prob_median": 0.7982486758110119,
85
+ "m5_prob_median": 0.7081123811200218,
86
+ "direction": "m3>m5"
87
+ },
88
+ "dense": {
89
+ "n": 1000,
90
+ "n_ties": 0,
91
+ "n_nonzero": 1000,
92
+ "median_diff": 0.00026002148661063984,
93
+ "mean_diff": 0.5418239545176224,
94
+ "std_diff": 2.9624931634035607,
95
+ "wilcoxon_stat": 217527.0,
96
+ "p_value": 0.0003410538200066114,
97
+ "p_bonferroni": 0.001705269100033057,
98
+ "effect_size_r": 0.11327084755354884,
99
+ "z_score": 3.581938707669256,
100
+ "m3_median_lp": -0.14667461067438126,
101
+ "m5_median_lp": -0.1057770662009716,
102
+ "m3_mean_lp": -1.6534713189456967,
103
+ "m5_mean_lp": -1.111647364428074,
104
+ "m3_prob_median": 0.8635749297684109,
105
+ "m5_prob_median": 0.8996251825817885,
106
+ "direction": "m5>m3"
107
+ }
108
+ }
109
+ }
experiments/wilcoxon_teacher_forced_m3_vs_m6.json ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "Wilcoxon signed-rank on teacher-forced species token log-probabilities",
3
+ "description": "Teacher-force through '### [Name] is a' prefix, then extract log P(species_token). Species token is the discriminative one requiring multi-hop reasoning. Comparison: m3 vs m6.",
4
+ "comparison": "m3_vs_m6",
5
+ "bonferroni_k": 5,
6
+ "adjusted_alpha": 0.01,
7
+ "results": {
8
+ "prosqa_id": {
9
+ "n": 500,
10
+ "n_ties": 0,
11
+ "n_nonzero": 500,
12
+ "median_diff": -0.00036554490657181304,
13
+ "mean_diff": -0.0860310743720925,
14
+ "std_diff": 1.2170259847022458,
15
+ "wilcoxon_stat": 13588.0,
16
+ "p_value": 5.51961631585719e-52,
17
+ "p_bonferroni": 2.759808157928595e-51,
18
+ "effect_size_r": 0.6784589997867199,
19
+ "z_score": 15.17080443469621,
20
+ "m3_median_lp": -1.9490528757160064e-05,
21
+ "m6_median_lp": -0.0005059154354967177,
22
+ "m3_mean_lp": -0.11297739825633377,
23
+ "m6_mean_lp": -0.19900847262842628,
24
+ "m3_prob_median": 0.999980509661182,
25
+ "m6_prob_median": 0.9994942125181384,
26
+ "direction": "m3>m6"
27
+ },
28
+ "7hop": {
29
+ "n": 1000,
30
+ "n_ties": 0,
31
+ "n_nonzero": 1000,
32
+ "median_diff": -0.00025782780085137347,
33
+ "mean_diff": 0.7641824833154048,
34
+ "std_diff": 2.6517137931258605,
35
+ "wilcoxon_stat": 218786.0,
36
+ "p_value": 0.0005729096557360483,
37
+ "p_bonferroni": 0.0028645482786802414,
38
+ "effect_size_r": 0.10891281201066103,
39
+ "z_score": 3.444125523274317,
40
+ "m3_median_lp": -0.026303233578801155,
41
+ "m6_median_lp": -0.03395153023302555,
42
+ "m3_mean_lp": -1.3909488494071465,
43
+ "m6_mean_lp": -0.6267663660917415,
44
+ "m3_prob_median": 0.974039683283398,
45
+ "m6_prob_median": 0.966618355268511,
46
+ "direction": "m3>m6"
47
+ },
48
+ "8hop": {
49
+ "n": 1000,
50
+ "n_ties": 0,
51
+ "n_nonzero": 1000,
52
+ "median_diff": -0.0003167157010466326,
53
+ "mean_diff": 0.5979843548741289,
54
+ "std_diff": 2.54137471092738,
55
+ "wilcoxon_stat": 226686.0,
56
+ "p_value": 0.009898001252072113,
57
+ "p_bonferroni": 0.04949000626036057,
58
+ "effect_size_r": 0.08156691781779862,
59
+ "z_score": 2.5793724202401473,
60
+ "m3_median_lp": -0.023225629702210426,
61
+ "m6_median_lp": -0.03081549145281315,
62
+ "m3_mean_lp": -1.310387969426784,
63
+ "m6_mean_lp": -0.7124036145526548,
64
+ "m3_prob_median": 0.977042009203823,
65
+ "m6_prob_median": 0.9696544661086867,
66
+ "direction": "m3>m6"
67
+ },
68
+ "dag": {
69
+ "n": 1000,
70
+ "n_ties": 0,
71
+ "n_nonzero": 1000,
72
+ "median_diff": -0.0005894981513847597,
73
+ "mean_diff": 0.5916298918708122,
74
+ "std_diff": 2.942288236114359,
75
+ "wilcoxon_stat": 229183.0,
76
+ "p_value": 0.021108131302877156,
77
+ "p_bonferroni": 0.10554065651438578,
78
+ "effect_size_r": 0.07292353834949769,
79
+ "z_score": 2.3060447622304863,
80
+ "m3_median_lp": -0.22532972693443298,
81
+ "m6_median_lp": -0.2986634820699692,
82
+ "m3_mean_lp": -1.802982120641987,
83
+ "m6_mean_lp": -1.2113522287711749,
84
+ "m3_prob_median": 0.7982529698569824,
85
+ "m6_prob_median": 0.7418089994658733,
86
+ "direction": "m3>m6"
87
+ },
88
+ "dense": {
89
+ "n": 1000,
90
+ "n_ties": 0,
91
+ "n_nonzero": 1000,
92
+ "median_diff": -0.00013662463516084244,
93
+ "mean_diff": 0.6544579904913999,
94
+ "std_diff": 2.796134035440923,
95
+ "wilcoxon_stat": 216195.0,
96
+ "p_value": 0.000193202514204543,
97
+ "p_bonferroni": 0.000966012571022715,
98
+ "effect_size_r": 0.1178815730048011,
99
+ "z_score": 3.7277426485859038,
100
+ "m3_median_lp": -0.1466745063662529,
101
+ "m6_median_lp": -0.16426319628953934,
102
+ "m3_mean_lp": -1.65347114397882,
103
+ "m6_mean_lp": -0.9990131534874199,
104
+ "m3_prob_median": 0.8635750198463003,
105
+ "m6_prob_median": 0.8485186655238433,
106
+ "direction": "m3>m6"
107
+ }
108
+ }
109
+ }
experiments/wilcoxon_teacher_forced_m5_vs_m6.json ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "Wilcoxon signed-rank on teacher-forced species token log-probabilities",
3
+ "description": "Teacher-force through '### [Name] is a' prefix, then extract log P(species_token). Species token is the discriminative one requiring multi-hop reasoning. Comparison: m5 vs m6.",
4
+ "comparison": "m5_vs_m6",
5
+ "bonferroni_k": 5,
6
+ "adjusted_alpha": 0.01,
7
+ "results": {
8
+ "prosqa_id": {
9
+ "n": 500,
10
+ "n_ties": 0,
11
+ "n_nonzero": 500,
12
+ "median_diff": -0.00013814407611789647,
13
+ "mean_diff": -0.016247838157780962,
14
+ "std_diff": 1.1177613567572349,
15
+ "wilcoxon_stat": 41957.0,
16
+ "p_value": 1.6143744744439177e-10,
17
+ "p_bonferroni": 8.071872372219589e-10,
18
+ "effect_size_r": 0.28595531145037273,
19
+ "z_score": 6.394155149301574,
20
+ "m5_median_lp": -0.00021824839495820925,
21
+ "m6_median_lp": -0.0005059154354967177,
22
+ "m5_mean_lp": -0.1827606344706453,
23
+ "m6_mean_lp": -0.19900847262842628,
24
+ "m5_prob_median": 0.9997817754194902,
25
+ "m6_prob_median": 0.9994942125181384,
26
+ "direction": "m5>m6"
27
+ },
28
+ "7hop": {
29
+ "n": 1000,
30
+ "n_ties": 0,
31
+ "n_nonzero": 1000,
32
+ "median_diff": 0.0001795582447812194,
33
+ "mean_diff": 0.48284089019898646,
34
+ "std_diff": 2.0365023730650087,
35
+ "wilcoxon_stat": 209203.0,
36
+ "p_value": 7.0192524328014474e-06,
37
+ "p_bonferroni": 3.5096262164007236e-05,
38
+ "effect_size_r": 0.1420844201182813,
39
+ "z_score": 4.493103875980196,
40
+ "m5_median_lp": -0.03740513697266579,
41
+ "m6_median_lp": -0.03395153023302555,
42
+ "m5_mean_lp": -1.109607256290728,
43
+ "m6_mean_lp": -0.6267663660917415,
44
+ "m5_prob_median": 0.9632857935932256,
45
+ "m6_prob_median": 0.966618355268511,
46
+ "direction": "m6>m5"
47
+ },
48
+ "8hop": {
49
+ "n": 1000,
50
+ "n_ties": 0,
51
+ "n_nonzero": 1000,
52
+ "median_diff": 0.0002561149522080086,
53
+ "mean_diff": 0.3832407745273106,
54
+ "std_diff": 2.227865684000055,
55
+ "wilcoxon_stat": 215715.0,
56
+ "p_value": 0.00015664918728829126,
57
+ "p_bonferroni": 0.0007832459364414563,
58
+ "effect_size_r": 0.119543095689937,
59
+ "z_score": 3.7802846092765865,
60
+ "m5_median_lp": -0.04081503301858902,
61
+ "m6_median_lp": -0.03081549145281315,
62
+ "m5_mean_lp": -1.0956443890799654,
63
+ "m6_mean_lp": -0.7124036145526548,
64
+ "m5_prob_median": 0.9600066830648615,
65
+ "m6_prob_median": 0.9696544661086867,
66
+ "direction": "m6>m5"
67
+ },
68
+ "dag": {
69
+ "n": 1000,
70
+ "n_ties": 0,
71
+ "n_nonzero": 1000,
72
+ "median_diff": 0.0026662750460673124,
73
+ "mean_diff": 0.4216908030509094,
74
+ "std_diff": 2.3045800471066755,
75
+ "wilcoxon_stat": 210880.0,
76
+ "p_value": 1.6359787033524354e-05,
77
+ "p_bonferroni": 8.179893516762177e-05,
78
+ "effect_size_r": 0.13627947523708756,
79
+ "z_score": 4.3095354008171185,
80
+ "m5_median_lp": -0.34515656530857086,
81
+ "m6_median_lp": -0.2986634820699692,
82
+ "m5_mean_lp": -1.6330430318220843,
83
+ "m6_mean_lp": -1.2113522287711749,
84
+ "m5_prob_median": 0.7081094794093652,
85
+ "m6_prob_median": 0.7418089994658733,
86
+ "direction": "m6>m5"
87
+ },
88
+ "dense": {
89
+ "n": 1000,
90
+ "n_ties": 0,
91
+ "n_nonzero": 1000,
92
+ "median_diff": -0.003164976616972126,
93
+ "mean_diff": 0.11263441988590966,
94
+ "std_diff": 2.239913360245438,
95
+ "wilcoxon_stat": 244040.0,
96
+ "p_value": 0.49665541423174153,
97
+ "p_bonferroni": 1.0,
98
+ "effect_size_r": 0.021495949738946255,
99
+ "z_score": 0.6797616164357204,
100
+ "m5_median_lp": -0.10577894374728203,
101
+ "m6_median_lp": -0.16426319628953934,
102
+ "m5_mean_lp": -1.1116475733733295,
103
+ "m6_mean_lp": -0.9990131534874199,
104
+ "m5_prob_median": 0.8996234934954318,
105
+ "m6_prob_median": 0.8485186655238433,
106
+ "direction": "m5>m6"
107
+ }
108
+ }
109
+ }