Hugging Face's logo

Spaces:
Dar3devil
/
promptops-arena
Sleeping

App Files Community
Dar3devil commited on
Commit
96e3856
·
verified ·
1 Parent(s): 4017fa2

submission update: untrained baseline + cleanup

Browse files
Files changed (1) hide show
  1. results/comparison.json +53 -104
results/comparison.json CHANGED
@@ -1,53 +1,5 @@
1
  {
2
  "policies": {
3
- "zero_shot (stub)": {
4
- "n": 30,
5
- "correct": 0,
6
- "format": 0,
7
- "mean_reward": 0.0,
8
- "by_type": {
9
- "math": {
10
- "n": 10,
11
- "correct": 0,
12
- "format": 0
13
- },
14
- "code": {
15
- "n": 10,
16
- "correct": 0,
17
- "format": 0
18
- },
19
- "json": {
20
- "n": 10,
21
- "correct": 0,
22
- "format": 0
23
- }
24
- },
25
- "backend": "stub"
26
- },
27
- "cot (stub)": {
28
- "n": 30,
29
- "correct": 0,
30
- "format": 30,
31
- "mean_reward": 0.1,
32
- "by_type": {
33
- "math": {
34
- "n": 10,
35
- "correct": 0,
36
- "format": 10
37
- },
38
- "code": {
39
- "n": 10,
40
- "correct": 0,
41
- "format": 10
42
- },
43
- "json": {
44
- "n": 10,
45
- "correct": 0,
46
- "format": 10
47
- }
48
- },
49
- "backend": "stub"
50
- },
51
  "zero_shot (real LLM)": {
52
  "n": 12,
53
  "correct": 8,
@@ -96,7 +48,31 @@
96
  },
97
  "backend": "transformers"
98
  },
99
- "trained agent (real LLM)": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  "n": 12,
101
  "correct": 10,
102
  "format": 10,
@@ -123,7 +99,34 @@
123
  },
124
  "ranking_by_mean_reward": [
125
  [
126
- "trained agent (real LLM)",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
  {
128
  "n": 12,
129
  "correct": 10,
@@ -202,60 +205,6 @@
202
  },
203
  "backend": "transformers"
204
  }
205
- ],
206
- [
207
- "cot (stub)",
208
- {
209
- "n": 30,
210
- "correct": 0,
211
- "format": 30,
212
- "mean_reward": 0.1,
213
- "by_type": {
214
- "math": {
215
- "n": 10,
216
- "correct": 0,
217
- "format": 10
218
- },
219
- "code": {
220
- "n": 10,
221
- "correct": 0,
222
- "format": 10
223
- },
224
- "json": {
225
- "n": 10,
226
- "correct": 0,
227
- "format": 10
228
- }
229
- },
230
- "backend": "stub"
231
- }
232
- ],
233
- [
234
- "zero_shot (stub)",
235
- {
236
- "n": 30,
237
- "correct": 0,
238
- "format": 0,
239
- "mean_reward": 0.0,
240
- "by_type": {
241
- "math": {
242
- "n": 10,
243
- "correct": 0,
244
- "format": 0
245
- },
246
- "code": {
247
- "n": 10,
248
- "correct": 0,
249
- "format": 0
250
- },
251
- "json": {
252
- "n": 10,
253
- "correct": 0,
254
- "format": 0
255
- }
256
- },
257
- "backend": "stub"
258
- }
259
  ]
260
  ]
261
  }
 
1
  {
2
  "policies": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "zero_shot (real LLM)": {
4
  "n": 12,
5
  "correct": 8,
 
48
  },
49
  "backend": "transformers"
50
  },
51
+ "untrained 1.5B agent (real LLM, 3 turns)": {
52
+ "n": 12,
53
+ "correct": 11,
54
+ "format": 10,
55
+ "mean_reward": 0.9999999999999999,
56
+ "by_type": {
57
+ "math": {
58
+ "n": 4,
59
+ "correct": 3,
60
+ "format": 2
61
+ },
62
+ "code": {
63
+ "n": 4,
64
+ "correct": 4,
65
+ "format": 4
66
+ },
67
+ "json": {
68
+ "n": 4,
69
+ "correct": 4,
70
+ "format": 4
71
+ }
72
+ },
73
+ "backend": "transformers"
74
+ },
75
+ "trained 1.5B agent (real LLM, 2 turns)": {
76
  "n": 12,
77
  "correct": 10,
78
  "format": 10,
 
99
  },
100
  "ranking_by_mean_reward": [
101
  [
102
+ "untrained 1.5B agent (real LLM, 3 turns)",
103
+ {
104
+ "n": 12,
105
+ "correct": 11,
106
+ "format": 10,
107
+ "mean_reward": 0.9999999999999999,
108
+ "by_type": {
109
+ "math": {
110
+ "n": 4,
111
+ "correct": 3,
112
+ "format": 2
113
+ },
114
+ "code": {
115
+ "n": 4,
116
+ "correct": 4,
117
+ "format": 4
118
+ },
119
+ "json": {
120
+ "n": 4,
121
+ "correct": 4,
122
+ "format": 4
123
+ }
124
+ },
125
+ "backend": "transformers"
126
+ }
127
+ ],
128
+ [
129
+ "trained 1.5B agent (real LLM, 2 turns)",
130
  {
131
  "n": 12,
132
  "correct": 10,
 
205
  },
206
  "backend": "transformers"
207
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
208
  ]
209
  ]
210
  }