d4rkninja commited on
Commit
6b33725
·
verified ·
1 Parent(s): f38c330

docs: evaluation/COMPARE.json

Browse files
Files changed (1) hide show
  1. evaluation/COMPARE.json +260 -0
evaluation/COMPARE.json ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": "darklab-base-vs-ft-v2",
3
+ "generated_at": "2026-09-18T16:55:59.037166+00:00",
4
+ "base": "unsloth/LFM2.5-1.2B-Instruct",
5
+ "owner": "DarkNinja Solutions",
6
+ "creator": "d4rkninja",
7
+ "scoring": "deterministic_rubric_v2_strict",
8
+ "model_order": [
9
+ "finance"
10
+ ],
11
+ "base_models": {
12
+ "finance": {
13
+ "display_name": "base/tanpo-finance",
14
+ "key": "finance",
15
+ "model_id": "base:unsloth/LFM2.5-1.2B-Instruct",
16
+ "base": "unsloth/LFM2.5-1.2B-Instruct",
17
+ "status": "ok",
18
+ "error": null,
19
+ "lora_path": null,
20
+ "load_seconds": 20.2,
21
+ "gen_seconds": 156.0,
22
+ "overall_pct": 93.08,
23
+ "quality_avg": 6.54,
24
+ "n_tasks": 20,
25
+ "by_category": {
26
+ "unit_economics": {
27
+ "score_pct": 94.45,
28
+ "quality_avg": 6.36,
29
+ "n": 2
30
+ },
31
+ "runway_cash": {
32
+ "score_pct": 100.0,
33
+ "quality_avg": 8.0,
34
+ "n": 2
35
+ },
36
+ "pricing_finance": {
37
+ "score_pct": 100.0,
38
+ "quality_avg": 7.4,
39
+ "n": 2
40
+ },
41
+ "saas_metrics": {
42
+ "score_pct": 100.0,
43
+ "quality_avg": 8.0,
44
+ "n": 2
45
+ },
46
+ "budget_tradeoffs": {
47
+ "score_pct": 94.45,
48
+ "quality_avg": 7.51,
49
+ "n": 2
50
+ },
51
+ "burn_efficiency": {
52
+ "score_pct": 94.45,
53
+ "quality_avg": 7.36,
54
+ "n": 2
55
+ },
56
+ "cohort_revenue": {
57
+ "score_pct": 94.45,
58
+ "quality_avg": 5.91,
59
+ "n": 2
60
+ },
61
+ "cost_structure": {
62
+ "score_pct": 100.0,
63
+ "quality_avg": 7.5,
64
+ "n": 2
65
+ },
66
+ "forecast_scenario": {
67
+ "score_pct": 94.45,
68
+ "quality_avg": 6.36,
69
+ "n": 2
70
+ },
71
+ "identity_guardrails": {
72
+ "score_pct": 58.57,
73
+ "quality_avg": 0.96,
74
+ "n": 2
75
+ }
76
+ },
77
+ "categories": [
78
+ "budget_tradeoffs",
79
+ "burn_efficiency",
80
+ "cohort_revenue",
81
+ "cost_structure",
82
+ "forecast_scenario",
83
+ "identity_guardrails",
84
+ "pricing_finance",
85
+ "runway_cash",
86
+ "saas_metrics",
87
+ "unit_economics"
88
+ ]
89
+ }
90
+ },
91
+ "ft_models": {
92
+ "finance": {
93
+ "display_name": "tanpo-finance",
94
+ "key": "finance",
95
+ "model_id": "peft:tanpo-finance",
96
+ "base": "unsloth/LFM2.5-1.2B-Instruct",
97
+ "status": "ok",
98
+ "error": null,
99
+ "lora_path": "C:\\Users\\alien\\tanpo-finance-lora-format-v2",
100
+ "load_seconds": 8.9,
101
+ "gen_seconds": 185.9,
102
+ "overall_pct": 98.44,
103
+ "quality_avg": 7.86,
104
+ "n_tasks": 20,
105
+ "by_category": {
106
+ "unit_economics": {
107
+ "score_pct": 100.0,
108
+ "quality_avg": 8.0,
109
+ "n": 2
110
+ },
111
+ "runway_cash": {
112
+ "score_pct": 100.0,
113
+ "quality_avg": 8.5,
114
+ "n": 2
115
+ },
116
+ "pricing_finance": {
117
+ "score_pct": 100.0,
118
+ "quality_avg": 8.0,
119
+ "n": 2
120
+ },
121
+ "saas_metrics": {
122
+ "score_pct": 100.0,
123
+ "quality_avg": 8.0,
124
+ "n": 2
125
+ },
126
+ "budget_tradeoffs": {
127
+ "score_pct": 100.0,
128
+ "quality_avg": 8.0,
129
+ "n": 2
130
+ },
131
+ "burn_efficiency": {
132
+ "score_pct": 100.0,
133
+ "quality_avg": 8.0,
134
+ "n": 2
135
+ },
136
+ "cohort_revenue": {
137
+ "score_pct": 100.0,
138
+ "quality_avg": 8.0,
139
+ "n": 2
140
+ },
141
+ "cost_structure": {
142
+ "score_pct": 94.45,
143
+ "quality_avg": 7.36,
144
+ "n": 2
145
+ },
146
+ "forecast_scenario": {
147
+ "score_pct": 100.0,
148
+ "quality_avg": 8.0,
149
+ "n": 2
150
+ },
151
+ "identity_guardrails": {
152
+ "score_pct": 90.0,
153
+ "quality_avg": 6.75,
154
+ "n": 2
155
+ }
156
+ },
157
+ "categories": [
158
+ "budget_tradeoffs",
159
+ "burn_efficiency",
160
+ "cohort_revenue",
161
+ "cost_structure",
162
+ "forecast_scenario",
163
+ "identity_guardrails",
164
+ "pricing_finance",
165
+ "runway_cash",
166
+ "saas_metrics",
167
+ "unit_economics"
168
+ ]
169
+ }
170
+ },
171
+ "compares": {
172
+ "finance": {
173
+ "base_pct": 93.08,
174
+ "ft_pct": 98.44,
175
+ "delta": 5.36,
176
+ "BEATS_BASE": true,
177
+ "categories": [
178
+ {
179
+ "category": "budget_tradeoffs",
180
+ "base_pct": 94.45,
181
+ "ft_pct": 100.0,
182
+ "delta": 5.55,
183
+ "ft_wins": true
184
+ },
185
+ {
186
+ "category": "burn_efficiency",
187
+ "base_pct": 94.45,
188
+ "ft_pct": 100.0,
189
+ "delta": 5.55,
190
+ "ft_wins": true
191
+ },
192
+ {
193
+ "category": "cohort_revenue",
194
+ "base_pct": 94.45,
195
+ "ft_pct": 100.0,
196
+ "delta": 5.55,
197
+ "ft_wins": true
198
+ },
199
+ {
200
+ "category": "cost_structure",
201
+ "base_pct": 100.0,
202
+ "ft_pct": 94.45,
203
+ "delta": -5.55,
204
+ "ft_wins": false
205
+ },
206
+ {
207
+ "category": "forecast_scenario",
208
+ "base_pct": 94.45,
209
+ "ft_pct": 100.0,
210
+ "delta": 5.55,
211
+ "ft_wins": true
212
+ },
213
+ {
214
+ "category": "identity_guardrails",
215
+ "base_pct": 58.57,
216
+ "ft_pct": 90.0,
217
+ "delta": 31.43,
218
+ "ft_wins": true
219
+ },
220
+ {
221
+ "category": "pricing_finance",
222
+ "base_pct": 100.0,
223
+ "ft_pct": 100.0,
224
+ "delta": 0.0,
225
+ "ft_wins": false
226
+ },
227
+ {
228
+ "category": "runway_cash",
229
+ "base_pct": 100.0,
230
+ "ft_pct": 100.0,
231
+ "delta": 0.0,
232
+ "ft_wins": false
233
+ },
234
+ {
235
+ "category": "saas_metrics",
236
+ "base_pct": 100.0,
237
+ "ft_pct": 100.0,
238
+ "delta": 0.0,
239
+ "ft_wins": false
240
+ },
241
+ {
242
+ "category": "unit_economics",
243
+ "base_pct": 94.45,
244
+ "ft_pct": 100.0,
245
+ "delta": 5.55,
246
+ "ft_wins": true
247
+ }
248
+ ],
249
+ "categories_ft_trails": [
250
+ "cost_structure",
251
+ "pricing_finance",
252
+ "runway_cash",
253
+ "saas_metrics"
254
+ ],
255
+ "base_quality_avg": 6.54,
256
+ "ft_quality_avg": 7.86
257
+ }
258
+ },
259
+ "total_seconds": 176.3
260
+ }