Publish Vela Halu model and evaluation scores

README.md

@@ -32,16 +32,18 @@ Vela Halu finds answer spans unsupported by the supplied evidence, for grounded
 
 Labels are `supported` (0) and `hallucinated` (1). Spans use Unicode character offsets in the answer. Evidence support is distinct from real-world factual truth; an empty result does not guarantee correctness.
 
+Answers requiring arithmetic or multi-step reasoning beyond explicit context may be incorrectly flagged.
+
 ## Evaluation
 
-Scores (0-100) on the 10,698-example held-out test set. Higher is better.
+Scores (0-100) on the 10,698-example fixed evaluation set. Higher is better.
 
 | Metric | Vela Halu |
 |---|---:|
-| Overall span F1 | 63.85 |
-| Overall example F1 | 87.12 |
-| Code span F1 | 51.07 |
-| Tool output span F1 | 59.86 |
+| Overall span F1 | 64.68 |
+| Overall example F1 | 87.49 |
+| Code span F1 | 53.04 |
+| Tool output span F1 | 64.28 |
 
 Span F1 measures character overlap; example F1 measures whether an answer contains any hallucinated span. Evaluation uses bfloat16, an 8,192-token limit, `only_first` pair truncation, and token scores strictly above 0.5. [Full scores](./scores.json) include all source groups and truncation statistics.
 

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d303e4ef50590c103e30e4ab9e16a4f015d14fdb52d1c1935fba55191df22441
+oid sha256:0f51bf4a6e1462e88733c36e1e59fa96c1188c8c470de7829b51c95298247be6
 size 1230141424

scores.json

@@ -9,92 +9,92 @@
   "threshold": 0.5,
   "overall": {
     "n": 10698,
-    "span_f1": 0.6385179438029605,
-    "span_p": 0.7104274748432198,
-    "span_r": 0.5798277303088283,
-    "ex_f1": 0.8712474983322215,
-    "ex_p": 0.8894942959305295,
-    "ex_r": 0.8537342703056054,
-    "iou": 0.6702612740887652,
-    "clean_fpr": 0.1417340030574361
+    "span_f1": 0.6468063721645629,
+    "span_p": 0.7001953973777478,
+    "span_r": 0.6009822273490223,
+    "ex_f1": 0.8749271864858118,
+    "ex_p": 0.8913190912173619,
+    "ex_r": 0.8591273083837229,
+    "iou": 0.6791407403861177,
+    "clean_fpr": 0.13998689670233674
   },
   "by_source": {
     "lettucedetect-acl": {
       "n": 440,
-      "span_f1": 0.5702109294923666,
-      "span_p": 0.7242702480471427,
-      "span_r": 0.47019572953736655,
-      "ex_f1": 0.8521739130434782,
-      "ex_p": 0.8166666666666667,
-      "ex_r": 0.8909090909090909,
-      "iou": 0.6280894829592452,
-      "clean_fpr": 0.2
+      "span_f1": 0.6033354880413704,
+      "span_p": 0.720407533189256,
+      "span_r": 0.5189946619217082,
+      "ex_f1": 0.8590021691973969,
+      "ex_p": 0.8215767634854771,
+      "ex_r": 0.9,
+      "iou": 0.6506637226712354,
+      "clean_fpr": 0.19545454545454546
     },
     "lettucedetect-code-agent": {
       "n": 2015,
-      "span_f1": 0.5107270439913081,
-      "span_p": 0.654087400055142,
-      "span_r": 0.4189116111730364,
-      "ex_f1": 0.7675568743818002,
-      "ex_p": 0.7698412698412699,
-      "ex_r": 0.7652859960552268,
-      "iou": 0.5639770662416372,
-      "clean_fpr": 0.23176823176823177
+      "span_f1": 0.5304085445334227,
+      "span_p": 0.6548436524601328,
+      "span_r": 0.44571299290373134,
+      "ex_f1": 0.7826510721247564,
+      "ex_p": 0.7736030828516378,
+      "ex_r": 0.7919132149901381,
+      "iou": 0.5799284591741568,
+      "clean_fpr": 0.23476523476523475
     },
     "lettucedetect-readme": {
       "n": 641,
-      "span_f1": 0.7605563835072032,
-      "span_p": 0.8224865694551036,
-      "span_r": 0.7072993664202746,
-      "ex_f1": 0.9209302325581395,
-      "ex_p": 0.9138461538461539,
-      "ex_r": 0.928125,
-      "iou": 0.7809105019493161,
-      "clean_fpr": 0.08722741433021806
+      "span_f1": 0.7668571829306778,
+      "span_p": 0.8180594000149622,
+      "span_r": 0.7216869060190074,
+      "ex_f1": 0.9153605015673981,
+      "ex_p": 0.9182389937106918,
+      "ex_r": 0.9125,
+      "iou": 0.7844094959106822,
+      "clean_fpr": 0.08099688473520249
     },
     "lettucedetect-tool-output": {
       "n": 617,
-      "span_f1": 0.598570881714942,
-      "span_p": 0.7726898369296656,
-      "span_r": 0.4884931792148287,
-      "ex_f1": 0.8082901554404145,
-      "ex_p": 0.8634686346863468,
-      "ex_r": 0.7597402597402597,
-      "iou": 0.6758541479351077,
-      "clean_fpr": 0.11974110032362459
+      "span_f1": 0.6427959358550618,
+      "span_p": 0.7796585003711952,
+      "span_r": 0.5468082890763303,
+      "ex_f1": 0.8273504273504273,
+      "ex_p": 0.8736462093862816,
+      "ex_r": 0.7857142857142857,
+      "iou": 0.7027343010780659,
+      "clean_fpr": 0.11326860841423948
     },
     "lettucedetect-wikipedia": {
       "n": 1388,
-      "span_f1": 0.7253597853484023,
-      "span_p": 0.7776770459221033,
-      "span_r": 0.6796379814724525,
-      "ex_f1": 0.9248067463106114,
-      "ex_p": 0.9026063100137174,
-      "ex_r": 0.9481268011527377,
-      "iou": 0.7810017784449338,
-      "clean_fpr": 0.10230547550432277
+      "span_f1": 0.7343291203881016,
+      "span_p": 0.7708389171803806,
+      "span_r": 0.7011214041930766,
+      "ex_f1": 0.9298369950389794,
+      "ex_p": 0.9149232914923291,
+      "ex_r": 0.9452449567723343,
+      "iou": 0.7905999840786986,
+      "clean_fpr": 0.08789625360230548
     },
     "psiloqa": {
       "n": 2897,
-      "span_f1": 0.7077433515248708,
-      "span_p": 0.7179186831478684,
-      "span_r": 0.6978524265074418,
-      "ex_f1": 0.94276875483372,
-      "ex_p": 0.9553291536050157,
-      "ex_r": 0.9305343511450381,
-      "iou": 0.6217888817095617,
-      "clean_fpr": 0.41155234657039713
+      "span_f1": 0.7084256755107189,
+      "span_p": 0.7054881112533835,
+      "span_r": 0.7113878053564034,
+      "ex_f1": 0.9406055900621118,
+      "ex_p": 0.9569510268562401,
+      "ex_r": 0.9248091603053435,
+      "iou": 0.6254885367916755,
+      "clean_fpr": 0.3935018050541516
     },
     "ragtruth": {
       "n": 2700,
-      "span_f1": 0.5119871112687714,
-      "span_p": 0.7006978861130778,
-      "span_r": 0.40335596259766876,
-      "ex_f1": 0.7392075694855116,
-      "ex_p": 0.8355614973262032,
-      "ex_r": 0.662778366914104,
-      "iou": 0.7239864627418552,
-      "clean_fpr": 0.07000569151963575
+      "span_f1": 0.5254962135743007,
+      "span_p": 0.6738326440634735,
+      "span_r": 0.43068574822129324,
+      "ex_f1": 0.7485448195576252,
+      "ex_p": 0.8296774193548387,
+      "ex_r": 0.6818663838812301,
+      "iou": 0.727708569558819,
+      "clean_fpr": 0.07512805919180421
     }
   },
   "truncation": {