| { |
| "benchmark": "LettuceDetect v2", |
| "split": "test", |
| "score_scale": "0-1", |
| "max_input_tokens": 8192, |
| "truncation_policy": "only_first", |
| "span_metric": "micro character overlap", |
| "dtype": "bfloat16", |
| "threshold": 0.5, |
| "overall": { |
| "n": 10698, |
| "span_f1": 0.6468063721645629, |
| "span_p": 0.7001953973777478, |
| "span_r": 0.6009822273490223, |
| "ex_f1": 0.8749271864858118, |
| "ex_p": 0.8913190912173619, |
| "ex_r": 0.8591273083837229, |
| "iou": 0.6791407403861177, |
| "clean_fpr": 0.13998689670233674 |
| }, |
| "by_source": { |
| "lettucedetect-acl": { |
| "n": 440, |
| "span_f1": 0.6033354880413704, |
| "span_p": 0.720407533189256, |
| "span_r": 0.5189946619217082, |
| "ex_f1": 0.8590021691973969, |
| "ex_p": 0.8215767634854771, |
| "ex_r": 0.9, |
| "iou": 0.6506637226712354, |
| "clean_fpr": 0.19545454545454546 |
| }, |
| "lettucedetect-code-agent": { |
| "n": 2015, |
| "span_f1": 0.5304085445334227, |
| "span_p": 0.6548436524601328, |
| "span_r": 0.44571299290373134, |
| "ex_f1": 0.7826510721247564, |
| "ex_p": 0.7736030828516378, |
| "ex_r": 0.7919132149901381, |
| "iou": 0.5799284591741568, |
| "clean_fpr": 0.23476523476523475 |
| }, |
| "lettucedetect-readme": { |
| "n": 641, |
| "span_f1": 0.7668571829306778, |
| "span_p": 0.8180594000149622, |
| "span_r": 0.7216869060190074, |
| "ex_f1": 0.9153605015673981, |
| "ex_p": 0.9182389937106918, |
| "ex_r": 0.9125, |
| "iou": 0.7844094959106822, |
| "clean_fpr": 0.08099688473520249 |
| }, |
| "lettucedetect-tool-output": { |
| "n": 617, |
| "span_f1": 0.6427959358550618, |
| "span_p": 0.7796585003711952, |
| "span_r": 0.5468082890763303, |
| "ex_f1": 0.8273504273504273, |
| "ex_p": 0.8736462093862816, |
| "ex_r": 0.7857142857142857, |
| "iou": 0.7027343010780659, |
| "clean_fpr": 0.11326860841423948 |
| }, |
| "lettucedetect-wikipedia": { |
| "n": 1388, |
| "span_f1": 0.7343291203881016, |
| "span_p": 0.7708389171803806, |
| "span_r": 0.7011214041930766, |
| "ex_f1": 0.9298369950389794, |
| "ex_p": 0.9149232914923291, |
| "ex_r": 0.9452449567723343, |
| "iou": 0.7905999840786986, |
| "clean_fpr": 0.08789625360230548 |
| }, |
| "psiloqa": { |
| "n": 2897, |
| "span_f1": 0.7084256755107189, |
| "span_p": 0.7054881112533835, |
| "span_r": 0.7113878053564034, |
| "ex_f1": 0.9406055900621118, |
| "ex_p": 0.9569510268562401, |
| "ex_r": 0.9248091603053435, |
| "iou": 0.6254885367916755, |
| "clean_fpr": 0.3935018050541516 |
| }, |
| "ragtruth": { |
| "n": 2700, |
| "span_f1": 0.5254962135743007, |
| "span_p": 0.6738326440634735, |
| "span_r": 0.43068574822129324, |
| "ex_f1": 0.7485448195576252, |
| "ex_p": 0.8296774193548387, |
| "ex_r": 0.6818663838812301, |
| "iou": 0.727708569558819, |
| "clean_fpr": 0.07512805919180421 |
| } |
| }, |
| "truncation": { |
| "answer_truncated": 0, |
| "prompt_truncated": 688, |
| "truncated": 688 |
| } |
| } |
|
|
