Add final DeBERTa span-level tool hallucination detector

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,108 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: transformers
+base_model: microsoft/deberta-v3-small
+tags:
+- hallucination-detection
+- span-detection
+- tool-use
+- deberta-v3
+- ragtruth
+datasets:
+- Ali-Bhai/toolace-ragtruth-style-hallucinations
+metrics:
+- f1
+- accuracy
+---
+
+# DeBERTa Tool Hallucination Span Detector
+
+This is the final span-level model for the tool hallucination coursework. It predicts RAGTruth-style hallucinated character spans in assistant answers grounded on tool outputs.
+
+## Task
+
+Input record fields:
+
+- `query`: user question
+- `context`: tool response or tool evidence
+- `output`: assistant answer
+- `available_tool_names`: optional list of available tools
+
+Output:
+
+- row-level hallucination decision
+- hallucination type
+- predicted character spans in `output`
+
+## Method
+
+The model is a `microsoft/deberta-v3-small` token classifier. Because some answers are long, inference uses overlapping answer windows. Each window is placed first in the sequence, followed by question, tool responses, and available tool names. Only answer-window tokens are supervised during training.
+
+## Decoding configuration
+
+The validation-selected decoding configuration is:
+
+```json
+{
+  "threshold": 0.5,
+  "min_span_chars": 1,
+  "min_span_tokens": 1,
+  "merge_gap_chars": 1,
+  "strip_predicted_span_whitespace": true,
+  "drop_spans_without_alnum": true,
+  "score_name": "sum_non_O_probability"
+}
+```
+
+## Held-out test results
+
+```json
+{
+  "row_accuracy": 0.9516908212560387,
+  "row_macro_f1": 0.9553091397849462,
+  "binary_f1": 0.9561403508771931,
+  "exact_span_f1": 0.8207171314741037,
+  "overlap_span_f1_iou_0_01": 0.8685258964143425,
+  "overlap_span_f1_iou_0_50": 0.8366533864541832,
+  "char_span_precision": 0.971356003950896,
+  "char_span_recall": 0.9955169920462762,
+  "char_span_f1": 0.9832881016997572
+}
+```
+
+## Loading
+
+```python
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+
+repo_id = "Ali-Bhai/deberta-tool-hallucination-span-detector"
+tokenizer = AutoTokenizer.from_pretrained(repo_id, use_fast=True)
+model = AutoModelForTokenClassification.from_pretrained(repo_id)
+```
+
+## Inference
+
+```python
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+from span_inference import predict_record
+
+repo_id = "Ali-Bhai/deberta-tool-hallucination-span-detector"
+tokenizer = AutoTokenizer.from_pretrained(repo_id, use_fast=True)
+model = AutoModelForTokenClassification.from_pretrained(repo_id)
+
+record = {
+    "query": "What did the tool return?",
+    "context": "... tool output ...",
+    "output": "... assistant answer ...",
+    "available_tool_names": [],
+}
+
+prediction = predict_record(record, model, tokenizer, device="cuda")
+print(prediction["predicted_spans"])
+```
+
+## Related repo
+
+The earlier row-level classifier is here: https://huggingface.co/Ali-Bhai/deberta-tool-hallucination-detector.

config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "DebertaV2ForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "tool_output_conflict",
+    "2": "overgeneration",
+    "3": "missing_tool_action_recommendation"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "O": 0,
+    "missing_tool_action_recommendation": 3,
+    "overgeneration": 2,
+    "tool_output_conflict": 1
+  },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval/span_deberta_test_token_predictions.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:695c67472ce44d19d9513e28a32c0524d64e7a0f000fe675e8a45c6b139a2262
+size 13126758

eval/span_deberta_validation_token_predictions.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529e7ad1712b611066deda3415d4f24cdbdb7a19e24f1d694184f8943e54111f
+size 13225709

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c936df2b89d254c05444e66afa788bc3f553f7ec36e4e7b6cc230bd11e74ce24
+size 565242144

reports/final_notebook_results_text.md

@@ -0,0 +1,41 @@
+# Final result summary
+
+The final system has two parts.
+
+First, I trained a DeBERTa-v3-small row-level classifier for the four labels: clean, tool_output_conflict, overgeneration, and missing_tool_action_recommendation. This model is useful for quick answer-level detection. On the held-out test split it reached accuracy 0.9275, macro F1 0.9249, and binary clean-vs-hallucinated macro F1 0.9418.
+
+Second, to satisfy the RAGTruth-style span requirement, I trained a DeBERTa-v3-small token classifier over sliding answer windows. The model predicts answer-token labels and then decodes them back into character spans. The threshold and span cleanup settings were selected on validation only. On the held-out test split the span model reached row accuracy 0.9517, row macro F1 0.9553, binary F1 0.9561, exact span F1 0.8207, overlap span F1 0.8685, and character-level span F1 0.9833.
+
+I also evaluated the required baselines. LettuceDetect reached binary F1 0.7718, exact span F1 0.1652, overlap span F1 0.4661, and character-level span F1 0.5653. The resource-aware LookBackLens-style attention baseline reached binary F1 0.7000, exact span F1 0.0095, overlap span F1 0.1956, and character-level span F1 0.2278.
+
+The strongest final span model was trained for 8 epochs. Its selected decoding configuration was:
+
+```json
+{
+  "threshold": 0.5,
+  "min_span_chars": 1,
+  "min_span_tokens": 1,
+  "merge_gap_chars": 1,
+  "strip_predicted_span_whitespace": true,
+  "drop_spans_without_alnum": true,
+  "score_name": "sum_non_O_probability"
+}
+```
+
+## Test-set comparison
+
+| System | Scope | Row acc. | Row macro F1 | Binary F1 | Exact span F1 | Overlap F1 | Char span F1 |
+|---|---|---|---|---|---|---|---|
+| DeBERTa-v3-small row classifier | row classification | 0.9275 | 0.9249 | 0.9418 | n/a | n/a | n/a |
+| LettuceDetect | span detection | 0.7343 | n/a | 0.7718 | 0.1652 | 0.4661 | 0.5653 |
+| LookBackLens-style attention baseline | span detection | 0.6232 | n/a | 0.7000 | 0.0095 | 0.1956 | 0.2278 |
+| DeBERTa-v3-small span-token classifier | row + span detection | 0.9517 | 0.9553 | 0.9561 | 0.8207 | 0.8685 | 0.9833 |
+
+## Repositories
+
+Dataset: https://huggingface.co/datasets/Ali-Bhai/toolace-ragtruth-style-hallucinations
+Model: https://huggingface.co/Ali-Bhai/deberta-tool-hallucination-detector
+
+## Short discussion
+
+The row classifier performs well for answer-level decisions but cannot return exact hallucinated text. The final span-token classifier fixes this by using RAGTruth-style character spans during training. A single 512-token sequence lost a small number of late answer spans, so I switched to overlapping answer windows and audited the alignment before training. This preserved every gold span and allowed the model to output exact predicted spans. The final model strongly outperformed LettuceDetect and the LookBackLens-style attention baseline on this constructed held-out test split, especially on character-level span F1.

reports/final_results_comparison_summary.json

@@ -0,0 +1,589 @@
+{
+  "created": "2026-05-23T11:17:16",
+  "runtime_report": {
+    "python": "3.12.13 (main, Mar  4 2026, 09:23:07) [GCC 11.4.0]",
+    "platform": "Linux-6.6.122+-x86_64-with-glibc2.35",
+    "packages": {
+      "numpy": "2.0.2",
+      "torch": "2.10.0+cu128",
+      "transformers": "5.0.0",
+      "huggingface_hub": "1.11.0"
+    }
+  },
+  "paths": {
+    "sequence_selected_config": "/content/drive/MyDrive/TLLM/04_submission/final_deberta_tool_hallucination_submission/selected_model_config.json",
+    "sequence_final_report": "/content/drive/MyDrive/TLLM/04_submission/final_deberta_tool_hallucination_submission/final_submission_report.json",
+    "lettucedetect_report": "/content/drive/MyDrive/TLLM/00_reports/official_baselines_and_spans/cell21_lettucedetect_full_report.json",
+    "lookbacklens_report": "/content/drive/MyDrive/TLLM/00_reports/official_baselines_and_spans/cell23_lookbacklens_full_report.json",
+    "span_report": "/content/drive/MyDrive/TLLM/00_reports/official_baselines_and_spans/cell26_span_deberta_training_report.json",
+    "span_config": "/content/drive/MyDrive/TLLM/03_models/span_deberta_v3_small_sliding_windows/selected_span_model_config.json",
+    "hf_publication_summary": "/content/drive/MyDrive/TLLM/04_submission/huggingface_publication_summary.json",
+    "hf_verification_summary": "/content/drive/MyDrive/TLLM/04_submission/huggingface_verification_and_card_polish_summary.json"
+  },
+  "path_status": {
+    "sequence_selected_config": true,
+    "sequence_final_report": true,
+    "lettucedetect_report": true,
+    "lookbacklens_report": true,
+    "span_report": true,
+    "span_config": true,
+    "hf_publication_summary": true,
+    "hf_verification_summary": true
+  },
+  "source_report_summary": {
+    "sequence_selected_model": {
+      "selected_run_name": "phase4e_deberta_v3_small_answer_evidence_pair_unweighted",
+      "base_model_name": "microsoft/deberta-v3-small",
+      "text_mode": "answer_evidence_pair",
+      "calibration": {
+        "mode": "conflict_gate",
+        "name": "conflict_gate__minp_0.00__margin_0.20__fallback_best_non_conflict",
+        "min_conflict_prob": 0.0,
+        "min_conflict_margin": 0.2,
+        "fallback_strategy": "best_non_conflict"
+      }
+    },
+    "lettucedetect": {
+      "model": "KRLabsOrg/lettucedect-base-modernbert-en-v1",
+      "selected_config": {
+        "threshold": 0.95,
+        "min_span_chars": 5,
+        "min_span_tokens": 2,
+        "merge_gap_chars": 1,
+        "hallucination_label_ids": [
+          1
+        ]
+      }
+    },
+    "lookbacklens_style": {
+      "model": "distilgpt2",
+      "selected_config": {
+        "threshold": 0.8,
+        "min_span_chars": 12,
+        "min_span_tokens": 2,
+        "merge_gap_chars": 1,
+        "drop_spans_without_alnum": true,
+        "score_name": "hallucination_score_mean"
+      }
+    },
+    "span_deberta": {
+      "model_name": "span_deberta_v3_small_sliding_windows",
+      "base_model_name": "microsoft/deberta-v3-small",
+      "best_epoch": 8,
+      "selected_config": {
+        "threshold": 0.5,
+        "min_span_chars": 1,
+        "min_span_tokens": 1,
+        "merge_gap_chars": 1,
+        "strip_predicted_span_whitespace": true,
+        "drop_spans_without_alnum": true,
+        "score_name": "sum_non_O_probability"
+      }
+    },
+    "huggingface": {
+      "publication_summary_available": true,
+      "verification_summary_available": true,
+      "dataset_url": "https://huggingface.co/datasets/Ali-Bhai/toolace-ragtruth-style-hallucinations",
+      "model_url": "https://huggingface.co/Ali-Bhai/deberta-tool-hallucination-detector"
+    }
+  },
+  "comparison_rows": [
+    {
+      "system": "DeBERTa-v3-small row classifier",
+      "type": "Our row-level classifier",
+      "scope": "row classification",
+      "row_accuracy": 0.927536231884058,
+      "row_macro_f1": 0.9249477834493081,
+      "row_weighted_f1": 0.9287091179440344,
+      "binary_f1": 0.9417994376757264,
+      "clean_f1": 0.9381443298969072,
+      "conflict_f1": 0.8461538461538461,
+      "overgeneration_f1": 0.9577464788732394,
+      "missing_tool_f1": 0.9577464788732394,
+      "exact_span_f1": null,
+      "overlap_span_f1_iou_0_01": null,
+      "overlap_span_f1_iou_0_50": null,
+      "char_span_f1": null,
+      "notes": "Best row-level model used answer/evidence pair input and validation-selected conflict calibration. It does not predict exact character spans.",
+      "raw_metrics": {
+        "accuracy": 0.927536231884058,
+        "macro_f1": 0.9249477834493081,
+        "weighted_f1": 0.9287091179440344,
+        "binary_macro_f1": 0.9417994376757264,
+        "clean_f1": 0.9381443298969072,
+        "clean_recall": 0.9479166666666666,
+        "conflict_f1": 0.8461538461538461,
+        "conflict_precision": 0.8048780487804879,
+        "conflict_recall": 0.8918918918918919,
+        "overgeneration_f1": 0.9577464788732394,
+        "missing_tool_f1": 0.9577464788732394
+      }
+    },
+    {
+      "system": "LettuceDetect",
+      "type": "Official baseline",
+      "scope": "span detection",
+      "row_accuracy": 0.7342995169082126,
+      "row_macro_f1": null,
+      "row_weighted_f1": null,
+      "binary_f1": 0.7717842323651452,
+      "clean_f1": null,
+      "conflict_f1": null,
+      "overgeneration_f1": null,
+      "missing_tool_f1": null,
+      "exact_span_f1": 0.16519174041297932,
+      "overlap_span_f1_iou_0_01": 0.46607669616519176,
+      "overlap_span_f1_iou_0_50": 0.2949852507374631,
+      "char_span_f1": 0.5653136531365313,
+      "span_precision_char": 0.4653271028037383,
+      "span_recall_char": 0.7200289226319595,
+      "num_gold_spans": 111,
+      "num_predicted_spans": 228,
+      "selected_config": {
+        "threshold": 0.95,
+        "min_span_chars": 5,
+        "min_span_tokens": 2,
+        "merge_gap_chars": 1,
+        "hallucination_label_ids": [
+          1
+        ]
+      },
+      "notes": "Official token-classification baseline run directly from Hugging Face. Validation-only threshold tuning was used for span decoding.",
+      "raw_metrics": {
+        "config": {
+          "threshold": 0.95,
+          "min_span_chars": 5,
+          "min_span_tokens": 2,
+          "merge_gap_chars": 1,
+          "hallucination_label_ids": [
+            1
+          ]
+        },
+        "num_rows": 207,
+        "binary_example_metrics": {
+          "accuracy": 0.7342995169082126,
+          "precision": 0.7153846153846154,
+          "recall": 0.8378378378378378,
+          "f1": 0.7717842323651452,
+          "tp": 93,
+          "fp": 37,
+          "fn": 18,
+          "tn": 59
+        },
+        "exact_span_metrics": {
+          "precision": 0.12280701754385964,
+          "recall": 0.25225225225225223,
+          "f1": 0.16519174041297932,
+          "matched": 28,
+          "gold_total": 111,
+          "pred_total": 228
+        },
+        "overlap_span_metrics_iou_0_01": {
+          "precision": 0.34649122807017546,
+          "recall": 0.7117117117117117,
+          "f1": 0.46607669616519176,
+          "matched": 79,
+          "gold_total": 111,
+          "pred_total": 228,
+          "iou_threshold": 0.01
+        },
+        "overlap_span_metrics_iou_0_50": {
+          "precision": 0.21929824561403508,
+          "recall": 0.45045045045045046,
+          "f1": 0.2949852507374631,
+          "matched": 50,
+          "gold_total": 111,
+          "pred_total": 228,
+          "iou_threshold": 0.5
+        },
+        "char_micro_metrics": {
+          "precision": 0.4653271028037383,
+          "recall": 0.7200289226319595,
+          "f1": 0.5653136531365313,
+          "overlap_chars": 4979,
+          "gold_chars": 6915,
+          "pred_chars": 10700
+        },
+        "per_type_char_micro_metrics": {
+          "clean": {
+            "precision": 0.0,
+            "recall": 0.0,
+            "f1": 0.0,
+            "overlap_chars": 0,
+            "gold_chars": 0,
+            "pred_chars": 2485
+          },
+          "missing_tool_action_recommendation": {
+            "precision": 0.7734467748318163,
+            "recall": 0.8652058432934927,
+            "f1": 0.8167572085248641,
+            "overlap_chars": 3909,
+            "gold_chars": 4518,
+            "pred_chars": 5054
+          },
+          "tool_output_conflict": {
+            "precision": 0.07041139240506329,
+            "recall": 0.40271493212669685,
+            "f1": 0.11986531986531987,
+            "overlap_chars": 89,
+            "gold_chars": 221,
+            "pred_chars": 1264
+          },
+          "overgeneration": {
+            "precision": 0.5171323141802847,
+            "recall": 0.4508272058823529,
+            "f1": 0.48170881414191014,
+            "overlap_chars": 981,
+            "gold_chars": 2176,
+            "pred_chars": 1897
+          }
+        },
+        "num_predicted_spans": 228,
+        "num_gold_spans": 111
+      }
+    },
+    {
+      "system": "LookBackLens-style attention baseline",
+      "type": "Official-baseline-inspired baseline",
+      "scope": "span detection",
+      "row_accuracy": 0.6231884057971014,
+      "row_macro_f1": null,
+      "row_weighted_f1": null,
+      "binary_f1": 0.7,
+      "clean_f1": null,
+      "conflict_f1": null,
+      "overgeneration_f1": null,
+      "missing_tool_f1": null,
+      "exact_span_f1": 0.009463722397476343,
+      "overlap_span_f1_iou_0_01": 0.19558359621451105,
+      "overlap_span_f1_iou_0_50": 0.11356466876971608,
+      "char_span_f1": 0.22782546494992845,
+      "span_precision_char": 0.13473995928188043,
+      "span_recall_char": 0.7369486623282718,
+      "num_gold_spans": 111,
+      "num_predicted_spans": 523,
+      "selected_config": {
+        "threshold": 0.8,
+        "min_span_chars": 12,
+        "min_span_tokens": 2,
+        "merge_gap_chars": 1,
+        "drop_spans_without_alnum": true,
+        "score_name": "hallucination_score_mean"
+      },
+      "notes": "Resource-aware implementation of the LookBackLens attention idea using distilgpt2 attention maps.",
+      "raw_metrics": {
+        "config": {
+          "threshold": 0.8,
+          "min_span_chars": 12,
+          "min_span_tokens": 2,
+          "merge_gap_chars": 1,
+          "drop_spans_without_alnum": true,
+          "score_name": "hallucination_score_mean"
+        },
+        "num_rows": 207,
+        "binary_example_metrics": {
+          "accuracy": 0.6231884057971014,
+          "precision": 0.610738255033557,
+          "recall": 0.8198198198198198,
+          "f1": 0.7,
+          "tp": 91,
+          "fp": 58,
+          "fn": 20,
+          "tn": 38
+        },
+        "exact_span_metrics": {
+          "precision": 0.0057361376673040155,
+          "recall": 0.02702702702702703,
+          "f1": 0.009463722397476343,
+          "matched": 3,
+          "gold_total": 111,
+          "pred_total": 523
+        },
+        "overlap_span_metrics_iou_0_01": {
+          "precision": 0.11854684512428298,
+          "recall": 0.5585585585585585,
+          "f1": 0.19558359621451105,
+          "matched": 62,
+          "gold_total": 111,
+          "pred_total": 523,
+          "iou_threshold": 0.01
+        },
+        "overlap_span_metrics_iou_0_50": {
+          "precision": 0.06883365200764818,
+          "recall": 0.32432432432432434,
+          "f1": 0.11356466876971608,
+          "matched": 36,
+          "gold_total": 111,
+          "pred_total": 523,
+          "iou_threshold": 0.5
+        },
+        "char_micro_metrics": {
+          "precision": 0.13473995928188043,
+          "recall": 0.7369486623282718,
+          "f1": 0.22782546494992845,
+          "overlap_chars": 5096,
+          "gold_chars": 6915,
+          "pred_chars": 37821
+        },
+        "per_type_char_micro_metrics": {
+          "clean": {
+            "precision": 0.0,
+            "recall": 0.0,
+            "f1": 0.0,
+            "overlap_chars": 0,
+            "gold_chars": 0,
+            "pred_chars": 13407
+          },
+          "missing_tool_action_recommendation": {
+            "precision": 0.38209050350541746,
+            "recall": 0.796148738379814,
+            "f1": 0.516365202411714,
+            "overlap_chars": 3597,
+            "gold_chars": 4518,
+            "pred_chars": 9414
+          },
+          "tool_output_conflict": {
+            "precision": 0.0,
+            "recall": 0.0,
+            "f1": 0.0,
+            "overlap_chars": 0,
+            "gold_chars": 221,
+            "pred_chars": 7651
+          },
+          "overgeneration": {
+            "precision": 0.20397332970472173,
+            "recall": 0.6888786764705882,
+            "f1": 0.31475065616797904,
+            "overlap_chars": 1499,
+            "gold_chars": 2176,
+            "pred_chars": 7349
+          }
+        },
+        "num_predicted_spans": 523,
+        "num_gold_spans": 111
+      }
+    },
+    {
+      "system": "DeBERTa-v3-small span-token classifier",
+      "type": "Our span-level model",
+      "scope": "row + span detection",
+      "row_accuracy": 0.9516908212560387,
+      "row_macro_f1": 0.9553091397849462,
+      "row_weighted_f1": 0.9527232351566153,
+      "binary_f1": 0.9561403508771931,
+      "clean_f1": 0.946236559139785,
+      "conflict_f1": 0.875,
+      "overgeneration_f1": 1.0,
+      "missing_tool_f1": 1.0,
+      "exact_span_f1": 0.8207171314741037,
+      "overlap_span_f1_iou_0_01": 0.8685258964143425,
+      "overlap_span_f1_iou_0_50": 0.8366533864541832,
+      "char_span_f1": 0.9832881016997572,
+      "span_precision_char": 0.971356003950896,
+      "span_recall_char": 0.9955169920462762,
+      "num_gold_spans": 111,
+      "num_predicted_spans": 140,
+      "token_macro_f1": 0.8905429210772722,
+      "token_weighted_f1": 0.9957200208094352,
+      "selected_config": {
+        "threshold": 0.5,
+        "min_span_chars": 1,
+        "min_span_tokens": 1,
+        "merge_gap_chars": 1,
+        "strip_predicted_span_whitespace": true,
+        "drop_spans_without_alnum": true,
+        "score_name": "sum_non_O_probability"
+      },
+      "best_epoch": 8,
+      "notes": "Final span-level model. It uses sliding answer windows, supervises only answer-window tokens, and decodes RAGTruth-style character spans.",
+      "raw_metrics": {
+        "config": {
+          "threshold": 0.5,
+          "min_span_chars": 1,
+          "min_span_tokens": 1,
+          "merge_gap_chars": 1,
+          "strip_predicted_span_whitespace": true,
+          "drop_spans_without_alnum": true,
+          "score_name": "sum_non_O_probability"
+        },
+        "num_rows": 207,
+        "row_multiclass_metrics": {
+          "clean": {
+            "precision": 0.9777777777777777,
+            "recall": 0.9166666666666666,
+            "f1": 0.946236559139785,
+            "support": 96,
+            "tp": 88,
+            "fp": 2,
+            "fn": 8
+          },
+          "tool_output_conflict": {
+            "precision": 0.813953488372093,
+            "recall": 0.9459459459459459,
+            "f1": 0.875,
+            "support": 37,
+            "tp": 35,
+            "fp": 8,
+            "fn": 2
+          },
+          "overgeneration": {
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1": 1.0,
+            "support": 37,
+            "tp": 37,
+            "fp": 0,
+            "fn": 0
+          },
+          "missing_tool_action_recommendation": {
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1": 1.0,
+            "support": 37,
+            "tp": 37,
+            "fp": 0,
+            "fn": 0
+          },
+          "accuracy": 0.9516908212560387,
+          "macro_f1": 0.9553091397849462,
+          "weighted_f1": 0.9527232351566153,
+          "support": 207
+        },
+        "binary_example_metrics": {
+          "accuracy": 0.9516908212560387,
+          "precision": 0.9316239316239316,
+          "recall": 0.9819819819819819,
+          "f1": 0.9561403508771931,
+          "tp": 109,
+          "fp": 8,
+          "fn": 2,
+          "tn": 88
+        },
+        "exact_span_metrics": {
+          "precision": 0.7357142857142858,
+          "recall": 0.9279279279279279,
+          "f1": 0.8207171314741037,
+          "matched": 103,
+          "gold_total": 111,
+          "pred_total": 140
+        },
+        "overlap_span_metrics_iou_0_01": {
+          "precision": 0.7785714285714286,
+          "recall": 0.9819819819819819,
+          "f1": 0.8685258964143425,
+          "matched": 109,
+          "gold_total": 111,
+          "pred_total": 140,
+          "iou_threshold": 0.01
+        },
+        "overlap_span_metrics_iou_0_50": {
+          "precision": 0.75,
+          "recall": 0.9459459459459459,
+          "f1": 0.8366533864541832,
+          "matched": 105,
+          "gold_total": 111,
+          "pred_total": 140,
+          "iou_threshold": 0.5
+        },
+        "char_micro_metrics": {
+          "precision": 0.971356003950896,
+          "recall": 0.9955169920462762,
+          "f1": 0.9832881016997572,
+          "overlap_chars": 6884,
+          "gold_chars": 6915,
+          "pred_chars": 7087
+        },
+        "per_type_char_micro_metrics": {
+          "clean": {
+            "precision": 0.0,
+            "recall": 0.0,
+            "f1": 0.0,
+            "overlap_chars": 0,
+            "gold_chars": 0,
+            "pred_chars": 72
+          },
+          "missing_tool_action_recommendation": {
+            "precision": 0.9879728843210146,
+            "recall": 1.0,
+            "f1": 0.9939500604993949,
+            "overlap_chars": 4518,
+            "gold_chars": 4518,
+            "pred_chars": 4573
+          },
+          "tool_output_conflict": {
+            "precision": 0.7251908396946565,
+            "recall": 0.8597285067873304,
+            "f1": 0.7867494824016563,
+            "overlap_chars": 190,
+            "gold_chars": 221,
+            "pred_chars": 262
+          },
+          "overgeneration": {
+            "precision": 0.998165137614679,
+            "recall": 1.0,
+            "f1": 0.9990817263544537,
+            "overlap_chars": 2176,
+            "gold_chars": 2176,
+            "pred_chars": 2180
+          }
+        },
+        "num_gold_spans": 111,
+        "num_predicted_spans": 140
+      },
+      "raw_token_metrics": {
+        "O": {
+          "precision": 0.9994592359321818,
+          "recall": 0.9951225692025084,
+          "f1": 0.9972861881258828,
+          "support": 31574,
+          "tp": 31420,
+          "fp": 17,
+          "fn": 154
+        },
+        "tool_output_conflict": {
+          "precision": 0.4188679245283019,
+          "recall": 0.8671875,
+          "f1": 0.564885496183206,
+          "support": 128,
+          "tp": 111,
+          "fp": 154,
+          "fn": 17
+        },
+        "overgeneration": {
+          "precision": 1.0,
+          "recall": 1.0,
+          "f1": 1.0,
+          "support": 406,
+          "tp": 406,
+          "fp": 0,
+          "fn": 0
+        },
+        "missing_tool_action_recommendation": {
+          "precision": 1.0,
+          "recall": 1.0,
+          "f1": 1.0,
+          "support": 925,
+          "tp": 925,
+          "fp": 0,
+          "fn": 0
+        },
+        "accuracy": 0.9948233584597221,
+        "macro_f1": 0.8905429210772722,
+        "weighted_f1": 0.9957200208094352,
+        "support": 33033
+      }
+    }
+  ],
+  "comparison_markdown_table": "| System | Scope | Row acc. | Row macro F1 | Binary F1 | Exact span F1 | Overlap F1 | Char span F1 |\n|---|---|---|---|---|---|---|---|\n| DeBERTa-v3-small row classifier | row classification | 0.9275 | 0.9249 | 0.9418 | n/a | n/a | n/a |\n| LettuceDetect | span detection | 0.7343 | n/a | 0.7718 | 0.1652 | 0.4661 | 0.5653 |\n| LookBackLens-style attention baseline | span detection | 0.6232 | n/a | 0.7000 | 0.0095 | 0.1956 | 0.2278 |\n| DeBERTa-v3-small span-token classifier | row + span detection | 0.9517 | 0.9553 | 0.9561 | 0.8207 | 0.8685 | 0.9833 |",
+  "notebook_text": "# Final result summary\n\nThe final system has two parts.\n\nFirst, I trained a DeBERTa-v3-small row-level classifier for the four labels: clean, tool_output_conflict, overgeneration, and missing_tool_action_recommendation. This model is useful for quick answer-level detection. On the held-out test split it reached accuracy 0.9275, macro F1 0.9249, and binary clean-vs-hallucinated macro F1 0.9418.\n\nSecond, to satisfy the RAGTruth-style span requirement, I trained a DeBERTa-v3-small token classifier over sliding answer windows. The model predicts answer-token labels and then decodes them back into character spans. The threshold and span cleanup settings were selected on validation only. On the held-out test split the span model reached row accuracy 0.9517, row macro F1 0.9553, binary F1 0.9561, exact span F1 0.8207, overlap span F1 0.8685, and character-level span F1 0.9833.\n\nI also evaluated the required baselines. LettuceDetect reached binary F1 0.7718, exact span F1 0.1652, overlap span F1 0.4661, and character-level span F1 0.5653. The resource-aware LookBackLens-style attention baseline reached binary F1 0.7000, exact span F1 0.0095, overlap span F1 0.1956, and character-level span F1 0.2278.\n\nThe strongest final span model was trained for 8 epochs. Its selected decoding configuration was:\n\n```json\n{\n  \"threshold\": 0.5,\n  \"min_span_chars\": 1,\n  \"min_span_tokens\": 1,\n  \"merge_gap_chars\": 1,\n  \"strip_predicted_span_whitespace\": true,\n  \"drop_spans_without_alnum\": true,\n  \"score_name\": \"sum_non_O_probability\"\n}\n```\n\n## Test-set comparison\n\n| System | Scope | Row acc. | Row macro F1 | Binary F1 | Exact span F1 | Overlap F1 | Char span F1 |\n|---|---|---|---|---|---|---|---|\n| DeBERTa-v3-small row classifier | row classification | 0.9275 | 0.9249 | 0.9418 | n/a | n/a | n/a |\n| LettuceDetect | span detection | 0.7343 | n/a | 0.7718 | 0.1652 | 0.4661 | 0.5653 |\n| LookBackLens-style attention baseline | span detection | 0.6232 | n/a | 0.7000 | 0.0095 | 0.1956 | 0.2278 |\n| DeBERTa-v3-small span-token classifier | row + span detection | 0.9517 | 0.9553 | 0.9561 | 0.8207 | 0.8685 | 0.9833 |\n\n## Repositories\n\nDataset: https://huggingface.co/datasets/Ali-Bhai/toolace-ragtruth-style-hallucinations\nModel: https://huggingface.co/Ali-Bhai/deberta-tool-hallucination-detector\n\n## Short discussion\n\nThe row classifier performs well for answer-level decisions but cannot return exact hallucinated text. The final span-token classifier fixes this by using RAGTruth-style character spans during training. A single 512-token sequence lost a small number of late answer spans, so I switched to overlapping answer windows and audited the alignment before training. This preserved every gold span and allowed the model to output exact predicted spans. The final model strongly outperformed LettuceDetect and the LookBackLens-style attention baseline on this constructed held-out test split, especially on character-level span F1.\n",
+  "recommended_final_system": {
+    "row_level_model": "DeBERTa-v3-small row classifier",
+    "span_level_model": "DeBERTa-v3-small span-token classifier with sliding answer windows",
+    "primary_metric_to_report_for_span_task": "character-level span F1",
+    "primary_span_test_score": 0.9832881016997572,
+    "test_exact_span_f1": 0.8207171314741037,
+    "test_overlap_span_f1_iou_0_01": 0.8685258964143425,
+    "test_binary_f1": 0.9561403508771931,
+    "test_row_macro_f1": 0.9553091397849462
+  }
+}

reports/final_results_comparison_summary.md

@@ -0,0 +1,69 @@
+# Final results comparison summary
+
+## Comparison table
+
+| System | Scope | Row acc. | Row macro F1 | Binary F1 | Exact span F1 | Overlap F1 | Char span F1 |
+|---|---|---|---|---|---|---|---|
+| DeBERTa-v3-small row classifier | row classification | 0.9275 | 0.9249 | 0.9418 | n/a | n/a | n/a |
+| LettuceDetect | span detection | 0.7343 | n/a | 0.7718 | 0.1652 | 0.4661 | 0.5653 |
+| LookBackLens-style attention baseline | span detection | 0.6232 | n/a | 0.7000 | 0.0095 | 0.1956 | 0.2278 |
+| DeBERTa-v3-small span-token classifier | row + span detection | 0.9517 | 0.9553 | 0.9561 | 0.8207 | 0.8685 | 0.9833 |
+
+## Compact JSON
+
+```json
+[
+  {
+    "system": "DeBERTa-v3-small row classifier",
+    "type": "Our row-level classifier",
+    "scope": "row classification",
+    "row_accuracy": 0.927536231884058,
+    "row_macro_f1": 0.9249477834493081,
+    "binary_f1": 0.9417994376757264,
+    "exact_span_f1": null,
+    "overlap_span_f1_iou_0_01": null,
+    "overlap_span_f1_iou_0_50": null,
+    "char_span_f1": null,
+    "notes": "Best row-level model used answer/evidence pair input and validation-selected conflict calibration. It does not predict exact character spans."
+  },
+  {
+    "system": "LettuceDetect",
+    "type": "Official baseline",
+    "scope": "span detection",
+    "row_accuracy": 0.7342995169082126,
+    "row_macro_f1": null,
+    "binary_f1": 0.7717842323651452,
+    "exact_span_f1": 0.16519174041297932,
+    "overlap_span_f1_iou_0_01": 0.46607669616519176,
+    "overlap_span_f1_iou_0_50": 0.2949852507374631,
+    "char_span_f1": 0.5653136531365313,
+    "notes": "Official token-classification baseline run directly from Hugging Face. Validation-only threshold tuning was used for span decoding."
+  },
+  {
+    "system": "LookBackLens-style attention baseline",
+    "type": "Official-baseline-inspired baseline",
+    "scope": "span detection",
+    "row_accuracy": 0.6231884057971014,
+    "row_macro_f1": null,
+    "binary_f1": 0.7,
+    "exact_span_f1": 0.009463722397476343,
+    "overlap_span_f1_iou_0_01": 0.19558359621451105,
+    "overlap_span_f1_iou_0_50": 0.11356466876971608,
+    "char_span_f1": 0.22782546494992845,
+    "notes": "Resource-aware implementation of the LookBackLens attention idea using distilgpt2 attention maps."
+  },
+  {
+    "system": "DeBERTa-v3-small span-token classifier",
+    "type": "Our span-level model",
+    "scope": "row + span detection",
+    "row_accuracy": 0.9516908212560387,
+    "row_macro_f1": 0.9553091397849462,
+    "binary_f1": 0.9561403508771931,
+    "exact_span_f1": 0.8207171314741037,
+    "overlap_span_f1_iou_0_01": 0.8685258964143425,
+    "overlap_span_f1_iou_0_50": 0.8366533864541832,
+    "char_span_f1": 0.9832881016997572,
+    "notes": "Final span-level model. It uses sliding answer windows, supervises only answer-window tokens, and decodes RAGTruth-style character spans."
+  }
+]
+```

reports/span_window_training_manifest.json

@@ -0,0 +1,69 @@
+{
+  "created": "2026-05-23T10:38:22",
+  "ready_for_training": true,
+  "reason": "Corrected token-level readiness audit. Whitespace-only character gaps are ignored because SentencePiece offset mappings may not expose them as standalone covered characters.",
+  "tokenizer_name": "microsoft/deberta-v3-small",
+  "max_length": 512,
+  "span_labels": [
+    "O",
+    "tool_output_conflict",
+    "overgeneration",
+    "missing_tool_action_recommendation"
+  ],
+  "label2id": {
+    "O": 0,
+    "tool_output_conflict": 1,
+    "overgeneration": 2,
+    "missing_tool_action_recommendation": 3
+  },
+  "id2label": {
+    "0": "O",
+    "1": "tool_output_conflict",
+    "2": "overgeneration",
+    "3": "missing_tool_action_recommendation"
+  },
+  "window_files": {
+    "train": "/content/drive/MyDrive/TLLM/02_processed/span_token_windows_deberta_v3_small/train_span_windows.jsonl",
+    "validation": "/content/drive/MyDrive/TLLM/02_processed/span_token_windows_deberta_v3_small/validation_span_windows.jsonl",
+    "test": "/content/drive/MyDrive/TLLM/02_processed/span_token_windows_deberta_v3_small/test_span_windows.jsonl"
+  },
+  "critical_problem_counts": {
+    "clean_row_has_positive_tokens": 0,
+    "non_clean_row_has_zero_positive_tokens": 0,
+    "gold_span_has_no_positive_token": 0,
+    "intended_answer_tokens_missing_after_tokenization": 0
+  },
+  "split_readiness": {
+    "train": {
+      "clean_rows_with_positive_tokens": 0,
+      "non_clean_rows_with_zero_positive_tokens": 0,
+      "non_clean_rows_with_uncovered_gold_span": 0,
+      "rows_with_missing_intended_answer_tokens": 0,
+      "windows_with_missing_intended_answer_tokens": 0
+    },
+    "validation": {
+      "clean_rows_with_positive_tokens": 0,
+      "non_clean_rows_with_zero_positive_tokens": 0,
+      "non_clean_rows_with_uncovered_gold_span": 0,
+      "rows_with_missing_intended_answer_tokens": 0,
+      "windows_with_missing_intended_answer_tokens": 0
+    },
+    "test": {
+      "clean_rows_with_positive_tokens": 0,
+      "non_clean_rows_with_zero_positive_tokens": 0,
+      "non_clean_rows_with_uncovered_gold_span": 0,
+      "rows_with_missing_intended_answer_tokens": 0,
+      "windows_with_missing_intended_answer_tokens": 0
+    }
+  },
+  "window_counts": {
+    "train": 1808,
+    "validation": 229,
+    "test": 235
+  },
+  "source_row_counts": {
+    "train": 1662,
+    "validation": 214,
+    "test": 207
+  }
+}

selected_span_model_config.json

@@ -0,0 +1,386 @@
+{
+  "model_name": "span_deberta_v3_small_sliding_windows",
+  "base_model_name": "microsoft/deberta-v3-small",
+  "best_checkpoint_dir": "/content/drive/MyDrive/TLLM/03_models/span_deberta_v3_small_sliding_windows/best_checkpoint",
+  "last_checkpoint_dir": "/content/drive/MyDrive/TLLM/03_models/span_deberta_v3_small_sliding_windows/last_checkpoint",
+  "best_epoch": 8,
+  "selected_config": {
+    "threshold": 0.5,
+    "min_span_chars": 1,
+    "min_span_tokens": 1,
+    "merge_gap_chars": 1,
+    "strip_predicted_span_whitespace": true,
+    "drop_spans_without_alnum": true,
+    "score_name": "sum_non_O_probability"
+  },
+  "labels": [
+    "O",
+    "tool_output_conflict",
+    "overgeneration",
+    "missing_tool_action_recommendation"
+  ],
+  "label2id": {
+    "O": 0,
+    "tool_output_conflict": 1,
+    "overgeneration": 2,
+    "missing_tool_action_recommendation": 3
+  },
+  "id2label": {
+    "0": "O",
+    "1": "tool_output_conflict",
+    "2": "overgeneration",
+    "3": "missing_tool_action_recommendation"
+  },
+  "final_validation_token_metrics": {
+    "O": {
+      "precision": 0.9997869091167383,
+      "recall": 0.9987582487759881,
+      "f1": 0.9992723142182701,
+      "support": 28186,
+      "tp": 28151,
+      "fp": 6,
+      "fn": 35
+    },
+    "tool_output_conflict": {
+      "precision": 0.7666666666666667,
+      "recall": 0.9504132231404959,
+      "f1": 0.8487084870848709,
+      "support": 121,
+      "tp": 115,
+      "fp": 35,
+      "fn": 6
+    },
+    "overgeneration": {
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "support": 428,
+      "tp": 428,
+      "fp": 0,
+      "fn": 0
+    },
+    "missing_tool_action_recommendation": {
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "support": 892,
+      "tp": 892,
+      "fp": 0,
+      "fn": 0
+    },
+    "accuracy": 0.9986161271812873,
+    "macro_f1": 0.9619952003257852,
+    "weighted_f1": 0.9986898158940638,
+    "support": 29627
+  },
+  "final_test_token_metrics": {
+    "O": {
+      "precision": 0.9994592359321818,
+      "recall": 0.9951225692025084,
+      "f1": 0.9972861881258828,
+      "support": 31574,
+      "tp": 31420,
+      "fp": 17,
+      "fn": 154
+    },
+    "tool_output_conflict": {
+      "precision": 0.4188679245283019,
+      "recall": 0.8671875,
+      "f1": 0.564885496183206,
+      "support": 128,
+      "tp": 111,
+      "fp": 154,
+      "fn": 17
+    },
+    "overgeneration": {
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "support": 406,
+      "tp": 406,
+      "fp": 0,
+      "fn": 0
+    },
+    "missing_tool_action_recommendation": {
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "support": 925,
+      "tp": 925,
+      "fp": 0,
+      "fn": 0
+    },
+    "accuracy": 0.9948233584597221,
+    "macro_f1": 0.8905429210772722,
+    "weighted_f1": 0.9957200208094352,
+    "support": 33033
+  },
+  "final_validation_metrics": {
+    "config": {
+      "threshold": 0.5,
+      "min_span_chars": 1,
+      "min_span_tokens": 1,
+      "merge_gap_chars": 1,
+      "strip_predicted_span_whitespace": true,
+      "drop_spans_without_alnum": true,
+      "score_name": "sum_non_O_probability"
+    },
+    "num_rows": 214,
+    "row_multiclass_metrics": {
+      "clean": {
+        "precision": 1.0,
+        "recall": 0.9611650485436893,
+        "f1": 0.9801980198019802,
+        "support": 103,
+        "tp": 99,
+        "fp": 0,
+        "fn": 4
+      },
+      "tool_output_conflict": {
+        "precision": 0.9024390243902439,
+        "recall": 1.0,
+        "f1": 0.9487179487179488,
+        "support": 37,
+        "tp": 37,
+        "fp": 4,
+        "fn": 0
+      },
+      "overgeneration": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 37,
+        "tp": 37,
+        "fp": 0,
+        "fn": 0
+      },
+      "missing_tool_action_recommendation": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 37,
+        "tp": 37,
+        "fp": 0,
+        "fn": 0
+      },
+      "accuracy": 0.9813084112149533,
+      "macro_f1": 0.9822289921299823,
+      "weighted_f1": 0.9816026174867667,
+      "support": 214
+    },
+    "binary_example_metrics": {
+      "accuracy": 0.9813084112149533,
+      "precision": 0.9652173913043478,
+      "recall": 1.0,
+      "f1": 0.9823008849557522,
+      "tp": 111,
+      "fp": 4,
+      "fn": 0,
+      "tn": 99
+    },
+    "exact_span_metrics": {
+      "precision": 0.8333333333333334,
+      "recall": 0.9009009009009009,
+      "f1": 0.8658008658008659,
+      "matched": 100,
+      "gold_total": 111,
+      "pred_total": 120
+    },
+    "overlap_span_metrics_iou_0_01": {
+      "precision": 0.925,
+      "recall": 1.0,
+      "f1": 0.961038961038961,
+      "matched": 111,
+      "gold_total": 111,
+      "pred_total": 120,
+      "iou_threshold": 0.01
+    },
+    "overlap_span_metrics_iou_0_50": {
+      "precision": 0.9083333333333333,
+      "recall": 0.9819819819819819,
+      "f1": 0.9437229437229437,
+      "matched": 109,
+      "gold_total": 111,
+      "pred_total": 120,
+      "iou_threshold": 0.5
+    },
+    "char_micro_metrics": {
+      "precision": 0.9916501556750636,
+      "recall": 0.9984326018808778,
+      "f1": 0.9950298210735586,
+      "overlap_chars": 7007,
+      "gold_chars": 7018,
+      "pred_chars": 7066
+    },
+    "per_type_char_micro_metrics": {
+      "clean": {
+        "precision": 0.0,
+        "recall": 0.0,
+        "f1": 0.0,
+        "overlap_chars": 0,
+        "gold_chars": 0,
+        "pred_chars": 18
+      },
+      "overgeneration": {
+        "precision": 0.9948006932409013,
+        "recall": 1.0,
+        "f1": 0.9973935708079931,
+        "overlap_chars": 2296,
+        "gold_chars": 2296,
+        "pred_chars": 2308
+      },
+      "tool_output_conflict": {
+        "precision": 0.9037656903765691,
+        "recall": 0.9515418502202643,
+        "f1": 0.927038626609442,
+        "overlap_chars": 216,
+        "gold_chars": 227,
+        "pred_chars": 239
+      },
+      "missing_tool_action_recommendation": {
+        "precision": 0.998666962897134,
+        "recall": 1.0,
+        "f1": 0.9993330369052912,
+        "overlap_chars": 4495,
+        "gold_chars": 4495,
+        "pred_chars": 4501
+      }
+    },
+    "num_gold_spans": 111,
+    "num_predicted_spans": 120
+  },
+  "final_test_metrics": {
+    "config": {
+      "threshold": 0.5,
+      "min_span_chars": 1,
+      "min_span_tokens": 1,
+      "merge_gap_chars": 1,
+      "strip_predicted_span_whitespace": true,
+      "drop_spans_without_alnum": true,
+      "score_name": "sum_non_O_probability"
+    },
+    "num_rows": 207,
+    "row_multiclass_metrics": {
+      "clean": {
+        "precision": 0.9777777777777777,
+        "recall": 0.9166666666666666,
+        "f1": 0.946236559139785,
+        "support": 96,
+        "tp": 88,
+        "fp": 2,
+        "fn": 8
+      },
+      "tool_output_conflict": {
+        "precision": 0.813953488372093,
+        "recall": 0.9459459459459459,
+        "f1": 0.875,
+        "support": 37,
+        "tp": 35,
+        "fp": 8,
+        "fn": 2
+      },
+      "overgeneration": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 37,
+        "tp": 37,
+        "fp": 0,
+        "fn": 0
+      },
+      "missing_tool_action_recommendation": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 37,
+        "tp": 37,
+        "fp": 0,
+        "fn": 0
+      },
+      "accuracy": 0.9516908212560387,
+      "macro_f1": 0.9553091397849462,
+      "weighted_f1": 0.9527232351566153,
+      "support": 207
+    },
+    "binary_example_metrics": {
+      "accuracy": 0.9516908212560387,
+      "precision": 0.9316239316239316,
+      "recall": 0.9819819819819819,
+      "f1": 0.9561403508771931,
+      "tp": 109,
+      "fp": 8,
+      "fn": 2,
+      "tn": 88
+    },
+    "exact_span_metrics": {
+      "precision": 0.7357142857142858,
+      "recall": 0.9279279279279279,
+      "f1": 0.8207171314741037,
+      "matched": 103,
+      "gold_total": 111,
+      "pred_total": 140
+    },
+    "overlap_span_metrics_iou_0_01": {
+      "precision": 0.7785714285714286,
+      "recall": 0.9819819819819819,
+      "f1": 0.8685258964143425,
+      "matched": 109,
+      "gold_total": 111,
+      "pred_total": 140,
+      "iou_threshold": 0.01
+    },
+    "overlap_span_metrics_iou_0_50": {
+      "precision": 0.75,
+      "recall": 0.9459459459459459,
+      "f1": 0.8366533864541832,
+      "matched": 105,
+      "gold_total": 111,
+      "pred_total": 140,
+      "iou_threshold": 0.5
+    },
+    "char_micro_metrics": {
+      "precision": 0.971356003950896,
+      "recall": 0.9955169920462762,
+      "f1": 0.9832881016997572,
+      "overlap_chars": 6884,
+      "gold_chars": 6915,
+      "pred_chars": 7087
+    },
+    "per_type_char_micro_metrics": {
+      "clean": {
+        "precision": 0.0,
+        "recall": 0.0,
+        "f1": 0.0,
+        "overlap_chars": 0,
+        "gold_chars": 0,
+        "pred_chars": 72
+      },
+      "missing_tool_action_recommendation": {
+        "precision": 0.9879728843210146,
+        "recall": 1.0,
+        "f1": 0.9939500604993949,
+        "overlap_chars": 4518,
+        "gold_chars": 4518,
+        "pred_chars": 4573
+      },
+      "tool_output_conflict": {
+        "precision": 0.7251908396946565,
+        "recall": 0.8597285067873304,
+        "f1": 0.7867494824016563,
+        "overlap_chars": 190,
+        "gold_chars": 221,
+        "pred_chars": 262
+      },
+      "overgeneration": {
+        "precision": 0.998165137614679,
+        "recall": 1.0,
+        "f1": 0.9990817263544537,
+        "overlap_chars": 2176,
+        "gold_chars": 2176,
+        "pred_chars": 2180
+      }
+    },
+    "num_gold_spans": 111,
+    "num_predicted_spans": 140
+  }
+}

span_inference.py

@@ -0,0 +1,320 @@
+# span_inference.py
+# Inference helper for the DeBERTa-v3-small span-token hallucination detector.
+
+from collections import defaultdict
+import torch
+
+MAX_LENGTH = 512
+ANSWER_WINDOW_TOKENS = 384
+ANSWER_WINDOW_STRIDE = 256
+
+DEFAULT_DECODING_CONFIG = {
+    "threshold": 0.5,
+    "min_span_chars": 1,
+    "min_span_tokens": 1,
+    "merge_gap_chars": 1,
+    "strip_predicted_span_whitespace": True,
+    "drop_spans_without_alnum": True,
+    "score_name": "sum_non_O_probability",
+}
+
+SPAN_LABELS = [
+    "O",
+    "tool_output_conflict",
+    "overgeneration",
+    "missing_tool_action_recommendation",
+]
+
+
+def normalize_tool_names(names, max_names=80):
+    if not isinstance(names, list):
+        return ""
+    out = []
+    seen = set()
+    for item in names:
+        if isinstance(item, str):
+            item = " ".join(item.split()).strip()
+            if item and item.lower() not in seen:
+                out.append(item)
+                seen.add(item.lower())
+        if len(out) >= max_names:
+            break
+    return "; ".join(out)
+
+
+def has_alnum(text):
+    return any(ch.isalnum() for ch in text)
+
+
+def safe_divide(num, den):
+    return float(num) / float(den) if den else 0.0
+
+
+def answer_token_offsets(answer, tokenizer):
+    enc = tokenizer(
+        answer,
+        add_special_tokens=False,
+        return_offsets_mapping=True,
+        truncation=False,
+    )
+    offsets = []
+    for i, (start, end) in enumerate(enc["offset_mapping"]):
+        start = int(start)
+        end = int(end)
+        if end > start:
+            offsets.append({"answer_token_index": i, "start": start, "end": end})
+    return offsets
+
+
+def make_answer_windows(answer, tokenizer):
+    offsets = answer_token_offsets(answer, tokenizer)
+
+    if not offsets:
+        return [{
+            "window_index": 0,
+            "answer_window_start": 0,
+            "answer_window_end": len(answer),
+        }]
+
+    n = len(offsets)
+    if n <= ANSWER_WINDOW_TOKENS:
+        token_ranges = [(0, n)]
+    else:
+        starts = list(range(0, n, ANSWER_WINDOW_STRIDE))
+        final_start = max(0, n - ANSWER_WINDOW_TOKENS)
+        if starts[-1] != final_start:
+            starts.append(final_start)
+        starts = sorted(set(starts))
+        token_ranges = [(s, min(s + ANSWER_WINDOW_TOKENS, n)) for s in starts]
+
+    windows = []
+    for wi, (tok_start, tok_end) in enumerate(token_ranges):
+        char_start = offsets[tok_start]["start"]
+        char_end = offsets[tok_end - 1]["end"]
+        windows.append({
+            "window_index": wi,
+            "answer_window_start": int(char_start),
+            "answer_window_end": int(char_end),
+        })
+    return windows
+
+
+def build_window_text(record, window):
+    answer = record.get("output", "")
+    query = record.get("query", "")
+    context = record.get("context", "")
+    tool_names = normalize_tool_names(record.get("available_tool_names", []))
+
+    answer_window_text = answer[window["answer_window_start"]:window["answer_window_end"]]
+
+    prefix = "Answer window:\n"
+    text = prefix + answer_window_text
+    text += "\n\nQuestion:\n" + query
+    text += "\n\nTool responses:\n" + context
+
+    if tool_names:
+        text += "\n\nAvailable tool names:\n" + tool_names
+
+    return text, answer_window_text, len(prefix), len(prefix) + len(answer_window_text)
+
+
+def deduplicate_candidates(candidates):
+    by_key = {}
+    for cand in candidates:
+        key = (int(cand["start"]), int(cand["end"]))
+        if key not in by_key:
+            by_key[key] = dict(cand)
+            continue
+        old = by_key[key]
+        if cand["hallucination_probability"] > old["hallucination_probability"]:
+            by_key[key] = dict(cand)
+        else:
+            old["hallucination_probability"] = max(
+                old["hallucination_probability"],
+                cand["hallucination_probability"],
+            )
+            for label, prob in cand["type_probabilities"].items():
+                old["type_probabilities"][label] = max(old["type_probabilities"].get(label, 0.0), prob)
+    return sorted(by_key.values(), key=lambda x: (x["start"], x["end"]))
+
+
+def strip_span_whitespace(answer, start, end):
+    start = max(0, min(int(start), len(answer)))
+    end = max(0, min(int(end), len(answer)))
+    while start < end and answer[start].isspace():
+        start += 1
+    while end > start and answer[end - 1].isspace():
+        end -= 1
+    return start, end
+
+
+def merge_candidates_to_spans(answer, candidates, config):
+    threshold = float(config.get("threshold", DEFAULT_DECODING_CONFIG["threshold"]))
+    min_span_chars = int(config.get("min_span_chars", DEFAULT_DECODING_CONFIG["min_span_chars"]))
+    min_span_tokens = int(config.get("min_span_tokens", DEFAULT_DECODING_CONFIG["min_span_tokens"]))
+    merge_gap_chars = int(config.get("merge_gap_chars", DEFAULT_DECODING_CONFIG["merge_gap_chars"]))
+
+    selected = [
+        c for c in deduplicate_candidates(candidates)
+        if c["hallucination_probability"] >= threshold and c["end"] > c["start"]
+    ]
+    selected.sort(key=lambda x: (x["start"], x["end"]))
+
+    merged = []
+    cur = None
+
+    for item in selected:
+        if cur is None:
+            cur = {
+                "start": item["start"],
+                "end": item["end"],
+                "token_count": 1,
+                "score": item["hallucination_probability"],
+                "score_sum": item["hallucination_probability"],
+                "type_score_sums": defaultdict(float),
+            }
+            for label, prob in item["type_probabilities"].items():
+                cur["type_score_sums"][label] += float(prob)
+            continue
+
+        if item["start"] <= cur["end"] + merge_gap_chars:
+            cur["end"] = max(cur["end"], item["end"])
+            cur["token_count"] += 1
+            cur["score"] = max(cur["score"], item["hallucination_probability"])
+            cur["score_sum"] += item["hallucination_probability"]
+            for label, prob in item["type_probabilities"].items():
+                cur["type_score_sums"][label] += float(prob)
+        else:
+            merged.append(cur)
+            cur = {
+                "start": item["start"],
+                "end": item["end"],
+                "token_count": 1,
+                "score": item["hallucination_probability"],
+                "score_sum": item["hallucination_probability"],
+                "type_score_sums": defaultdict(float),
+            }
+            for label, prob in item["type_probabilities"].items():
+                cur["type_score_sums"][label] += float(prob)
+
+    if cur is not None:
+        merged.append(cur)
+
+    spans = []
+    for span in merged:
+        start, end = strip_span_whitespace(answer, span["start"], span["end"])
+        if end <= start:
+            continue
+        if (end - start) < min_span_chars:
+            continue
+        if span["token_count"] < min_span_tokens:
+            continue
+        text = answer[start:end]
+        if config.get("drop_spans_without_alnum", True) and not has_alnum(text):
+            continue
+
+        type_scores = {
+            label: safe_divide(value, span["token_count"])
+            for label, value in span["type_score_sums"].items()
+        }
+        pred_type = max(type_scores.items(), key=lambda x: x[1])[0] if type_scores else "overgeneration"
+
+        spans.append({
+            "start": int(start),
+            "end": int(end),
+            "text": text,
+            "label_type": "Predicted Hallucination",
+            "hallucination_type": pred_type,
+            "score": float(span["score"]),
+            "mean_score": float(safe_divide(span["score_sum"], span["token_count"])),
+            "type_scores": type_scores,
+            "token_count": int(span["token_count"]),
+        })
+
+    return spans
+
+
+def predict_record(record, model, tokenizer, device=None, decoding_config=None):
+    if device is None:
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+    if decoding_config is None:
+        decoding_config = DEFAULT_DECODING_CONFIG
+
+    model.to(device)
+    model.eval()
+
+    answer = record.get("output", "")
+    candidates = []
+
+    id2label = getattr(model.config, "id2label", None) or {i: label for i, label in enumerate(SPAN_LABELS)}
+    id2label = {int(k): v for k, v in id2label.items()}
+
+    for window in make_answer_windows(answer, tokenizer):
+        text, answer_window_text, local_answer_start, local_answer_end = build_window_text(record, window)
+        enc = tokenizer(
+            text,
+            truncation=True,
+            max_length=MAX_LENGTH,
+            padding=False,
+            return_offsets_mapping=True,
+            return_tensors="pt",
+            add_special_tokens=True,
+        )
+        offsets = enc.pop("offset_mapping")[0].tolist()
+        inputs = {k: v.to(device) for k, v in enc.items()}
+
+        with torch.no_grad():
+            logits = model(**inputs).logits[0]
+            probs = torch.softmax(logits.float(), dim=-1).detach().cpu()
+
+        for token_idx, (start, end) in enumerate(offsets):
+            start = int(start)
+            end = int(end)
+            if end <= start:
+                continue
+            if not (end > local_answer_start and start < local_answer_end):
+                continue
+
+            local_start = max(0, start - local_answer_start)
+            local_end = min(local_answer_end - local_answer_start, end - local_answer_start)
+
+            if local_end <= local_start:
+                continue
+
+            global_start = window["answer_window_start"] + local_start
+            global_end = window["answer_window_start"] + local_end
+
+            token_probs = probs[token_idx]
+            non_o_probs = token_probs[1:]
+            hallucination_probability = float(non_o_probs.sum().item())
+            best_non_o_id = int(torch.argmax(non_o_probs).item()) + 1
+
+            candidates.append({
+                "start": int(global_start),
+                "end": int(global_end),
+                "text": answer[global_start:global_end],
+                "hallucination_probability": hallucination_probability,
+                "pred_label": id2label.get(best_non_o_id, str(best_non_o_id)),
+                "type_probabilities": {
+                    id2label.get(i, str(i)): float(token_probs[i].item())
+                    for i in range(1, token_probs.shape[-1])
+                },
+                "o_probability": float(token_probs[0].item()),
+                "window_index": window["window_index"],
+            })
+
+    spans = merge_candidates_to_spans(answer, candidates, decoding_config)
+
+    if spans:
+        best_span = max(spans, key=lambda x: (x.get("score", 0.0), x["end"] - x["start"]))
+        row_label = best_span.get("hallucination_type", "overgeneration")
+    else:
+        row_label = "clean"
+
+    return {
+        "pred_hallucination_type": row_label,
+        "pred_is_hallucinated": bool(spans),
+        "predicted_spans": spans,
+        "num_token_candidates": len(candidates),
+        "decoding_config": decoding_config,
+    }

span_training_extra_config.json

@@ -0,0 +1,132 @@
+{
+  "saved_at": "2026-05-23T11:10:39",
+  "best_epoch": 8,
+  "best_validation_score_char_f1": 0.9950298210735586,
+  "best_validation_config": {
+    "threshold": 0.5,
+    "min_span_chars": 1,
+    "min_span_tokens": 1,
+    "merge_gap_chars": 1,
+    "strip_predicted_span_whitespace": true,
+    "drop_spans_without_alnum": true,
+    "score_name": "sum_non_O_probability"
+  },
+  "training_config": {
+    "base_model_name": "microsoft/deberta-v3-small",
+    "max_length": 512,
+    "span_labels": [
+      "O",
+      "tool_output_conflict",
+      "overgeneration",
+      "missing_tool_action_recommendation"
+    ],
+    "label2id": {
+      "O": 0,
+      "tool_output_conflict": 1,
+      "overgeneration": 2,
+      "missing_tool_action_recommendation": 3
+    },
+    "id2label": {
+      "0": "O",
+      "1": "tool_output_conflict",
+      "2": "overgeneration",
+      "3": "missing_tool_action_recommendation"
+    },
+    "micro_batch_size": 4,
+    "gradient_accumulation_steps": 4,
+    "effective_batch_size": 16,
+    "num_epochs": 8,
+    "patience": 3,
+    "learning_rate": 2e-05,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.1,
+    "warmup_steps": 90,
+    "total_update_steps": 904,
+    "max_grad_norm": 1.0,
+    "use_amp": false,
+    "force_fp32_training": true,
+    "amp_disabled_reason": "Previous run failed with GradScaler ValueError: Attempting to unscale FP16 gradients.",
+    "use_class_weights": true,
+    "class_weights": {
+      "O": 0.14653727412223816,
+      "tool_output_conflict": 1.7199265956878662,
+      "overgeneration": 1.2583367824554443,
+      "missing_tool_action_recommendation": 0.8751992583274841
+    },
+    "threshold_grid": [
+      0.05,
+      0.08,
+      0.1,
+      0.12,
+      0.15,
+      0.18,
+      0.2,
+      0.25,
+      0.3,
+      0.35,
+      0.4,
+      0.45,
+      0.5,
+      0.55,
+      0.6,
+      0.65,
+      0.7,
+      0.75,
+      0.8,
+      0.85,
+      0.9,
+      0.95
+    ],
+    "min_span_chars_grid": [
+      1,
+      2,
+      3,
+      5,
+      8,
+      12
+    ],
+    "min_span_tokens_grid": [
+      1,
+      2
+    ],
+    "merge_gap_chars": 1
+  },
+  "dataset_report": {
+    "tokenizer_class": "DebertaV2Tokenizer",
+    "model_max_length": 1000000000000000019884624838656,
+    "max_length": 512,
+    "train_windows": 1808,
+    "validation_windows": 229,
+    "test_windows": 235,
+    "train_supervised_token_count": 246653,
+    "validation_supervised_token_count": 29627,
+    "test_supervised_token_count": 33033,
+    "train_positive_token_count": 10688,
+    "validation_positive_token_count": 1441,
+    "test_positive_token_count": 1459,
+    "train_token_label_counts": {
+      "O": 235965,
+      "overgeneration": 3200,
+      "missing_tool_action_recommendation": 6615,
+      "tool_output_conflict": 873
+    },
+    "validation_token_label_counts": {
+      "O": 28186,
+      "overgeneration": 428,
+      "tool_output_conflict": 121,
+      "missing_tool_action_recommendation": 892
+    },
+    "test_token_label_counts": {
+      "O": 31574,
+      "missing_tool_action_recommendation": 925,
+      "tool_output_conflict": 128,
+      "overgeneration": 406
+    },
+    "class_weights": {
+      "O": 0.14653727412223816,
+      "tool_output_conflict": 1.7199265956878662,
+      "overgeneration": 1.2583367824554443,
+      "missing_tool_action_recommendation": 0.8751992583274841
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": [
+    "[PAD]",
+    "[CLS]",
+    "[SEP]"
+  ],
+  "is_local": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_id": 3,
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}