| { |
| "datasets": { |
| "hard_nei_source": "andreiaalexa/scifact-relevance-pairs title/train and BM25-retrieved non-cited SciFact train documents", |
| "test_policy": "Validation rows with train-overlapping normalized claims are removed; remaining validation rows are split by normalized claim key into validation and derived test.", |
| "verifier_primary": "allenai/scifact_entailment train plus allenai/scifact train retrieval-aware snippets" |
| }, |
| "full_scifact_validation": { |
| "device": "cuda", |
| "end_to_end": { |
| "top1": { |
| "accuracy": 0.5711111111111111, |
| "confusion_matrix": { |
| "NOT_ENOUGH_INFO": { |
| "NOT_ENOUGH_INFO": 82, |
| "REFUTES": 7, |
| "SUPPORTS": 23 |
| }, |
| "REFUTES": { |
| "NOT_ENOUGH_INFO": 67, |
| "REFUTES": 47, |
| "SUPPORTS": 8 |
| }, |
| "SUPPORTS": { |
| "NOT_ENOUGH_INFO": 68, |
| "REFUTES": 20, |
| "SUPPORTS": 128 |
| } |
| }, |
| "examples": 450, |
| "gold_label_distribution": { |
| "NOT_ENOUGH_INFO": 112, |
| "REFUTES": 122, |
| "SUPPORTS": 216 |
| }, |
| "macro_f1": 0.5535795821874849, |
| "per_class": { |
| "NOT_ENOUGH_INFO": { |
| "f1-score": 0.49848024316109424, |
| "precision": 0.3778801843317972, |
| "recall": 0.7321428571428571, |
| "support": 112.0 |
| }, |
| "REFUTES": { |
| "f1-score": 0.47959183673469385, |
| "precision": 0.6351351351351351, |
| "recall": 0.38524590163934425, |
| "support": 122.0 |
| }, |
| "SUPPORTS": { |
| "f1-score": 0.6826666666666666, |
| "precision": 0.8050314465408805, |
| "recall": 0.5925925925925926, |
| "support": 216.0 |
| }, |
| "accuracy": 0.5711111111111111, |
| "macro avg": { |
| "f1-score": 0.5535795821874849, |
| "precision": 0.606015588669271, |
| "recall": 0.5699937837915979, |
| "support": 450.0 |
| }, |
| "weighted avg": { |
| "f1-score": 0.5817688695903894, |
| "precision": 0.6526574657432843, |
| "recall": 0.5711111111111111, |
| "support": 450.0 |
| } |
| }, |
| "predicted_label_distribution": { |
| "NOT_ENOUGH_INFO": 217, |
| "REFUTES": 74, |
| "SUPPORTS": 159 |
| }, |
| "strategy": "top1", |
| "supported_refuted_accuracy_when_evidence_retrieved_at_5": 0.6415094339622641, |
| "supported_refuted_examples_with_evidence_retrieved_at_5": 265 |
| }, |
| "top5_decisive": { |
| "accuracy": 0.6355555555555555, |
| "confusion_matrix": { |
| "NOT_ENOUGH_INFO": { |
| "NOT_ENOUGH_INFO": 61, |
| "REFUTES": 12, |
| "SUPPORTS": 39 |
| }, |
| "REFUTES": { |
| "NOT_ENOUGH_INFO": 48, |
| "REFUTES": 53, |
| "SUPPORTS": 21 |
| }, |
| "SUPPORTS": { |
| "NOT_ENOUGH_INFO": 29, |
| "REFUTES": 15, |
| "SUPPORTS": 172 |
| } |
| }, |
| "examples": 450, |
| "gold_label_distribution": { |
| "NOT_ENOUGH_INFO": 112, |
| "REFUTES": 122, |
| "SUPPORTS": 216 |
| }, |
| "macro_f1": 0.5935365393682225, |
| "per_class": { |
| "NOT_ENOUGH_INFO": { |
| "f1-score": 0.488, |
| "precision": 0.4420289855072464, |
| "recall": 0.5446428571428571, |
| "support": 112.0 |
| }, |
| "REFUTES": { |
| "f1-score": 0.5247524752475248, |
| "precision": 0.6625, |
| "recall": 0.4344262295081967, |
| "support": 122.0 |
| }, |
| "SUPPORTS": { |
| "f1-score": 0.7678571428571429, |
| "precision": 0.7413793103448276, |
| "recall": 0.7962962962962963, |
| "support": 216.0 |
| }, |
| "accuracy": 0.6355555555555555, |
| "macro avg": { |
| "f1-score": 0.5935365393682225, |
| "precision": 0.6153027652840247, |
| "recall": 0.5917884609824501, |
| "support": 450.0 |
| }, |
| "weighted avg": { |
| "f1-score": 0.6322954329718686, |
| "precision": 0.6454892831362097, |
| "recall": 0.6355555555555555, |
| "support": 450.0 |
| } |
| }, |
| "predicted_label_distribution": { |
| "NOT_ENOUGH_INFO": 138, |
| "REFUTES": 80, |
| "SUPPORTS": 232 |
| }, |
| "strategy": "top5_decisive", |
| "supported_refuted_accuracy_when_evidence_retrieved_at_5": 0.7660377358490567, |
| "supported_refuted_examples_with_evidence_retrieved_at_5": 265 |
| } |
| }, |
| "retrieval": { |
| "evaluated_queries": 450.0, |
| "mrr@5": 0.5610740740740741, |
| "ndcg@1": 0.4866666666666667, |
| "ndcg@3": 0.5487532554897722, |
| "ndcg@5": 0.5739882576045233, |
| "recall@1": 0.4866666666666667, |
| "recall@3": 0.6355555555555555, |
| "recall@5": 0.6755555555555556 |
| } |
| }, |
| "hub_model_id": "rishhh/verisci-claim-verifier-dense-adapted-seed123", |
| "latency_seconds": 193.2834756150114, |
| "limitations": [ |
| "The derived test split is small and originates from SciFact validation, not an official hidden test set.", |
| "Retrieval-aware negatives are generated by BM25 over SciFact train and may overfit SciFact retrieval artifacts.", |
| "Full SciFact validation is public and should be treated as a promotion gate, not a hidden benchmark.", |
| "SciFact is CC-BY-NC-2.0, so downstream use should respect non-commercial licensing constraints." |
| ], |
| "max_length": 512, |
| "model_name": "rishhh/verisci-claim-verifier-retrieval-adapted-seed123", |
| "seed": 123, |
| "test": { |
| "accuracy": 0.893491124260355, |
| "confusion_matrix": { |
| "NOT_ENOUGH_INFO": { |
| "NOT_ENOUGH_INFO": 70, |
| "REFUTES": 0, |
| "SUPPORTS": 1 |
| }, |
| "REFUTES": { |
| "NOT_ENOUGH_INFO": 4, |
| "REFUTES": 20, |
| "SUPPORTS": 7 |
| }, |
| "SUPPORTS": { |
| "NOT_ENOUGH_INFO": 1, |
| "REFUTES": 5, |
| "SUPPORTS": 61 |
| } |
| }, |
| "examples": 169, |
| "expected_calibration_error": 0.09891798686699052, |
| "gold_label_distribution": { |
| "NOT_ENOUGH_INFO": 71, |
| "REFUTES": 31, |
| "SUPPORTS": 67 |
| }, |
| "macro_f1": 0.8567495491347223, |
| "per_class": { |
| "NOT_ENOUGH_INFO": { |
| "f1-score": 0.958904109589041, |
| "precision": 0.9333333333333333, |
| "recall": 0.9859154929577465, |
| "support": 71.0 |
| }, |
| "REFUTES": { |
| "f1-score": 0.7142857142857143, |
| "precision": 0.8, |
| "recall": 0.6451612903225806, |
| "support": 31.0 |
| }, |
| "SUPPORTS": { |
| "f1-score": 0.8970588235294118, |
| "precision": 0.8840579710144928, |
| "recall": 0.9104477611940298, |
| "support": 67.0 |
| }, |
| "accuracy": 0.893491124260355, |
| "macro avg": { |
| "f1-score": 0.8567495491347223, |
| "precision": 0.8724637681159421, |
| "recall": 0.847174848158119, |
| "support": 169.0 |
| }, |
| "weighted avg": { |
| "f1-score": 0.8895147343204121, |
| "precision": 0.8893405368321755, |
| "recall": 0.893491124260355, |
| "support": 169.0 |
| } |
| }, |
| "predicted_label_distribution": { |
| "NOT_ENOUGH_INFO": 75, |
| "REFUTES": 25, |
| "SUPPORTS": 69 |
| }, |
| "strategy": "test" |
| }, |
| "training": { |
| "base_scifact_entailment_train_examples": 916, |
| "class_weights": { |
| "NOT_ENOUGH_INFO": 0.9040744304656982, |
| "REFUTES": 1.4886423349380493, |
| "SUPPORTS": 0.8182346224784851 |
| }, |
| "dense_retrieved_nei_max": 350, |
| "dense_retriever_id": "rishhh/verisci-scifact-e5-retriever", |
| "effective_batch_size": 16, |
| "epochs": 2, |
| "hard_title_nei_max": 150, |
| "hybrid_alpha": 0.75, |
| "learning_rate": 6e-06, |
| "retrieval_aware_examples_added": 2514, |
| "retrieved_nei_max": 250, |
| "test_examples": 169, |
| "train_examples": 2818, |
| "train_label_distribution": { |
| "NOT_ENOUGH_INFO": 1039, |
| "REFUTES": 631, |
| "SUPPORTS": 1148 |
| }, |
| "validation_examples": 168, |
| "validation_rows_removed_for_train_claim_overlap": 2 |
| }, |
| "validation": { |
| "accuracy": 0.8630952380952381, |
| "confusion_matrix": { |
| "NOT_ENOUGH_INFO": { |
| "NOT_ENOUGH_INFO": 55, |
| "REFUTES": 0, |
| "SUPPORTS": 2 |
| }, |
| "REFUTES": { |
| "NOT_ENOUGH_INFO": 5, |
| "REFUTES": 28, |
| "SUPPORTS": 7 |
| }, |
| "SUPPORTS": { |
| "NOT_ENOUGH_INFO": 3, |
| "REFUTES": 6, |
| "SUPPORTS": 62 |
| } |
| }, |
| "examples": 168, |
| "expected_calibration_error": 0.11418107826085311, |
| "gold_label_distribution": { |
| "NOT_ENOUGH_INFO": 57, |
| "REFUTES": 40, |
| "SUPPORTS": 71 |
| }, |
| "macro_f1": 0.8488876200143807, |
| "per_class": { |
| "NOT_ENOUGH_INFO": { |
| "f1-score": 0.9166666666666666, |
| "precision": 0.873015873015873, |
| "recall": 0.9649122807017544, |
| "support": 57.0 |
| }, |
| "REFUTES": { |
| "f1-score": 0.7567567567567568, |
| "precision": 0.8235294117647058, |
| "recall": 0.7, |
| "support": 40.0 |
| }, |
| "SUPPORTS": { |
| "f1-score": 0.8732394366197183, |
| "precision": 0.8732394366197183, |
| "recall": 0.8732394366197183, |
| "support": 71.0 |
| }, |
| "accuracy": 0.8630952380952381, |
| "macro avg": { |
| "f1-score": 0.8488876200143807, |
| "precision": 0.8565949071334323, |
| "recall": 0.8460505724404909, |
| "support": 168.0 |
| }, |
| "weighted avg": { |
| "f1-score": 0.860239703989704, |
| "precision": 0.8613278644791249, |
| "recall": 0.8630952380952381, |
| "support": 168.0 |
| } |
| }, |
| "predicted_label_distribution": { |
| "NOT_ENOUGH_INFO": 63, |
| "REFUTES": 34, |
| "SUPPORTS": 71 |
| }, |
| "strategy": "validation" |
| } |
| } |
