{
  "datasets": {
    "hard_nei_source": "andreiaalexa/scifact-relevance-pairs title/train and BM25-retrieved non-cited SciFact train documents",
    "test_policy": "Validation rows with train-overlapping normalized claims are removed; remaining validation rows are split by normalized claim key into validation and derived test.",
    "verifier_primary": "allenai/scifact_entailment train plus allenai/scifact train retrieval-aware snippets"
  },
  "full_scifact_validation": {
    "device": "cuda",
    "end_to_end": {
      "top1": {
        "accuracy": 0.5711111111111111,
        "confusion_matrix": {
          "NOT_ENOUGH_INFO": {
            "NOT_ENOUGH_INFO": 82,
            "REFUTES": 7,
            "SUPPORTS": 23
          },
          "REFUTES": {
            "NOT_ENOUGH_INFO": 67,
            "REFUTES": 47,
            "SUPPORTS": 8
          },
          "SUPPORTS": {
            "NOT_ENOUGH_INFO": 68,
            "REFUTES": 20,
            "SUPPORTS": 128
          }
        },
        "examples": 450,
        "gold_label_distribution": {
          "NOT_ENOUGH_INFO": 112,
          "REFUTES": 122,
          "SUPPORTS": 216
        },
        "macro_f1": 0.5535795821874849,
        "per_class": {
          "NOT_ENOUGH_INFO": {
            "f1-score": 0.49848024316109424,
            "precision": 0.3778801843317972,
            "recall": 0.7321428571428571,
            "support": 112.0
          },
          "REFUTES": {
            "f1-score": 0.47959183673469385,
            "precision": 0.6351351351351351,
            "recall": 0.38524590163934425,
            "support": 122.0
          },
          "SUPPORTS": {
            "f1-score": 0.6826666666666666,
            "precision": 0.8050314465408805,
            "recall": 0.5925925925925926,
            "support": 216.0
          },
          "accuracy": 0.5711111111111111,
          "macro avg": {
            "f1-score": 0.5535795821874849,
            "precision": 0.606015588669271,
            "recall": 0.5699937837915979,
            "support": 450.0
          },
          "weighted avg": {
            "f1-score": 0.5817688695903894,
            "precision": 0.6526574657432843,
            "recall": 0.5711111111111111,
            "support": 450.0
          }
        },
        "predicted_label_distribution": {
          "NOT_ENOUGH_INFO": 217,
          "REFUTES": 74,
          "SUPPORTS": 159
        },
        "strategy": "top1",
        "supported_refuted_accuracy_when_evidence_retrieved_at_5": 0.6415094339622641,
        "supported_refuted_examples_with_evidence_retrieved_at_5": 265
      },
      "top5_decisive": {
        "accuracy": 0.6355555555555555,
        "confusion_matrix": {
          "NOT_ENOUGH_INFO": {
            "NOT_ENOUGH_INFO": 61,
            "REFUTES": 12,
            "SUPPORTS": 39
          },
          "REFUTES": {
            "NOT_ENOUGH_INFO": 48,
            "REFUTES": 53,
            "SUPPORTS": 21
          },
          "SUPPORTS": {
            "NOT_ENOUGH_INFO": 29,
            "REFUTES": 15,
            "SUPPORTS": 172
          }
        },
        "examples": 450,
        "gold_label_distribution": {
          "NOT_ENOUGH_INFO": 112,
          "REFUTES": 122,
          "SUPPORTS": 216
        },
        "macro_f1": 0.5935365393682225,
        "per_class": {
          "NOT_ENOUGH_INFO": {
            "f1-score": 0.488,
            "precision": 0.4420289855072464,
            "recall": 0.5446428571428571,
            "support": 112.0
          },
          "REFUTES": {
            "f1-score": 0.5247524752475248,
            "precision": 0.6625,
            "recall": 0.4344262295081967,
            "support": 122.0
          },
          "SUPPORTS": {
            "f1-score": 0.7678571428571429,
            "precision": 0.7413793103448276,
            "recall": 0.7962962962962963,
            "support": 216.0
          },
          "accuracy": 0.6355555555555555,
          "macro avg": {
            "f1-score": 0.5935365393682225,
            "precision": 0.6153027652840247,
            "recall": 0.5917884609824501,
            "support": 450.0
          },
          "weighted avg": {
            "f1-score": 0.6322954329718686,
            "precision": 0.6454892831362097,
            "recall": 0.6355555555555555,
            "support": 450.0
          }
        },
        "predicted_label_distribution": {
          "NOT_ENOUGH_INFO": 138,
          "REFUTES": 80,
          "SUPPORTS": 232
        },
        "strategy": "top5_decisive",
        "supported_refuted_accuracy_when_evidence_retrieved_at_5": 0.7660377358490567,
        "supported_refuted_examples_with_evidence_retrieved_at_5": 265
      }
    },
    "retrieval": {
      "evaluated_queries": 450.0,
      "mrr@5": 0.5610740740740741,
      "ndcg@1": 0.4866666666666667,
      "ndcg@3": 0.5487532554897722,
      "ndcg@5": 0.5739882576045233,
      "recall@1": 0.4866666666666667,
      "recall@3": 0.6355555555555555,
      "recall@5": 0.6755555555555556
    }
  },
  "hub_model_id": "rishhh/verisci-claim-verifier-dense-adapted-seed123",
  "latency_seconds": 193.2834756150114,
  "limitations": [
    "The derived test split is small and originates from SciFact validation, not an official hidden test set.",
    "Retrieval-aware negatives are generated by BM25 over SciFact train and may overfit SciFact retrieval artifacts.",
    "Full SciFact validation is public and should be treated as a promotion gate, not a hidden benchmark.",
    "SciFact is CC-BY-NC-2.0, so downstream use should respect non-commercial licensing constraints."
  ],
  "max_length": 512,
  "model_name": "rishhh/verisci-claim-verifier-retrieval-adapted-seed123",
  "seed": 123,
  "test": {
    "accuracy": 0.893491124260355,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 70,
        "REFUTES": 0,
        "SUPPORTS": 1
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 4,
        "REFUTES": 20,
        "SUPPORTS": 7
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 1,
        "REFUTES": 5,
        "SUPPORTS": 61
      }
    },
    "examples": 169,
    "expected_calibration_error": 0.09891798686699052,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 71,
      "REFUTES": 31,
      "SUPPORTS": 67
    },
    "macro_f1": 0.8567495491347223,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.958904109589041,
        "precision": 0.9333333333333333,
        "recall": 0.9859154929577465,
        "support": 71.0
      },
      "REFUTES": {
        "f1-score": 0.7142857142857143,
        "precision": 0.8,
        "recall": 0.6451612903225806,
        "support": 31.0
      },
      "SUPPORTS": {
        "f1-score": 0.8970588235294118,
        "precision": 0.8840579710144928,
        "recall": 0.9104477611940298,
        "support": 67.0
      },
      "accuracy": 0.893491124260355,
      "macro avg": {
        "f1-score": 0.8567495491347223,
        "precision": 0.8724637681159421,
        "recall": 0.847174848158119,
        "support": 169.0
      },
      "weighted avg": {
        "f1-score": 0.8895147343204121,
        "precision": 0.8893405368321755,
        "recall": 0.893491124260355,
        "support": 169.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 75,
      "REFUTES": 25,
      "SUPPORTS": 69
    },
    "strategy": "test"
  },
  "training": {
    "base_scifact_entailment_train_examples": 916,
    "class_weights": {
      "NOT_ENOUGH_INFO": 0.9040744304656982,
      "REFUTES": 1.4886423349380493,
      "SUPPORTS": 0.8182346224784851
    },
    "dense_retrieved_nei_max": 350,
    "dense_retriever_id": "rishhh/verisci-scifact-e5-retriever",
    "effective_batch_size": 16,
    "epochs": 2,
    "hard_title_nei_max": 150,
    "hybrid_alpha": 0.75,
    "learning_rate": 6e-06,
    "retrieval_aware_examples_added": 2514,
    "retrieved_nei_max": 250,
    "test_examples": 169,
    "train_examples": 2818,
    "train_label_distribution": {
      "NOT_ENOUGH_INFO": 1039,
      "REFUTES": 631,
      "SUPPORTS": 1148
    },
    "validation_examples": 168,
    "validation_rows_removed_for_train_claim_overlap": 2
  },
  "validation": {
    "accuracy": 0.8630952380952381,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 55,
        "REFUTES": 0,
        "SUPPORTS": 2
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 5,
        "REFUTES": 28,
        "SUPPORTS": 7
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 3,
        "REFUTES": 6,
        "SUPPORTS": 62
      }
    },
    "examples": 168,
    "expected_calibration_error": 0.11418107826085311,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 57,
      "REFUTES": 40,
      "SUPPORTS": 71
    },
    "macro_f1": 0.8488876200143807,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.9166666666666666,
        "precision": 0.873015873015873,
        "recall": 0.9649122807017544,
        "support": 57.0
      },
      "REFUTES": {
        "f1-score": 0.7567567567567568,
        "precision": 0.8235294117647058,
        "recall": 0.7,
        "support": 40.0
      },
      "SUPPORTS": {
        "f1-score": 0.8732394366197183,
        "precision": 0.8732394366197183,
        "recall": 0.8732394366197183,
        "support": 71.0
      },
      "accuracy": 0.8630952380952381,
      "macro avg": {
        "f1-score": 0.8488876200143807,
        "precision": 0.8565949071334323,
        "recall": 0.8460505724404909,
        "support": 168.0
      },
      "weighted avg": {
        "f1-score": 0.860239703989704,
        "precision": 0.8613278644791249,
        "recall": 0.8630952380952381,
        "support": 168.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 63,
      "REFUTES": 34,
      "SUPPORTS": 71
    },
    "strategy": "validation"
  }
}