{
  "dataset": "allenai/scifact",
  "decision_policy": {
    "min_nei_recall": 0.74,
    "strategy": "argmax_fallback_no_policy_met_nei_recall_floor"
  },
  "evaluation_scope": "constrained_selector_joint_verifier_validation",
  "hub_model_id": "rishhh/verisci-claim-verifier-constrained-selector-joint-seed123",
  "hybrid_alpha": 0.75,
  "latency_seconds": 789.2572160660002,
  "limitations": [
    "Selector supervision uses SciFact evidence sentence IDs when the evidence document is retrieved.",
    "SciFact validation is public and is not a hidden benchmark.",
    "This is still limited by retrieval recall and by the small SciFact train set."
  ],
  "max_selector_sentences_per_doc": 5,
  "max_snippets": 6,
  "max_snippets_per_doc": 2,
  "min_dev_nei_recall": 0.74,
  "promotion_gate": {
    "baseline_accuracy": 0.691,
    "candidate_accuracy": 0.7,
    "candidate_macro_f1": 0.661419819837583,
    "min_accuracy_delta": 0.01,
    "passed": false,
    "reason": "candidate_accuracy_below_target: 0.700 < 0.910",
    "target_accuracy": 0.91
  },
  "retrieval_oracle": {
    "accuracy": 0.9288888888888889,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 112,
        "REFUTES": 0,
        "SUPPORTS": 0
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 20,
        "REFUTES": 102,
        "SUPPORTS": 0
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 12,
        "REFUTES": 0,
        "SUPPORTS": 204
      }
    },
    "examples": 450,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 112,
      "REFUTES": 122,
      "SUPPORTS": 216
    },
    "macro_f1": 0.919047619047619,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.875,
        "precision": 0.7777777777777778,
        "recall": 1.0,
        "support": 112.0
      },
      "REFUTES": {
        "f1-score": 0.9107142857142857,
        "precision": 1.0,
        "recall": 0.8360655737704918,
        "support": 122.0
      },
      "SUPPORTS": {
        "f1-score": 0.9714285714285714,
        "precision": 1.0,
        "recall": 0.9444444444444444,
        "support": 216.0
      },
      "accuracy": 0.9288888888888889,
      "macro avg": {
        "f1-score": 0.919047619047619,
        "precision": 0.9259259259259259,
        "recall": 0.926836672738312,
        "support": 450.0
      },
      "weighted avg": {
        "f1-score": 0.930968253968254,
        "precision": 0.9446913580246913,
        "recall": 0.9288888888888889,
        "support": 450.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 144,
      "REFUTES": 102,
      "SUPPORTS": 204
    },
    "strategy": "retrieval_oracle_nei_when_missing"
  },
  "retriever_id": "rishhh/verisci-scifact-e5-retriever",
  "seed": 123,
  "selection_protocol": {
    "scope": "SciFact train only; train/dev split by claim id",
    "selector_selection_metric": "best epoch by SciFact train-dev macro F1",
    "threshold_selection_metric": "best train-dev accuracy with NEI recall floor",
    "validation_used_for_model_selection": false,
    "validation_used_for_training_examples": false,
    "verifier_selection_metric": "best epoch by SciFact train-dev macro F1"
  },
  "selector_base_model_id": "microsoft/deberta-v3-small",
  "selector_dev_examples": 13506,
  "selector_train_examples": 17733,
  "split": "validation",
  "top_k": 5,
  "validation": {
    "accuracy": 0.7,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 85,
        "REFUTES": 7,
        "SUPPORTS": 20
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 50,
        "REFUTES": 52,
        "SUPPORTS": 20
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 30,
        "REFUTES": 8,
        "SUPPORTS": 178
      }
    },
    "examples": 450,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 112,
      "REFUTES": 122,
      "SUPPORTS": 216
    },
    "macro_f1": 0.661419819837583,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.6137184115523465,
        "precision": 0.5151515151515151,
        "recall": 0.7589285714285714,
        "support": 112.0
      },
      "REFUTES": {
        "f1-score": 0.5502645502645502,
        "precision": 0.7761194029850746,
        "recall": 0.4262295081967213,
        "support": 122.0
      },
      "SUPPORTS": {
        "f1-score": 0.8202764976958525,
        "precision": 0.8165137614678899,
        "recall": 0.8240740740740741,
        "support": 216.0
      },
      "accuracy": 0.7,
      "macro avg": {
        "f1-score": 0.661419819837583,
        "precision": 0.7025948932014933,
        "recall": 0.6697440512331223,
        "support": 450.0
      },
      "weighted avg": {
        "f1-score": 0.6956632460632046,
        "precision": 0.7305566874182512,
        "recall": 0.7,
        "support": 450.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 165,
      "REFUTES": 67,
      "SUPPORTS": 218
    },
    "strategy": "scifact_validation_original_labels"
  },
  "validation_examples": 450,
  "validation_retrieved_evidence_label_eval": {
    "accuracy": 0.7355555555555555,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 109,
        "REFUTES": 7,
        "SUPPORTS": 28
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 30,
        "REFUTES": 52,
        "SUPPORTS": 20
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 26,
        "REFUTES": 8,
        "SUPPORTS": 170
      }
    },
    "examples": 450,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 144,
      "REFUTES": 102,
      "SUPPORTS": 204
    },
    "macro_f1": 0.7088578124330206,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.7055016181229773,
        "precision": 0.6606060606060606,
        "recall": 0.7569444444444444,
        "support": 144.0
      },
      "REFUTES": {
        "f1-score": 0.6153846153846154,
        "precision": 0.7761194029850746,
        "recall": 0.5098039215686274,
        "support": 102.0
      },
      "SUPPORTS": {
        "f1-score": 0.8056872037914692,
        "precision": 0.7798165137614679,
        "recall": 0.8333333333333334,
        "support": 204.0
      },
      "accuracy": 0.7355555555555555,
      "macro avg": {
        "f1-score": 0.7088578124330206,
        "precision": 0.7388473257842011,
        "recall": 0.7000272331154683,
        "support": 450.0
      },
      "weighted avg": {
        "f1-score": 0.7304925630053316,
        "precision": 0.7408311569757551,
        "recall": 0.7355555555555555,
        "support": 450.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 165,
      "REFUTES": 67,
      "SUPPORTS": 218
    },
    "strategy": "scifact_validation_retrieved_evidence_labels"
  },
  "validation_retrieved_evidence_label_eval_thresholded": {
    "accuracy": 0.7355555555555555,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 109,
        "REFUTES": 7,
        "SUPPORTS": 28
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 30,
        "REFUTES": 52,
        "SUPPORTS": 20
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 26,
        "REFUTES": 8,
        "SUPPORTS": 170
      }
    },
    "examples": 450,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 144,
      "REFUTES": 102,
      "SUPPORTS": 204
    },
    "macro_f1": 0.7088578124330206,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.7055016181229773,
        "precision": 0.6606060606060606,
        "recall": 0.7569444444444444,
        "support": 144.0
      },
      "REFUTES": {
        "f1-score": 0.6153846153846154,
        "precision": 0.7761194029850746,
        "recall": 0.5098039215686274,
        "support": 102.0
      },
      "SUPPORTS": {
        "f1-score": 0.8056872037914692,
        "precision": 0.7798165137614679,
        "recall": 0.8333333333333334,
        "support": 204.0
      },
      "accuracy": 0.7355555555555555,
      "macro avg": {
        "f1-score": 0.7088578124330206,
        "precision": 0.7388473257842011,
        "recall": 0.7000272331154683,
        "support": 450.0
      },
      "weighted avg": {
        "f1-score": 0.7304925630053316,
        "precision": 0.7408311569757551,
        "recall": 0.7355555555555555,
        "support": 450.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 165,
      "REFUTES": 67,
      "SUPPORTS": 218
    },
    "strategy": "scifact_validation_retrieved_evidence_labels_thresholded"
  },
  "validation_thresholded": {
    "accuracy": 0.7,
    "confusion_matrix": {
      "NOT_ENOUGH_INFO": {
        "NOT_ENOUGH_INFO": 85,
        "REFUTES": 7,
        "SUPPORTS": 20
      },
      "REFUTES": {
        "NOT_ENOUGH_INFO": 50,
        "REFUTES": 52,
        "SUPPORTS": 20
      },
      "SUPPORTS": {
        "NOT_ENOUGH_INFO": 30,
        "REFUTES": 8,
        "SUPPORTS": 178
      }
    },
    "examples": 450,
    "gold_label_distribution": {
      "NOT_ENOUGH_INFO": 112,
      "REFUTES": 122,
      "SUPPORTS": 216
    },
    "macro_f1": 0.661419819837583,
    "per_class": {
      "NOT_ENOUGH_INFO": {
        "f1-score": 0.6137184115523465,
        "precision": 0.5151515151515151,
        "recall": 0.7589285714285714,
        "support": 112.0
      },
      "REFUTES": {
        "f1-score": 0.5502645502645502,
        "precision": 0.7761194029850746,
        "recall": 0.4262295081967213,
        "support": 122.0
      },
      "SUPPORTS": {
        "f1-score": 0.8202764976958525,
        "precision": 0.8165137614678899,
        "recall": 0.8240740740740741,
        "support": 216.0
      },
      "accuracy": 0.7,
      "macro avg": {
        "f1-score": 0.661419819837583,
        "precision": 0.7025948932014933,
        "recall": 0.6697440512331223,
        "support": 450.0
      },
      "weighted avg": {
        "f1-score": 0.6956632460632046,
        "precision": 0.7305566874182512,
        "recall": 0.7,
        "support": 450.0
      }
    },
    "predicted_label_distribution": {
      "NOT_ENOUGH_INFO": 165,
      "REFUTES": 67,
      "SUPPORTS": 218
    },
    "strategy": "scifact_validation_original_labels_thresholded"
  },
  "verifier_base_model_id": "rishhh/verisci-claim-verifier-retrieval-adapted-seed123",
  "verifier_dev_examples": 545,
  "verifier_train_examples": 716
}