holdout_metrics.json

{
  "n_eval": 3487,
  "overall": {
    "tp": 5214,
    "fp": 1086,
    "fn": 1028,
    "precision": 0.8276190476190476,
    "recall": 0.8353091957705864,
    "f05": 0.829145728643216,
    "f1": 0.8314463402966035
  },
  "per_key": {
    "producer": {
      "tp": 1143,
      "fp": 296,
      "fn": 306,
      "precision": 0.7943015983321752,
      "recall": 0.7888198757763976,
      "f05": 0.7931991672449688,
      "f1": 0.7915512465373963
    },
    "year": {
      "tp": 1085,
      "fp": 207,
      "fn": 265,
      "precision": 0.8397832817337462,
      "recall": 0.8037037037037037,
      "f05": 0.8323105247008286,
      "f1": 0.8213474640423922
    },
    "geography": {
      "tp": 1715,
      "fp": 371,
      "fn": 312,
      "precision": 0.8221476510067114,
      "recall": 0.8460779477059694,
      "f05": 0.8268247999228617,
      "f1": 0.8339411621687333
    },
    "acronym": {
      "tp": 1271,
      "fp": 212,
      "fn": 145,
      "precision": 0.8570465273095077,
      "recall": 0.8975988700564972,
      "f05": 0.8648611867174743,
      "f1": 0.8768540876164194
    }
  },
  "verbatim": {
    "emitted": 6300,
    "verbatim": 6290,
    "rate": 0.9984126984126984,
    "per_key": {
      "producer": {
        "emitted": 1439,
        "verbatim": 1436
      },
      "year": {
        "emitted": 1292,
        "verbatim": 1292
      },
      "geography": {
        "emitted": 2086,
        "verbatim": 2080
      },
      "acronym": {
        "emitted": 1483,
        "verbatim": 1482
      }
    }
  },
  "examples": [
    {
      "mention": "plant-level data",
      "gold": "{\"geography\": \"Indonesian\"}",
      "pred": "{\"geography\": \"Indonesian\"}",
      "correct": "geography",
      "verbatim": "geography"
    },
    {
      "mention": "GIDD database",
      "gold": "{\"geography\": \"all regions in the world\"}",
      "pred": "{\"acronym\": \"GIDD\", \"geography\": \"all regions in the world\"}",
      "correct": "geography",
      "verbatim": "geography,acronym"
    },
    {
      "mention": "NFHS-3 data",
      "gold": "{\"acronym\": \"NFHS-3\", \"geography\": \"India\"}",
      "pred": "{\"acronym\": \"NFHS-3\", \"geography\": \"India\"}",
      "correct": "geography,acronym",
      "verbatim": "geography,acronym"
    },
    {
      "mention": "data from Hallegatte ( 2012 )",
      "gold": "{\"year\": \"2012\"}",
      "pred": "{\"geography\": \"LICs\", \"year\": \"2012\"}",
      "correct": "year",
      "verbatim": "year,geography"
    },
    {
      "mention": "national accounts",
      "gold": "{\"geography\": \"Turkey\"}",
      "pred": "{\"geography\": \"Turkey\"}",
      "correct": "geography",
      "verbatim": "geography"
    },
    {
      "mention": "SRF registry of firms",
      "gold": "{\"acronym\": \"SRF\", \"geography\": \"Brazil\", \"producer\": \"Brazilian tax authority\", \"year\": \"2012\"}",
      "pred": "{\"acronym\": \"SRF\", \"geography\": \"Brazil\", \"producer\": \"Brazilian tax authority\", \"year\": \"2012\"}",
      "correct": "producer,year,geography,acronym",
      "verbatim": "producer,year,geography,acronym"
    },
    {
      "mention": "Human Capital Index",
      "gold": "{\"geography\": \"Kenya\", \"producer\": \"World Bank\", \"year\": \"2020\"}",
      "pred": "{\"geography\": \"Kenya\", \"producer\": \"World Bank\", \"year\": \"2020\"}",
      "correct": "producer,year,geography",
      "verbatim": "producer,year,geography"
    },
    {
      "mention": "United Nations population counts by age",
      "gold": "{\"producer\": \"United Nations\"}",
      "pred": "{\"producer\": \"United Nations\"}",
      "correct": "producer",
      "verbatim": "producer"
    },
    {
      "mention": "2002 Census",
      "gold": "{\"geography\": \"Rwanda\", \"year\": \"2002\"}",
      "pred": "{\"geography\": \"Rwanda\", \"year\": \"2002\"}",
      "correct": "year,geography",
      "verbatim": "year,geography"
    },
    {
      "mention": "ENCASEH 1997",
      "gold": "{\"acronym\": \"ENCASEH\", \"year\": \"1997\"}",
      "pred": "{\"acronym\": \"ENCASEH\", \"year\": \"1997\"}",
      "correct": "year,acronym",
      "verbatim": "year,acronym"
    },
    {
      "mention": "Agricultural Wages in India",
      "gold": "{\"acronym\": \"AWI\", \"geography\": \"India\", \"producer\": \"Ministry of Agriculture\"}",
      "pred": "{\"acronym\": \"AWI\", \"geography\": \"India\", \"producer\": \"Ministry of Agriculture\"}",
      "correct": "producer,geography,acronym",
      "verbatim": "producer,geography,acronym"
    },
    {
      "mention": "COVID-19 Enterprise Survey",
      "gold": "{\"geography\": \"Chad\", \"year\": \"2020\"}",
      "pred": "{\"acronym\": \"ES\", \"geography\": \"Chad\", \"year\": \"2020\"}",
      "correct": "year,geography",
      "verbatim": "year,geography,acronym"
    }
  ]
}