{
  "real": {
    "n_eval": 14776,
    "overall": {
      "tp": 8778,
      "fp": 2928,
      "fn": 2962,
      "precision": 0.7498718605843158,
      "recall": 0.7477001703577513,
      "f05": 0.7494365138993239,
      "f1": 0.7487844408427877
    },
    "per_key": {
      "producer": {
        "tp": 1909,
        "fp": 968,
        "fn": 914,
        "precision": 0.6635384080639555,
        "recall": 0.6762309599716614,
        "f05": 0.6660386574558649,
        "f1": 0.6698245614035088
      },
      "year": {
        "tp": 2264,
        "fp": 520,
        "fn": 631,
        "precision": 0.8132183908045977,
        "recall": 0.7820379965457686,
        "f05": 0.8067849761242963,
        "f1": 0.7973234724423313
      },
      "geography": {
        "tp": 2632,
        "fp": 984,
        "fn": 968,
        "precision": 0.7278761061946902,
        "recall": 0.7311111111111112,
        "f05": 0.7285208148804252,
        "f1": 0.7294900221729491
      },
      "acronym": {
        "tp": 1973,
        "fp": 456,
        "fn": 449,
        "precision": 0.8122684232194318,
        "recall": 0.814616019818332,
        "f05": 0.8127368594496622,
        "f1": 0.8134405277262421
      }
    },
    "verbatim": {
      "emitted": 11706,
      "verbatim": 11659,
      "rate": 0.9959849649752264,
      "per_key": {
        "producer": {
          "emitted": 2877,
          "verbatim": 2858
        },
        "year": {
          "emitted": 2784,
          "verbatim": 2784
        },
        "geography": {
          "emitted": 3616,
          "verbatim": 3599
        },
        "acronym": {
          "emitted": 2429,
          "verbatim": 2418
        }
      }
    },
    "usage_impact": {
      "data_type_macro_f1": 0.7005048753198976,
      "usage_action_macro_f1": 0.6554475216890102,
      "impact_label_macro_f1": 0.5842432103101727,
      "per_label": {
        "data_type": {
          "administrative": {
            "tp": 1595,
            "fp": 754,
            "fn": 381,
            "precision": 0.6790123456790124,
            "recall": 0.8071862348178138,
            "f1": 0.7375722543352602
          },
          "database": {
            "tp": 855,
            "fp": 248,
            "fn": 616,
            "precision": 0.7751586582048957,
            "recall": 0.5812372535690007,
            "f1": 0.6643356643356644
          },
          "geospatial": {
            "tp": 264,
            "fp": 87,
            "fn": 84,
            "precision": 0.7521367521367521,
            "recall": 0.7586206896551724,
            "f1": 0.7553648068669527
          },
          "indicator": {
            "tp": 383,
            "fp": 328,
            "fn": 136,
            "precision": 0.5386779184247539,
            "recall": 0.7379576107899807,
            "f1": 0.6227642276422765
          },
          "other": {
            "tp": 341,
            "fp": 263,
            "fn": 360,
            "precision": 0.5645695364238411,
            "recall": 0.4864479315263909,
            "f1": 0.5226053639846744
          },
          "report": {
            "tp": 538,
            "fp": 182,
            "fn": 222,
            "precision": 0.7472222222222222,
            "recall": 0.7078947368421052,
            "f1": 0.727027027027027
          },
          "survey": {
            "tp": 2165,
            "fp": 281,
            "fn": 344,
            "precision": 0.8851185609157809,
            "recall": 0.862893583100837,
            "f1": 0.8738647830474269
          }
        },
        "usage_action": {
          "analyze": {
            "tp": 1646,
            "fp": 803,
            "fn": 732,
            "precision": 0.6721110657411188,
            "recall": 0.6921783010933558,
            "f1": 0.6819970996478143
          },
          "curate": {
            "tp": 1634,
            "fp": 647,
            "fn": 414,
            "precision": 0.716352476983779,
            "recall": 0.7978515625,
            "f1": 0.7549087549087549
          },
          "inform": {
            "tp": 922,
            "fp": 566,
            "fn": 669,
            "precision": 0.6196236559139785,
            "recall": 0.57950974230044,
            "f1": 0.598895745371874
          },
          "source": {
            "tp": 1232,
            "fp": 469,
            "fn": 435,
            "precision": 0.7242798353909465,
            "recall": 0.7390521895620876,
            "f1": 0.7315914489311164
          },
          "validate": {
            "tp": 246,
            "fp": 119,
            "fn": 354,
            "precision": 0.673972602739726,
            "recall": 0.41,
            "f1": 0.5098445595854921
          }
        },
        "impact_label": {
          "evidence-base": {
            "tp": 3123,
            "fp": 963,
            "fn": 785,
            "precision": 0.76431718061674,
            "recall": 0.7991299897645855,
            "f1": 0.7813360020015012
          },
          "none": {
            "tp": 1408,
            "fp": 564,
            "fn": 597,
            "precision": 0.7139959432048681,
            "recall": 0.7022443890274315,
            "f1": 0.7080714106110133
          },
          "policy": {
            "tp": 1453,
            "fp": 712,
            "fn": 674,
            "precision": 0.6711316397228637,
            "recall": 0.6831217677480019,
            "f1": 0.6770736253494875
          },
          "uptake": {
            "tp": 26,
            "fp": 35,
            "fn": 218,
            "precision": 0.4262295081967213,
            "recall": 0.10655737704918032,
            "f1": 0.17049180327868851
          }
        }
      },
      "usage_summary": {
        "n": 8284,
        "mean_sim": 0.5522229075431824,
        "grounded_rate": 0.6512554321583776
      }
    },
    "examples": [
      {
        "mention": "ILO ' s Yearbook of Labor Statistics",
        "gold": "{\"geography\": \"Kenya\", \"producer\": \"ILO\", \"year\": \"1995\"}",
        "pred": "{\"geography\": \"Kenya\", \"producer\": \"ILO\", \"year\": \"1995\"}",
        "correct": "producer,year,geography",
        "verbatim": "producer,year,geography"
      },
      {
        "mention": "BEEPS survey",
        "gold": "{\"acronym\": \"BEEPS\", \"year\": \"2008\"}",
        "pred": "{\"acronym\": \"BEEPS\", \"year\": \"2008\"}",
        "correct": "year,acronym",
        "verbatim": "year,acronym"
      },
      {
        "mention": "ENCOVI data",
        "gold": "{\"geography\": \"Venezuela\"}",
        "pred": "{\"acronym\": \"ENCOVI\", \"geography\": \"Colombia\"}",
        "correct": "-",
        "verbatim": "geography,acronym"
      },
      {
        "mention": "panel data",
        "gold": "{\"geography\": \"Egypt\"}",
        "pred": "{\"geography\": \"Egypt\"}",
        "correct": "geography",
        "verbatim": "geography"
      },
      {
        "mention": "WHO Coronavirus Disease (COVID-19) Dashboard",
        "gold": "{\"producer\": \"WHO\", \"geography\": \"worldwide\", \"acronym\": \"COVID-19\"}",
        "pred": "{\"producer\": \"WHO\", \"geography\": \"Uganda\"}",
        "correct": "producer",
        "verbatim": "producer,geography"
      },
      {
        "mention": "Integrated Household Panel Surveys",
        "gold": "{\"acronym\": \"IHPS\", \"geography\": \"Malawi\", \"year\": \"2013\"}",
        "pred": "{\"acronym\": \"IHPS\", \"geography\": \"Malawi\"}",
        "correct": "geography,acronym",
        "verbatim": "geography,acronym"
      },
      {
        "mention": "socio-economic survey",
        "gold": "{\"year\": \"2010\"}",
        "pred": "{\"year\": \"2010\"}",
        "correct": "year",
        "verbatim": "year"
      },
      {
        "mention": "World Bank Enterprise Survey Data for Uganda",
        "gold": "{\"producer\": \"World Bank\", \"year\": \"2013\", \"geography\": \"Uganda\"}",
        "pred": "{\"producer\": \"World Bank\", \"year\": \"2013\", \"geography\": \"Uganda\"}",
        "correct": "producer,year,geography",
        "verbatim": "producer,year,geography"
      },
      {
        "mention": "internally displaced person (IDP) data",
        "gold": "{\"producer\": \"Internal Displacement Monitoring Centre\", \"geography\": \"Philippines\", \"acronym\": \"IDMC\"}",
        "pred": "{\"producer\": \"Internal Displacement Monitoring Centre\", \"year\": \"2020\", \"geography\": \"Philippines\", \"acronym\": \"IDMC\"}",
        "correct": "producer,geography,acronym",
        "verbatim": "producer,year,geography,acronym"
      },
      {
        "mention": "Displacement data",
        "gold": "{\"producer\": \"Internal Displacement Monitoring Centre\"}",
        "pred": "{\"producer\": \"Internal Displacement Monitoring Centre\"}",
        "correct": "producer",
        "verbatim": "producer"
      },
      {
        "mention": "regulatory surveys",
        "gold": "{\"geography\": \"East Africa\", \"producer\": \"World Bank\"}",
        "pred": "{\"geography\": \"Kenya\", \"producer\": \"local consultants\"}",
        "correct": "-",
        "verbatim": "producer,geography"
      },
      {
        "mention": "UNHCR archives",
        "gold": "{\"producer\": \"UNHCR\"}",
        "pred": "{\"producer\": \"UNHCR\"}",
        "correct": "producer",
        "verbatim": "producer"
      }
    ]
  },
  "synthetic": {
    "n_eval": 2305,
    "overall": {
      "tp": 985,
      "fp": 236,
      "fn": 185,
      "precision": 0.8067158067158067,
      "recall": 0.8418803418803419,
      "f05": 0.8135117277832838,
      "f1": 0.8239230447511501
    },
    "per_key": {
      "producer": {
        "tp": 231,
        "fp": 39,
        "fn": 56,
        "precision": 0.8555555555555555,
        "recall": 0.8048780487804879,
        "f05": 0.8449158741770301,
        "f1": 0.829443447037702
      },
      "year": {
        "tp": 265,
        "fp": 50,
        "fn": 40,
        "precision": 0.8412698412698413,
        "recall": 0.8688524590163934,
        "f05": 0.8466453674121406,
        "f1": 0.8548387096774194
      },
      "geography": {
        "tp": 344,
        "fp": 63,
        "fn": 53,
        "precision": 0.8452088452088452,
        "recall": 0.8664987405541562,
        "f05": 0.8493827160493826,
        "f1": 0.855721393034826
      },
      "acronym": {
        "tp": 145,
        "fp": 84,
        "fn": 36,
        "precision": 0.6331877729257642,
        "recall": 0.8011049723756906,
        "f05": 0.6608933454876937,
        "f1": 0.7073170731707317
      }
    },
    "verbatim": {
      "emitted": 1221,
      "verbatim": 1216,
      "rate": 0.9959049959049959,
      "per_key": {
        "producer": {
          "emitted": 270,
          "verbatim": 270
        },
        "year": {
          "emitted": 315,
          "verbatim": 315
        },
        "geography": {
          "emitted": 407,
          "verbatim": 402
        },
        "acronym": {
          "emitted": 229,
          "verbatim": 229
        }
      }
    },
    "usage_impact": {
      "data_type_macro_f1": 0.7818281325560671,
      "usage_action_macro_f1": 0.6488552648711132,
      "impact_label_macro_f1": 0.5147452129823803,
      "per_label": {
        "data_type": {
          "administrative": {
            "tp": 232,
            "fp": 48,
            "fn": 46,
            "precision": 0.8285714285714286,
            "recall": 0.8345323741007195,
            "f1": 0.8315412186379929
          },
          "database": {
            "tp": 170,
            "fp": 33,
            "fn": 89,
            "precision": 0.8374384236453202,
            "recall": 0.6563706563706564,
            "f1": 0.7359307359307359
          },
          "geospatial": {
            "tp": 106,
            "fp": 33,
            "fn": 11,
            "precision": 0.762589928057554,
            "recall": 0.905982905982906,
            "f1": 0.828125
          },
          "indicator": {
            "tp": 138,
            "fp": 59,
            "fn": 16,
            "precision": 0.700507614213198,
            "recall": 0.8961038961038961,
            "f1": 0.7863247863247862
          },
          "other": {
            "tp": 115,
            "fp": 61,
            "fn": 55,
            "precision": 0.6534090909090909,
            "recall": 0.6764705882352942,
            "f1": 0.6647398843930635
          },
          "report": {
            "tp": 27,
            "fp": 5,
            "fn": 18,
            "precision": 0.84375,
            "recall": 0.6,
            "f1": 0.7012987012987012
          },
          "survey": {
            "tp": 566,
            "fp": 44,
            "fn": 48,
            "precision": 0.9278688524590164,
            "recall": 0.9218241042345277,
            "f1": 0.9248366013071895
          }
        },
        "usage_action": {
          "analyze": {
            "tp": 77,
            "fp": 11,
            "fn": 7,
            "precision": 0.875,
            "recall": 0.9166666666666666,
            "f1": 0.8953488372093024
          },
          "causal": {
            "tp": 59,
            "fp": 22,
            "fn": 49,
            "precision": 0.7283950617283951,
            "recall": 0.5462962962962963,
            "f1": 0.6243386243386243
          },
          "curate": {
            "tp": 34,
            "fp": 6,
            "fn": 6,
            "precision": 0.85,
            "recall": 0.85,
            "f1": 0.85
          },
          "inform": {
            "tp": 140,
            "fp": 77,
            "fn": 49,
            "precision": 0.6451612903225806,
            "recall": 0.7407407407407407,
            "f1": 0.689655172413793
          },
          "measure": {
            "tp": 169,
            "fp": 119,
            "fn": 101,
            "precision": 0.5868055555555556,
            "recall": 0.6259259259259259,
            "f1": 0.6057347670250895
          },
          "model": {
            "tp": 79,
            "fp": 36,
            "fn": 79,
            "precision": 0.6869565217391305,
            "recall": 0.5,
            "f1": 0.5787545787545788
          },
          "other": {
            "tp": 28,
            "fp": 24,
            "fn": 33,
            "precision": 0.5384615384615384,
            "recall": 0.45901639344262296,
            "f1": 0.4955752212389381
          },
          "produce": {
            "tp": 75,
            "fp": 74,
            "fn": 43,
            "precision": 0.5033557046979866,
            "recall": 0.635593220338983,
            "f1": 0.5617977528089888
          },
          "source": {
            "tp": 242,
            "fp": 126,
            "fn": 155,
            "precision": 0.657608695652174,
            "recall": 0.6095717884130982,
            "f1": 0.6326797385620916
          },
          "validate": {
            "tp": 103,
            "fp": 47,
            "fn": 57,
            "precision": 0.6866666666666666,
            "recall": 0.64375,
            "f1": 0.664516129032258
          },
          "wrangle": {
            "tp": 38,
            "fp": 51,
            "fn": 14,
            "precision": 0.42696629213483145,
            "recall": 0.7307692307692307,
            "f1": 0.5390070921985815
          }
        },
        "impact_label": {
          "evidence-base": {
            "tp": 1000,
            "fp": 287,
            "fn": 28,
            "precision": 0.777000777000777,
            "recall": 0.9727626459143969,
            "f1": 0.8639308855291578
          },
          "market-use": {
            "tp": 0,
            "fp": 0,
            "fn": 8,
            "precision": 0.0,
            "recall": 0.0,
            "f1": 0.0
          },
          "none": {
            "tp": 43,
            "fp": 21,
            "fn": 53,
            "precision": 0.671875,
            "recall": 0.4479166666666667,
            "f1": 0.5375
          },
          "operational-use": {
            "tp": 8,
            "fp": 11,
            "fn": 58,
            "precision": 0.42105263157894735,
            "recall": 0.12121212121212122,
            "f1": 0.18823529411764706
          },
          "policy": {
            "tp": 163,
            "fp": 11,
            "fn": 201,
            "precision": 0.9367816091954023,
            "recall": 0.4478021978021978,
            "f1": 0.6059479553903345
          },
          "uptake": {
            "tp": 75,
            "fp": 18,
            "fn": 0,
            "precision": 0.8064516129032258,
            "recall": 1.0,
            "f1": 0.8928571428571428
          }
        }
      },
      "usage_summary": {
        "n": 1637,
        "mean_sim": 0.5752291679382324,
        "grounded_rate": 0.6768478924862553
      }
    },
    "examples": [
      {
        "mention": "C8te d ' Ivoire data set",
        "gold": "{\"geography\": \"C8te d ' Ivoire\"}",
        "pred": "{\"geography\": \"C8te d ' Ivoire\"}",
        "correct": "geography",
        "verbatim": "geography"
      },
      {
        "mention": "Access to Finance Survey",
        "gold": "{\"acronym\": \"Access to Finance Survey\", \"geography\": \"Nigeria\", \"year\": \"2008\"}",
        "pred": "{\"acronym\": \"Access to Finance\", \"geography\": \"Nigeria\", \"year\": \"2008\"}",
        "correct": "year,geography",
        "verbatim": "year,geography,acronym"
      },
      {
        "mention": "Data from Mongolia and Myanmar",
        "gold": "{\"geography\": \"Mongolia and Myanmar\"}",
        "pred": "{\"geography\": \"Mongolia and Myanmar\"}",
        "correct": "geography",
        "verbatim": "geography"
      },
      {
        "mention": "HIES 2016 / 17",
        "gold": "{\"geography\": \"Bangladesh\", \"year\": \"2016\"}",
        "pred": "{\"acronym\": \"HIES\", \"geography\": \"Bangladesh\", \"year\": \"2016\"}",
        "correct": "year,geography",
        "verbatim": "year,geography,acronym"
      },
      {
        "mention": "survey of households",
        "gold": "{\"geography\": \"Nicaraguan\"}",
        "pred": "{\"geography\": \"Nicaraguan\", \"year\": \"1998\"}",
        "correct": "geography",
        "verbatim": "year,geography"
      },
      {
        "mention": "National Annual Industrial Survey",
        "gold": "{\"geography\": \"Chile\", \"year\": \"2001\"}",
        "pred": "{\"geography\": \"Chile\", \"year\": \"2001\"}",
        "correct": "year,geography",
        "verbatim": "year,geography"
      },
      {
        "mention": "1985 Living Standards Survey",
        "gold": "{\"acronym\": \"CILSS\", \"geography\": \"Côte d'Ivoire\", \"year\": \"1985\"}",
        "pred": "{\"acronym\": \"CILSS\", \"geography\": \"Côte d'Ivoire\", \"year\": \"1985\"}",
        "correct": "year,geography,acronym",
        "verbatim": "year,geography,acronym"
      },
      {
        "mention": "World Bank Survey on Applied Policies in Services",
        "gold": "{\"producer\": \"The World Bank Research Department\", \"year\": \"2008\"}",
        "pred": "{\"producer\": \"The World Bank Research Department\", \"year\": \"2008\"}",
        "correct": "producer,year",
        "verbatim": "producer,year"
      },
      {
        "mention": "Pesquisa de Orçamentos Familiares",
        "gold": "{\"acronym\": \"POF\", \"geography\": \"Brazil\"}",
        "pred": "{\"acronym\": \"POF\", \"geography\": \"Brazil\"}",
        "correct": "geography,acronym",
        "verbatim": "geography,acronym"
      },
      {
        "mention": "administrative data",
        "gold": "{\"year\": \"2015\"}",
        "pred": "{\"year\": \"2015\"}",
        "correct": "year",
        "verbatim": "year"
      },
      {
        "mention": "Bangladesh Return Migrant Survey",
        "gold": "{\"acronym\": \"( BRMS )\", \"geography\": \"Bangladesh\", \"producer\": \"World Bank\", \"year\": \"2018 / 19\"}",
        "pred": "{\"acronym\": \"BRMS\", \"geography\": \"Bangladesh\", \"producer\": \"World Bank\", \"year\": \"2018\"}",
        "correct": "producer,geography",
        "verbatim": "producer,year,geography,acronym"
      },
      {
        "mention": "ACLED data",
        "gold": "{\"acronym\": \"ACLED\", \"geography\": \"Novosilka city in Donetsk oblast\", \"producer\": \"Armed Conflict Location and Event Data\", \"year\": \"2022\"}",
        "pred": "{\"acronym\": \"ACLED\", \"geography\": \"Novosilka city\", \"year\": \"2022\"}",
        "correct": "year,acronym",
        "verbatim": "year,geography,acronym"
      }
    ]
  }
}