Hugging Face's logo

robbalian
/
rev-qwen3.5-4b

Text Classification
English
lora
pointer-head
decision-model
qwen3.5
rev
Model card Files
xet
 Community
rev-qwen3.5-4b / trained_development.json
robbalian's picture
robbalian
Publish jb_20260923-002700_4b
6c07755 verified
Raw History Blame Contribute Delete
298 kB
{
 "accuracy": 0.8769230769230769,
 "nll": 0.41225560257177074,
 "by_source": {
 "socialiqa": [
 166,
 200
 ],
 "cosmosqa": [
 181,
 200
 ],
 "race": [
 177,
 200
 ],
 "multirc": [
 175,
 200
 ],
 "contractnli": [
 156,
 175
 ]
 },
 "seconds": 15.43062357000008,
 "answers": [
 {
 "id": "socialiqa/validation/1651",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0003437305858824402,
 "2": 0.9662266969680786,
 "3": 0.03342962637543678
 }
 },
 {
 "id": "socialiqa/validation/429",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 1.1293941497569904e-05,
 "2": 0.9999417066574097,
 "3": 4.699360215454362e-05
 }
 },
 {
 "id": "socialiqa/validation/976",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.049021460115909576,
 "2": 9.033065907715354e-06,
 "3": 0.9509694576263428
 }
 },
 {
 "id": "socialiqa/validation/639",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0005730170523747802,
 "2": 0.00028223279514349997,
 "3": 0.9991446733474731
 }
 },
 {
 "id": "socialiqa/validation/1142",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00014808627020101994,
 "2": 0.9962249994277954,
 "3": 0.0036269568372517824
 }
 },
 {
 "id": "socialiqa/validation/1784",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.007721150293946266,
 "2": 0.006718126591295004,
 "3": 0.9855607151985168
 }
 },
 {
 "id": "socialiqa/validation/783",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.05290420725941658,
 "2": 0.029265062883496284,
 "3": 0.917830765247345
 }
 },
 {
 "id": "socialiqa/validation/803",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.4902803897857666,
 "2": 0.11067654937505722,
 "3": 0.39904308319091797
 }
 },
 {
 "id": "socialiqa/validation/674",
 "source": "socialiqa",
 "expected": "3",
 "choice": "2",
 "probabilities": {
 "1": 0.0008843819377943873,
 "2": 0.5093563795089722,
 "3": 0.4897592067718506
 }
 },
 {
 "id": "socialiqa/validation/1011",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.5776129961013794,
 "2": 0.00016041223716456443,
 "3": 0.42222654819488525
 }
 },
 {
 "id": "socialiqa/validation/1396",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.03282792866230011,
 "2": 0.9297998547554016,
 "3": 0.03737219423055649
 }
 },
 {
 "id": "socialiqa/validation/1697",
 "source": "socialiqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "1": 0.05222958326339722,
 "2": 0.9469363689422607,
 "3": 0.0008340091444551945
 }
 },
 {
 "id": "socialiqa/validation/1318",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.009695399552583694,
 "2": 0.9810448884963989,
 "3": 0.009259641170501709
 }
 },
 {
 "id": "socialiqa/validation/1701",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00023028053692542017,
 "2": 0.00014086435839999467,
 "3": 0.9996289014816284
 }
 },
 {
 "id": "socialiqa/validation/1687",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.004642904736101627,
 "2": 0.9913495779037476,
 "3": 0.004007481038570404
 }
 },
 {
 "id": "socialiqa/validation/1486",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.005323007702827454,
 "2": 0.0008509699837304652,
 "3": 0.9938259720802307
 }
 },
 {
 "id": "socialiqa/validation/1229",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.9703510403633118,
 "2": 0.017853835597634315,
 "3": 0.011795106343925
 }
 },
 {
 "id": "socialiqa/validation/338",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9988932013511658,
 "2": 7.279691635631025e-05,
 "3": 0.001034033834002912
 }
 },
 {
 "id": "socialiqa/validation/1851",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9836481213569641,
 "2": 0.0012980186147615314,
 "3": 0.015053798444569111
 }
 },
 {
 "id": "socialiqa/validation/151",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.24243538081645966,
 "2": 0.7033284902572632,
 "3": 0.05423611029982567
 }
 },
 {
 "id": "socialiqa/validation/1782",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.3429940342903137,
 "2": 0.6569627523422241,
 "3": 4.3246276618447155e-05
 }
 },
 {
 "id": "socialiqa/validation/577",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.3707544505596161,
 "2": 0.000140237549203448,
 "3": 0.6291053295135498
 }
 },
 {
 "id": "socialiqa/validation/1302",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9956074357032776,
 "2": 0.0002948548353742808,
 "3": 0.004097737837582827
 }
 },
 {
 "id": "socialiqa/validation/907",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.6650920510292053,
 "2": 0.022942140698432922,
 "3": 0.31196579337120056
 }
 },
 {
 "id": "socialiqa/validation/329",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00013348560605663806,
 "2": 0.9997459053993225,
 "3": 0.0001206363012897782
 }
 },
 {
 "id": "socialiqa/validation/300",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 2.622254760353826e-05,
 "2": 0.9998666048049927,
 "3": 0.00010717644909163937
 }
 },
 {
 "id": "socialiqa/validation/759",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9942078590393066,
 "2": 0.005765409208834171,
 "3": 2.668554770934861e-05
 }
 },
 {
 "id": "socialiqa/validation/1666",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9199962615966797,
 "2": 0.023278290405869484,
 "3": 0.05672542378306389
 }
 },
 {
 "id": "socialiqa/validation/1311",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0004315116675570607,
 "2": 0.9993757605552673,
 "3": 0.00019281223649159074
 }
 },
 {
 "id": "socialiqa/validation/161",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.07405336946249008,
 "2": 0.9068818092346191,
 "3": 0.01906474493443966
 }
 },
 {
 "id": "socialiqa/validation/529",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0034705072175711393,
 "2": 0.0010591845493763685,
 "3": 0.9954702854156494
 }
 },
 {
 "id": "socialiqa/validation/1676",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9996469020843506,
 "2": 0.00023900189262349159,
 "3": 0.00011409815488150343
 }
 },
 {
 "id": "socialiqa/validation/1693",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.6488336324691772,
 "2": 0.017885439097881317,
 "3": 0.33328089118003845
 }
 },
 {
 "id": "socialiqa/validation/930",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 2.9200089102232596e-06,
 "2": 0.9999611377716064,
 "3": 3.596184615162201e-05
 }
 },
 {
 "id": "socialiqa/validation/153",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.009601807221770287,
 "2": 0.041574981063604355,
 "3": 0.9488232731819153
 }
 },
 {
 "id": "socialiqa/validation/748",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00336927967146039,
 "2": 4.2642990592867136e-05,
 "3": 0.9965881109237671
 }
 },
 {
 "id": "socialiqa/validation/548",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.04559122025966644,
 "2": 0.9101112484931946,
 "3": 0.04429755359888077
 }
 },
 {
 "id": "socialiqa/validation/1451",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00014939230459276587,
 "2": 0.000647153181489557,
 "3": 0.9992035031318665
 }
 },
 {
 "id": "socialiqa/validation/1045",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.41917046904563904,
 "2": 0.0021602315828204155,
 "3": 0.5786692500114441
 }
 },
 {
 "id": "socialiqa/validation/266",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.8843205571174622,
 "2": 0.07059546560049057,
 "3": 0.04508400335907936
 }
 },
 {
 "id": "socialiqa/validation/1464",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.16053402423858643,
 "2": 0.005870827008038759,
 "3": 0.8335951566696167
 }
 },
 {
 "id": "socialiqa/validation/1150",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.03764539957046509,
 "2": 0.0014855479821562767,
 "3": 0.9608690738677979
 }
 },
 {
 "id": "socialiqa/validation/1395",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9996623992919922,
 "2": 0.00014099273539613932,
 "3": 0.00019660062389448285
 }
 },
 {
 "id": "socialiqa/validation/170",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9967404007911682,
 "2": 0.002634716685861349,
 "3": 0.0006249042926356196
 }
 },
 {
 "id": "socialiqa/validation/60",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.015282252803444862,
 "2": 0.9758029580116272,
 "3": 0.008914805948734283
 }
 },
 {
 "id": "socialiqa/validation/14",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.002714692149311304,
 "2": 0.0004360657185316086,
 "3": 0.9968492388725281
 }
 },
 {
 "id": "socialiqa/validation/271",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.005805446300655603,
 "2": 0.9940405488014221,
 "3": 0.00015400140546262264
 }
 },
 {
 "id": "socialiqa/validation/1262",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00016819764277897775,
 "2": 4.0099799662129954e-05,
 "3": 0.9997918009757996
 }
 },
 {
 "id": "socialiqa/validation/12",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.9035963416099548,
 "2": 0.0912371277809143,
 "3": 0.005166494287550449
 }
 },
 {
 "id": "socialiqa/validation/1355",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 1.4651897117801127e-06,
 "2": 0.0003197516780346632,
 "3": 0.9996788501739502
 }
 },
 {
 "id": "socialiqa/validation/831",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9990725517272949,
 "2": 0.0007729033823125064,
 "3": 0.0001544895931147039
 }
 },
 {
 "id": "socialiqa/validation/765",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.5718927979469299,
 "2": 0.2881658375263214,
 "3": 0.13994131982326508
 }
 },
 {
 "id": "socialiqa/validation/1129",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.004286591429263353,
 "2": 0.9873824715614319,
 "3": 0.008330904878675938
 }
 },
 {
 "id": "socialiqa/validation/665",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0022686917800456285,
 "2": 0.04949183389544487,
 "3": 0.9482394456863403
 }
 },
 {
 "id": "socialiqa/validation/457",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9375256299972534,
 "2": 0.053434040397405624,
 "3": 0.009040254168212414
 }
 },
 {
 "id": "socialiqa/validation/245",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.0714871734380722,
 "2": 0.04030457139015198,
 "3": 0.8882082104682922
 }
 },
 {
 "id": "socialiqa/validation/1314",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.7559061050415039,
 "2": 0.24216869473457336,
 "3": 0.0019251967314630747
 }
 },
 {
 "id": "socialiqa/validation/1735",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 3.309903604531428e-06,
 "2": 0.9083107709884644,
 "3": 0.09168589115142822
 }
 },
 {
 "id": "socialiqa/validation/868",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.42622998356819153,
 "2": 0.5732235908508301,
 "3": 0.0005464015412144363
 }
 },
 {
 "id": "socialiqa/validation/1450",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.37916117906570435,
 "2": 0.4017745852470398,
 "3": 0.21906420588493347
 }
 },
 {
 "id": "socialiqa/validation/1258",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.5357338786125183,
 "2": 0.3010312020778656,
 "3": 0.16323482990264893
 }
 },
 {
 "id": "socialiqa/validation/1788",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 4.400108991831075e-06,
 "2": 1.3701879879590706e-06,
 "3": 0.9999942779541016
 }
 },
 {
 "id": "socialiqa/validation/1319",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 4.554451152216643e-06,
 "2": 4.419675406097667e-06,
 "3": 0.9999910593032837
 }
 },
 {
 "id": "socialiqa/validation/1276",
 "source": "socialiqa",
 "expected": "3",
 "choice": "2",
 "probabilities": {
 "1": 8.98054895515088e-06,
 "2": 0.9882813096046448,
 "3": 0.011709701269865036
 }
 },
 {
 "id": "socialiqa/validation/1664",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.016601363196969032,
 "2": 0.016642268747091293,
 "3": 0.9667564034461975
 }
 },
 {
 "id": "socialiqa/validation/65",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.0036391073372215033,
 "2": 0.37763121724128723,
 "3": 0.6187296509742737
 }
 },
 {
 "id": "socialiqa/validation/1723",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.4725148379802704,
 "2": 0.03473830968141556,
 "3": 0.49274682998657227
 }
 },
 {
 "id": "socialiqa/validation/817",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.5651370882987976,
 "2": 0.4346194565296173,
 "3": 0.0002434618945699185
 }
 },
 {
 "id": "socialiqa/validation/722",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 3.551109102772898e-06,
 "2": 6.33593617749284e-06,
 "3": 0.9999901056289673
 }
 },
 {
 "id": "socialiqa/validation/823",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.08661805093288422,
 "2": 0.9117893576622009,
 "3": 0.0015926087507978082
 }
 },
 {
 "id": "socialiqa/validation/1116",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00013225857401266694,
 "2": 0.9997044205665588,
 "3": 0.00016328092897310853
 }
 },
 {
 "id": "socialiqa/validation/150",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9447717070579529,
 "2": 0.0059843119233846664,
 "3": 0.04924394562840462
 }
 },
 {
 "id": "socialiqa/validation/380",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.997012734413147,
 "2": 0.00035251289955340326,
 "3": 0.0026347502134740353
 }
 },
 {
 "id": "socialiqa/validation/1564",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00021750348969362676,
 "2": 0.0018036369001492858,
 "3": 0.9979788661003113
 }
 },
 {
 "id": "socialiqa/validation/1287",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9823840856552124,
 "2": 0.013832501135766506,
 "3": 0.0037834495306015015
 }
 },
 {
 "id": "socialiqa/validation/1408",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 5.52740202692803e-05,
 "2": 0.0005978893022984266,
 "3": 0.9993467926979065
 }
 },
 {
 "id": "socialiqa/validation/1740",
 "source": "socialiqa",
 "expected": "3",
 "choice": "2",
 "probabilities": {
 "1": 0.0008579677087254822,
 "2": 0.9788861870765686,
 "3": 0.02025585062801838
 }
 },
 {
 "id": "socialiqa/validation/945",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0028544869273900986,
 "2": 0.0011127734323963523,
 "3": 0.9960326552391052
 }
 },
 {
 "id": "socialiqa/validation/1340",
 "source": "socialiqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "1": 0.007875432260334492,
 "2": 0.26930874586105347,
 "3": 0.7228158116340637
 }
 },
 {
 "id": "socialiqa/validation/195",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9887853860855103,
 "2": 0.006927188020199537,
 "3": 0.0042873769998550415
 }
 },
 {
 "id": "socialiqa/validation/1101",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.8187103867530823,
 "2": 0.10386520624160767,
 "3": 0.07742442935705185
 }
 },
 {
 "id": "socialiqa/validation/1798",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9718475341796875,
 "2": 0.012815853580832481,
 "3": 0.015336710028350353
 }
 },
 {
 "id": "socialiqa/validation/644",
 "source": "socialiqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "1": 0.01695762388408184,
 "2": 0.06346116960048676,
 "3": 0.9195811748504639
 }
 },
 {
 "id": "socialiqa/validation/1685",
 "source": "socialiqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "1": 0.41409391164779663,
 "2": 0.0011190574150532484,
 "3": 0.5847870111465454
 }
 },
 {
 "id": "socialiqa/validation/1251",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.006287437863647938,
 "2": 0.9918128252029419,
 "3": 0.0018997202860191464
 }
 },
 {
 "id": "socialiqa/validation/263",
 "source": "socialiqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "1": 0.039158258587121964,
 "2": 0.96068274974823,
 "3": 0.00015897999401204288
 }
 },
 {
 "id": "socialiqa/validation/972",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00022111703583505005,
 "2": 0.9997007846832275,
 "3": 7.813688716851175e-05
 }
 },
 {
 "id": "socialiqa/validation/1132",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9972054362297058,
 "2": 0.0017270459793508053,
 "3": 0.001067457371391356
 }
 },
 {
 "id": "socialiqa/validation/1532",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00969073735177517,
 "2": 0.002080368809401989,
 "3": 0.9882289171218872
 }
 },
 {
 "id": "socialiqa/validation/1467",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.010090867057442665,
 "2": 0.9897477626800537,
 "3": 0.00016138560022227466
 }
 },
 {
 "id": "socialiqa/validation/1563",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.9150795936584473,
 "2": 0.026605596765875816,
 "3": 0.05831480026245117
 }
 },
 {
 "id": "socialiqa/validation/1849",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.63193678855896,
 "2": 0.34936267137527466,
 "3": 0.01870052143931389
 }
 },
 {
 "id": "socialiqa/validation/1950",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.005528136156499386,
 "2": 0.988578200340271,
 "3": 0.005893709138035774
 }
 },
 {
 "id": "socialiqa/validation/1216",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.000987718696705997,
 "2": 0.6478844285011292,
 "3": 0.35112789273262024
 }
 },
 {
 "id": "socialiqa/validation/723",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9831944108009338,
 "2": 0.0002572116209194064,
 "3": 0.016548313200473785
 }
 },
 {
 "id": "socialiqa/validation/1642",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00201587681658566,
 "2": 0.001118413987569511,
 "3": 0.9968656897544861
 }
 },
 {
 "id": "socialiqa/validation/81",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9236883521080017,
 "2": 8.745371451368555e-05,
 "3": 0.07622417062520981
 }
 },
 {
 "id": "socialiqa/validation/314",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9791503548622131,
 "2": 0.002278428990393877,
 "3": 0.018571270629763603
 }
 },
 {
 "id": "socialiqa/validation/505",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0007561314851045609,
 "2": 0.966128408908844,
 "3": 0.03311547636985779
 }
 },
 {
 "id": "socialiqa/validation/1414",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 4.4229160266695544e-05,
 "2": 0.3766147196292877,
 "3": 0.6233410835266113
 }
 },
 {
 "id": "socialiqa/validation/1565",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9990280866622925,
 "2": 0.0007030162960290909,
 "3": 0.00026888924185186625
 }
 },
 {
 "id": "socialiqa/validation/1734",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.000666637031827122,
 "2": 0.007183089852333069,
 "3": 0.9921503067016602
 }
 },
 {
 "id": "socialiqa/validation/599",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.7127967476844788,
 "2": 0.002129879780113697,
 "3": 0.28507331013679504
 }
 },
 {
 "id": "socialiqa/validation/1476",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.995059609413147,
 "2": 0.0034886463545262814,
 "3": 0.0014518375974148512
 }
 },
 {
 "id": "socialiqa/validation/360",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0025601463858038187,
 "2": 0.5127121210098267,
 "3": 0.4847277104854584
 }
 },
 {
 "id": "socialiqa/validation/295",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00010245199518976733,
 "2": 0.9994708895683289,
 "3": 0.0004267260374035686
 }
 },
 {
 "id": "socialiqa/validation/773",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.04869259148836136,
 "2": 2.2576450646738522e-05,
 "3": 0.9512848854064941
 }
 },
 {
 "id": "socialiqa/validation/234",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9995763897895813,
 "2": 0.0003965378273278475,
 "3": 2.7078678613179363e-05
 }
 },
 {
 "id": "socialiqa/validation/265",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.028093023225665092,
 "2": 0.7685547471046448,
 "3": 0.20335222780704498
 }
 },
 {
 "id": "socialiqa/validation/1622",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.12361543625593185,
 "2": 0.8763286471366882,
 "3": 5.586445331573486e-05
 }
 },
 {
 "id": "socialiqa/validation/412",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9998822212219238,
 "2": 9.147606033366174e-05,
 "3": 2.6390242055640556e-05
 }
 },
 {
 "id": "socialiqa/validation/126",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9996343851089478,
 "2": 0.0003579139884095639,
 "3": 7.730230208835565e-06
 }
 },
 {
 "id": "socialiqa/validation/91",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.8223695755004883,
 "2": 0.17746677994728088,
 "3": 0.0001636497472645715
 }
 },
 {
 "id": "socialiqa/validation/1795",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.19344089925289154,
 "2": 0.8035779595375061,
 "3": 0.002981118857860565
 }
 },
 {
 "id": "socialiqa/validation/1465",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9996922016143799,
 "2": 4.034189259982668e-05,
 "3": 0.00026741105830296874
 }
 },
 {
 "id": "socialiqa/validation/1385",
 "source": "socialiqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "1": 0.019941972568631172,
 "2": 0.9799758791923523,
 "3": 8.215614798245952e-05
 }
 },
 {
 "id": "socialiqa/validation/382",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9910756349563599,
 "2": 0.0037025667261332273,
 "3": 0.005221765022724867
 }
 },
 {
 "id": "socialiqa/validation/228",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.3035449683666229,
 "2": 0.12547072768211365,
 "3": 0.5709843635559082
 }
 },
 {
 "id": "socialiqa/validation/888",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.05631301924586296,
 "2": 0.5776437520980835,
 "3": 0.3660432994365692
 }
 },
 {
 "id": "socialiqa/validation/794",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9493699073791504,
 "2": 0.04472773149609566,
 "3": 0.005902393721044064
 }
 },
 {
 "id": "socialiqa/validation/1535",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9896695017814636,
 "2": 0.010155397467315197,
 "3": 0.00017508155724499375
 }
 },
 {
 "id": "socialiqa/validation/1176",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 2.6911966415354982e-05,
 "2": 0.9999028444290161,
 "3": 7.025070954114199e-05
 }
 },
 {
 "id": "socialiqa/validation/293",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9990662932395935,
 "2": 0.0008100505219772458,
 "3": 0.00012361716653686017
 }
 },
 {
 "id": "socialiqa/validation/739",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0001285616890527308,
 "2": 4.903650278720306e-06,
 "3": 0.9998664855957031
 }
 },
 {
 "id": "socialiqa/validation/1028",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00027408136520534754,
 "2": 0.900262176990509,
 "3": 0.09946370124816895
 }
 },
 {
 "id": "socialiqa/validation/507",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.23323558270931244,
 "2": 0.004305093549191952,
 "3": 0.7624592781066895
 }
 },
 {
 "id": "socialiqa/validation/489",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.8219078183174133,
 "2": 0.09717880189418793,
 "3": 0.08091343939304352
 }
 },
 {
 "id": "socialiqa/validation/1298",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.023606790229678154,
 "2": 2.6021150915767066e-05,
 "3": 0.976367175579071
 }
 },
 {
 "id": "socialiqa/validation/606",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.5069268941879272,
 "2": 0.0032869449350982904,
 "3": 0.48978620767593384
 }
 },
 {
 "id": "socialiqa/validation/574",
 "source": "socialiqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "1": 0.00020652584498748183,
 "2": 0.006058983970433474,
 "3": 0.9937345385551453
 }
 },
 {
 "id": "socialiqa/validation/366",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.50062096118927,
 "2": 0.0011450343299657106,
 "3": 0.4982339143753052
 }
 },
 {
 "id": "socialiqa/validation/67",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0005758192855864763,
 "2": 0.9892758131027222,
 "3": 0.010148380883038044
 }
 },
 {
 "id": "socialiqa/validation/640",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.013627659529447556,
 "2": 0.9681981205940247,
 "3": 0.01817415840923786
 }
 },
 {
 "id": "socialiqa/validation/965",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.6677587032318115,
 "2": 0.0024742349050939083,
 "3": 0.32976704835891724
 }
 },
 {
 "id": "socialiqa/validation/1524",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9478731155395508,
 "2": 0.0028129490092396736,
 "3": 0.049313969910144806
 }
 },
 {
 "id": "socialiqa/validation/557",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9770053625106812,
 "2": 0.012477868236601353,
 "3": 0.010516837239265442
 }
 },
 {
 "id": "socialiqa/validation/356",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9920686483383179,
 "2": 0.007661138195544481,
 "3": 0.0002702061028685421
 }
 },
 {
 "id": "socialiqa/validation/1144",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 2.78817842627177e-05,
 "2": 0.9998859167098999,
 "3": 8.624422480352223e-05
 }
 },
 {
 "id": "socialiqa/validation/1646",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.002690724330022931,
 "2": 0.00394989550113678,
 "3": 0.993359386920929
 }
 },
 {
 "id": "socialiqa/validation/778",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.001825198414735496,
 "2": 0.05352291837334633,
 "3": 0.9446519017219543
 }
 },
 {
 "id": "socialiqa/validation/496",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.10318192839622498,
 "2": 0.03813321143388748,
 "3": 0.8586848974227905
 }
 },
 {
 "id": "socialiqa/validation/656",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0007378910668194294,
 "2": 0.9991771578788757,
 "3": 8.491218613926321e-05
 }
 },
 {
 "id": "socialiqa/validation/1543",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0012145162327215075,
 "2": 6.744504389644135e-06,
 "3": 0.9987786412239075
 }
 },
 {
 "id": "socialiqa/validation/1558",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 4.157962030149065e-05,
 "2": 0.00025147636188194156,
 "3": 0.9997069239616394
 }
 },
 {
 "id": "socialiqa/validation/1872",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00011512243509059772,
 "2": 0.00046365769230760634,
 "3": 0.999421238899231
 }
 },
 {
 "id": "socialiqa/validation/1899",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.005813571624457836,
 "2": 0.9754627346992493,
 "3": 0.018723642453551292
 }
 },
 {
 "id": "socialiqa/validation/511",
 "source": "socialiqa",
 "expected": "3",
 "choice": "2",
 "probabilities": {
 "1": 0.0008486604201607406,
 "2": 0.6912782788276672,
 "3": 0.30787307024002075
 }
 },
 {
 "id": "socialiqa/validation/387",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.38236868381500244,
 "2": 0.617453396320343,
 "3": 0.00017796234169509262
 }
 },
 {
 "id": "socialiqa/validation/1584",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 4.845954754273407e-05,
 "2": 0.9991248250007629,
 "3": 0.0008267610101029277
 }
 },
 {
 "id": "socialiqa/validation/1833",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0001614582142792642,
 "2": 0.03548554703593254,
 "3": 0.9643530249595642
 }
 },
 {
 "id": "socialiqa/validation/1771",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.00069954595528543,
 "2": 4.408603490446694e-05,
 "3": 0.9992563128471375
 }
 },
 {
 "id": "socialiqa/validation/164",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 7.445141818607226e-05,
 "2": 6.85494342178572e-06,
 "3": 0.9999185800552368
 }
 },
 {
 "id": "socialiqa/validation/1847",
 "source": "socialiqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "1": 0.0033251820132136345,
 "2": 0.36342447996139526,
 "3": 0.63325035572052
 }
 },
 {
 "id": "socialiqa/validation/1913",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0037583669181913137,
 "2": 7.277533586602658e-05,
 "3": 0.9961687922477722
 }
 },
 {
 "id": "socialiqa/validation/41",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.005113918334245682,
 "2": 0.9412387609481812,
 "3": 0.053647346794605255
 }
 },
 {
 "id": "socialiqa/validation/69",
 "source": "socialiqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "1": 0.09610597044229507,
 "2": 0.8976677060127258,
 "3": 0.006226335186511278
 }
 },
 {
 "id": "socialiqa/validation/872",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0030383646953850985,
 "2": 9.105789104069117e-06,
 "3": 0.9969524145126343
 }
 },
 {
 "id": "socialiqa/validation/422",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.99978107213974,
 "2": 4.531845115707256e-05,
 "3": 0.00017367217515129596
 }
 },
 {
 "id": "socialiqa/validation/734",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.06234591826796532,
 "2": 0.015899060294032097,
 "3": 0.9217550754547119
 }
 },
 {
 "id": "socialiqa/validation/668",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.29717791080474854,
 "2": 0.19112853705883026,
 "3": 0.51169353723526
 }
 },
 {
 "id": "socialiqa/validation/1466",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9955729246139526,
 "2": 0.0014747792156413198,
 "3": 0.0029523344710469246
 }
 },
 {
 "id": "socialiqa/validation/1945",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 1.8224960513180122e-05,
 "2": 4.821933907805942e-05,
 "3": 0.9999334812164307
 }
 },
 {
 "id": "socialiqa/validation/538",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 7.312498928513378e-05,
 "2": 1.673862789175473e-05,
 "3": 0.9999102354049683
 }
 },
 {
 "id": "socialiqa/validation/796",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.5862758755683899,
 "2": 0.41359496116638184,
 "3": 0.00012908903590869159
 }
 },
 {
 "id": "socialiqa/validation/1305",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0014155417447909713,
 "2": 0.021199939772486687,
 "3": 0.9773845076560974
 }
 },
 {
 "id": "socialiqa/validation/1728",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00564607186242938,
 "2": 0.9440173506736755,
 "3": 0.050336580723524094
 }
 },
 {
 "id": "socialiqa/validation/1760",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0011187467025592923,
 "2": 0.00794958509504795,
 "3": 0.9909316301345825
 }
 },
 {
 "id": "socialiqa/validation/882",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9553396105766296,
 "2": 0.001142145600169897,
 "3": 0.04351820796728134
 }
 },
 {
 "id": "socialiqa/validation/1626",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.999393105506897,
 "2": 0.0005727108800783753,
 "3": 3.4238026273669675e-05
 }
 },
 {
 "id": "socialiqa/validation/44",
 "source": "socialiqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "1": 0.6830800771713257,
 "2": 0.27287065982818604,
 "3": 0.04404924809932709
 }
 },
 {
 "id": "socialiqa/validation/1835",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0010024867951869965,
 "2": 0.03987298533320427,
 "3": 0.9591245651245117
 }
 },
 {
 "id": "socialiqa/validation/1427",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9898282289505005,
 "2": 0.008235248737037182,
 "3": 0.0019365667831152678
 }
 },
 {
 "id": "socialiqa/validation/1574",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.000193800704437308,
 "2": 0.9998007416725159,
 "3": 5.523168965737568e-06
 }
 },
 {
 "id": "socialiqa/validation/1443",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9957277774810791,
 "2": 5.385258191381581e-05,
 "3": 0.004218376241624355
 }
 },
 {
 "id": "socialiqa/validation/491",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.08910928666591644,
 "2": 0.9091673493385315,
 "3": 0.001723363297060132
 }
 },
 {
 "id": "socialiqa/validation/1252",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9705923199653625,
 "2": 0.014560675248503685,
 "3": 0.01484694518148899
 }
 },
 {
 "id": "socialiqa/validation/630",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9149309992790222,
 "2": 0.08137742429971695,
 "3": 0.00369147676974535
 }
 },
 {
 "id": "socialiqa/validation/1352",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0031640324741601944,
 "2": 0.0006947190267965198,
 "3": 0.996141254901886
 }
 },
 {
 "id": "socialiqa/validation/1523",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0466579906642437,
 "2": 0.953326940536499,
 "3": 1.5002521649876144e-05
 }
 },
 {
 "id": "socialiqa/validation/66",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.003336424008011818,
 "2": 0.996631920337677,
 "3": 3.161861241096631e-05
 }
 },
 {
 "id": "socialiqa/validation/1445",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.13364611566066742,
 "2": 0.8523592948913574,
 "3": 0.013994641602039337
 }
 },
 {
 "id": "socialiqa/validation/1611",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.031099967658519745,
 "2": 0.9687754511833191,
 "3": 0.0001246389001607895
 }
 },
 {
 "id": "socialiqa/validation/17",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.000399072770960629,
 "2": 0.9994862079620361,
 "3": 0.00011467233707662672
 }
 },
 {
 "id": "socialiqa/validation/837",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9914440512657166,
 "2": 0.0004703312588389963,
 "3": 0.008085678331553936
 }
 },
 {
 "id": "socialiqa/validation/1066",
 "source": "socialiqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "1": 0.8964216709136963,
 "2": 0.00042056787060573697,
 "3": 0.10315772145986557
 }
 },
 {
 "id": "socialiqa/validation/1947",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 1.923263153003063e-05,
 "2": 4.8734742449596524e-05,
 "3": 0.999932050704956
 }
 },
 {
 "id": "socialiqa/validation/1777",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.8318039178848267,
 "2": 0.1644279807806015,
 "3": 0.0037680435925722122
 }
 },
 {
 "id": "socialiqa/validation/1686",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.021155070513486862,
 "2": 0.000772045343182981,
 "3": 0.9780728220939636
 }
 },
 {
 "id": "socialiqa/validation/522",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9923703670501709,
 "2": 0.0020869902800768614,
 "3": 0.005542632658034563
 }
 },
 {
 "id": "socialiqa/validation/1618",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9733766913414001,
 "2": 0.0010223629651591182,
 "3": 0.025600964203476906
 }
 },
 {
 "id": "socialiqa/validation/821",
 "source": "socialiqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "1": 0.0003115235595032573,
 "2": 0.1998879760503769,
 "3": 0.7998005151748657
 }
 },
 {
 "id": "socialiqa/validation/1840",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.17043361067771912,
 "2": 0.015238785184919834,
 "3": 0.8143276572227478
 }
 },
 {
 "id": "socialiqa/validation/816",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0034293653443455696,
 "2": 0.0026292866095900536,
 "3": 0.9939413070678711
 }
 },
 {
 "id": "socialiqa/validation/1757",
 "source": "socialiqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "1": 0.0004642140702344477,
 "2": 4.075893957633525e-05,
 "3": 0.9994950294494629
 }
 },
 {
 "id": "socialiqa/validation/1732",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00820161122828722,
 "2": 0.9917460083961487,
 "3": 5.236700235400349e-05
 }
 },
 {
 "id": "socialiqa/validation/1671",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0007105571567080915,
 "2": 0.9988138675689697,
 "3": 0.0004755442787427455
 }
 },
 {
 "id": "socialiqa/validation/622",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.0001758451689966023,
 "2": 0.9997069239616394,
 "3": 0.00011716940207406878
 }
 },
 {
 "id": "socialiqa/validation/297",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00036277607432566583,
 "2": 0.9988319277763367,
 "3": 0.0008052707416936755
 }
 },
 {
 "id": "cosmosqa/validation/36AZSFEYZ4YNUT7RK7W0NHUM3FIBVD##3Z7EFSHGNAC1U72MOPZFB6TDGEGCXM##A2MHM7JK1T9XHR##Blog_171727##q2_a1##3MG8450X2QO4FXI8IKGWEDL0ZDCPUP",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.6764927295298548e-06,
 "1": 8.56883852975443e-05,
 "2": 1.8897435438702814e-05,
 "3": 0.9998936653137207
 }
 },
 {
 "id": "socialiqa/validation/1826",
 "source": "socialiqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "1": 0.00012958339357282966,
 "2": 0.9997685551643372,
 "3": 0.00010189544263994321
 }
 },
 {
 "id": "socialiqa/validation/390",
 "source": "socialiqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "1": 0.9995278120040894,
 "2": 0.00025248664314858615,
 "3": 0.0002197611756855622
 }
 },
 {
 "id": "cosmosqa/validation/3GKAWYFRAR7IYV7MWPQ74DHCDIKDPG##3CPLWGV3MRE9F4GMUWEF3CPGFR7N9G##AJ7NSZOAFP05U##Blog_1449081##q1_a1##3NFWQRSHVGS9J2SJKCVAY466PH1FG6",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 4.827066368306987e-05,
 "1": 1.0645929251040798e-05,
 "2": 4.616374837951298e-07,
 "3": 0.9999406337738037
 }
 },
 {
 "id": "cosmosqa/validation/3SCKNODZ0ZU2E1UZPXAT0W2LRLBN7I##37XITHEISZOWS0FHZN5NEMVYLFCRCD##A2KYOM2ZGYDJGT##Blog_633917##q1_a1##31S7M7DAGI4ZOZOLQNXFS0QEBS5LT3",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.0002673917624633759,
 "1": 0.00027629677788354456,
 "2": 0.0003146483504679054,
 "3": 0.9991415739059448
 }
 },
 {
 "id": "cosmosqa/validation/3IZPORCT1HNKMOS3VDDM0TWP6C3HRR##33LKR6A5KHZPLCIBNFLYX5DP42BT1S##A2KYOM2ZGYDJGT##Blog_484244##q1_a1##3YLTXLH3DHKZABCOIL9XVYWA56QHPY",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999923706054688,
 "1": 1.2626856005226728e-06,
 "2": 6.1145410654717125e-06,
 "3": 3.452121859481849e-07
 }
 },
 {
 "id": "cosmosqa/validation/3HFWPF5AK9H2Z6MABJCPUBC3G7HS3G##3A4TN5196LG48H27JLHT8K5JR46HC2##AJ7NSZOAFP05U##Blog_371625##q1_a1##3RWB1RTQDL16H0BVFHFRIL3B2I3P8K",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 9.432020533495233e-07,
 "1": 1.7991502545555704e-06,
 "2": 8.018475637072697e-05,
 "3": 0.9999170303344727
 }
 },
 {
 "id": "cosmosqa/validation/3MJ28H2Y1E6QFTHLILJ9P30MXQYO5M##3GM6G9ZBKOV7MC5TW4SWPZDJXS9MT0##AISNLDPD2DFEG##Blog_291148##q1_a1##34R0BODSP3DJXR3385SDCPUGMPIE5C",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9971014857292175,
 "1": 0.0023949407041072845,
 "2": 0.00021916208788752556,
 "3": 0.00028430900420062244
 }
 },
 {
 "id": "cosmosqa/validation/3O71U79SRD3IFHTR1ZT4I89IAKZMSE##31N2WW6R9U5BCAEPFZUS4UDLDMV3FK##A2KYOM2ZGYDJGT##Blog_1291728##q1_a1##3MJ28H2Y1GM59IGS2JV9H1OVR9HO5M",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.002335276687517762,
 "1": 0.9964684247970581,
 "2": 0.0009476903360337019,
 "3": 0.00024863469297997653
 }
 },
 {
 "id": "race/validation/4477",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9864054322242737,
 "B": 0.010629922151565552,
 "C": 0.002082533435896039,
 "D": 0.0008821196388453245
 }
 },
 {
 "id": "cosmosqa/validation/3EHIMLB7F9DE4JD7W47M9C1LH4TH80##3N1FSUEFL8FZWMJXO74LQZ26F904DK##A2MHM7JK1T9XHR##Blog_878241##q1_a1##36U4VBVNQQRPRAYNVCKLJ8LG1DUURJ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "0",
 "probabilities": {
 "0": 0.9543530941009521,
 "1": 0.001805181847885251,
 "2": 0.0021118302829563618,
 "3": 0.04172990098595619
 }
 },
 {
 "id": "race/validation/4481",
 "source": "race",
 "expected": "B",
 "choice": "C",
 "probabilities": {
 "A": 0.000644440995529294,
 "B": 0.29752156138420105,
 "C": 0.7000513672828674,
 "D": 0.001782639417797327
 }
 },
 {
 "id": "cosmosqa/validation/3DA79LNS5B9I8VYWKHTFWEWXA3G3T2##3LS2AMNW5I5XPOXKCDPE4WFACRQQO7##A2KYOM2ZGYDJGT##Blog_1029073##q1_a1##3I6NF2WGIIAHH5ZUE0I1CSFSZ3T5GC",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.999795138835907,
 "1": 6.756505172234029e-05,
 "2": 0.00013359394506551325,
 "3": 3.7181337120273383e-06
 }
 },
 {
 "id": "cosmosqa/validation/3IV1AEQ4DTRAVP8XK4A7LPHZTQE8JY##3A0EX8ZRNB3MFWZXZI1MBYID7AMYBX##AGUF6B4Q9991D##Blog_929518##q1_a1##3ZUE82NE0CFUQNM11EJM6RP81YB8F6",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.021337848156690598,
 "1": 0.0001322846073890105,
 "2": 0.9686663746833801,
 "3": 0.009863492101430893
 }
 },
 {
 "id": "cosmosqa/validation/3R4QIDVOJRQJTAWWJ6LGGJ0TZ96EEK##32KTQ2V7RGU3XMVMIVZFQX0G8HFM9R##A2KYOM2ZGYDJGT##Blog_321812##q1_a1##3K1H3NEY7NDCLIENT60KA43KCNEDGL",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999899864196777,
 "1": 5.349515504349256e-06,
 "2": 2.3544300802313956e-06,
 "3": 2.2739625364920357e-06
 }
 },
 {
 "id": "cosmosqa/validation/3X52SWXE0X3JJNZ2OHXRK3JIS9JWCE##3EFE17QCRD3KFYR3G77TNP91EQ8HSF##AJ7NSZOAFP05U##Blog_1129220##q1_a1##3VP28W7DUPY9AJNOA4JZJBMBZP4ZFL",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "2",
 "probabilities": {
 "0": 0.001542618265375495,
 "1": 0.005401781760156155,
 "2": 0.9926490187644958,
 "3": 0.0004065804532729089
 }
 },
 {
 "id": "cosmosqa/validation/3M47JKRKCZFY7UTQ0LGICLCU7AY68A##3ZGVPD4G6WWMG83JVQUQPZX1I4YZT8##A2KYOM2ZGYDJGT##Blog_148956##q1_a1##3UZUVSO3P99H5YO9049ZTR528M2ME8",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9997492432594299,
 "1": 4.404380524647422e-05,
 "2": 9.959046292351559e-05,
 "3": 0.00010708162881201133
 }
 },
 {
 "id": "cosmosqa/validation/3FJ2RVH251KA3YTZIBATM4QT8HU92A##3XUHV3NRVNDY4LSZO86LMCNIZ11H5O##AO33H4GL9KZX9##Blog_887974##q1_a1##3HFWPF5AKBXHTVLHVHOPM90B97PS30",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 1.300500753131928e-05,
 "1": 0.9999803304672241,
 "2": 5.852309186593629e-06,
 "3": 8.594371365688858e-07
 }
 },
 {
 "id": "cosmosqa/validation/3ZTE0JGGCEQBEEQ0XV6H396WP22COQ##308Q0PEVB9B26E6ODJPN665QM0K9IH##A2KYOM2ZGYDJGT##Blog_558527##q1_a1##3SCKNODZ0ZU2E1UZPXAT0W2LRLM7ND",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9967467784881592,
 "1": 0.0029262967873364687,
 "2": 0.00022911677660886198,
 "3": 9.78292227955535e-05
 }
 },
 {
 "id": "cosmosqa/validation/3PZDSVZ3J7V5VECGNZCMXKIB0ER4NB##3DR23U6WE8TO5DCNZ3GD4PWJTQXTEN##A2KYOM2ZGYDJGT##Blog_1272712##q1_a1##3OB6JN3A9S36E2TEMATZSY4LNBSRMQ",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0007056870963424444,
 "1": 0.999154806137085,
 "2": 9.02012106962502e-05,
 "3": 4.936749246553518e-05
 }
 },
 {
 "id": "cosmosqa/validation/3WGCNLZJKHMFH3OKBNFH3HZEAMG1D6##338JKRMM29EVAR4GJ4X2UN34XZAAH5##A394JO4NEPCY3M##Blog_1312805##q1_a2##3OID399FXIL67IISIMU2SJ38B8AFDK",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.04541895538568497,
 "1": 0.004261186346411705,
 "2": 0.004948689602315426,
 "3": 0.945371150970459
 }
 },
 {
 "id": "cosmosqa/validation/3R6RZGK0XHQY1QZ9EXMKNQGVHNMVY8##3I0BTBYZA00LV9WN124XRHMUFCP0YO##A2KYOM2ZGYDJGT##Blog_82927##q1_a1##3R16PJFTS554N8RDQMK65S176TK4KO",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "0",
 "probabilities": {
 "0": 0.748157799243927,
 "1": 2.953496550617274e-05,
 "2": 0.0014868517173454165,
 "3": 0.2503258287906647
 }
 },
 {
 "id": "cosmosqa/validation/3SX4X51T807NLZT2DHAQXEG8PS6OAB##34X6J5FLPUW4TO7GC3L942AF8AWQJ4##AO33H4GL9KZX9##Blog_1361284##q1_a1##3G9UA71JVX86VB3EAZI17PL4IZR7JF",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.004536914639174938,
 "1": 1.880131208054081e-06,
 "2": 5.613626490230672e-05,
 "3": 0.9954050779342651
 }
 },
 {
 "id": "cosmosqa/validation/3JU8CV4BRLAU83Z420SNN953OCHPOZ##3AAPLD8UCDFLUY4I57CJXU5NFHLHTI##AXJ1PE87PD0KK##Blog_974653##q1_a2##34KYK9TV2TMMWQPUAA3V4SYNX7KSBP",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.010912950150668621,
 "1": 0.00870376918464899,
 "2": 0.028791511431336403,
 "3": 0.9515916705131531
 }
 },
 {
 "id": "cosmosqa/validation/3BDORL6HKKBEBRIGPNNLX4B10GMCRH##3WOKGM4L72EU66TEBIJZXS0JUSCO0B##AXJ1PE87PD0KK##Blog_461979##q1_a2##3IQ9O0AYW8DXYYGF5K020KYMUGITIO",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.17278409004211426,
 "1": 0.0010052507277578115,
 "2": 0.8251758217811584,
 "3": 0.0010347871575504541
 }
 },
 {
 "id": "cosmosqa/validation/3SSN80MU8E2VLAFNAXMB23Y5ZJLXKI##3LRKMWOKB8WSW996CBAUOCOBI4RZ2C##A2KYOM2ZGYDJGT##Blog_1297526##q1_a1##31GECDVA9L0B3GA276JKS7G31P6666",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 3.37977269282419e-07,
 "1": 2.7734733521356247e-05,
 "2": 3.848525011562742e-06,
 "3": 0.9999680519104004
 }
 },
 {
 "id": "cosmosqa/validation/35U0MRQMULLM7CV78ADMLZIT36EVOA##36TFCYNS47P764C9UEPGEVE2YVEXHD##A394JO4NEPCY3M##Blog_1564154##q1_a1##3QMELQS6Y7P4PZ8MZUMI327ZXA26RI",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.013079170137643814,
 "1": 0.005687070079147816,
 "2": 0.0003292332694400102,
 "3": 0.9809046387672424
 }
 },
 {
 "id": "cosmosqa/validation/3R0WOCG21M7GWBGBTDD1MLG3SMEUDT##3OHYZ19UGD3Q7JRW3KGSMYCW4GQOA9##A2KYOM2ZGYDJGT##Blog_973528##q1_a1##33KGGVH24WVTF9T12NGTXO26QDRX1L",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 2.4939577087934595e-06,
 "1": 0.0005018282681703568,
 "2": 1.628890117899573e-06,
 "3": 0.9994939565658569
 }
 },
 {
 "id": "cosmosqa/validation/3P4C70TRMTVVJU2F6NY8THBPLRHLG0##3U088ZLJVN8H641NM1M96K1E4ZGW02##A394JO4NEPCY3M##Blog_1099909##q1_a2##3XT3KXP241CRW05397S9PA68AGF6IE",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.011827025562524796,
 "1": 0.8995356559753418,
 "2": 0.06278897821903229,
 "3": 0.025848351418972015
 }
 },
 {
 "id": "cosmosqa/validation/3YGE63DINABEJK3TWZG7H05PB0E0WO##3X0H8UUIT43H7DLZV35TGQ7XVSXSWH##AJ7NSZOAFP05U##Blog_707860##q1_a1##3CMV9YRYP5F5AWSGETJBQU35DITJLV",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.4325995834951755e-05,
 "1": 4.9770664190873504e-05,
 "2": 4.510678536462365e-06,
 "3": 0.9999313354492188
 }
 },
 {
 "id": "cosmosqa/validation/3QGHA0EA0LEF3763UFDW1H183GYWBI##3OONKJ5DKFYALIVVTVSO97KBRB7OBG##A2KYOM2ZGYDJGT##Blog_257329##q1_a1##38LRF35D5NAX88DLKMJP1AIZUI43UD",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 2.673627648164256e-07,
 "1": 2.819132021159021e-07,
 "2": 4.725222879642388e-08,
 "3": 0.9999994039535522
 }
 },
 {
 "id": "cosmosqa/validation/3ZFRE2BDQ9CBGGF2Q0DLXKYAAEXZX1##3Z7EFSHGNAC1U72MOPZFB6TDEEXXCM##AJ7NSZOAFP05U##Blog_1119261##q1_a1##37Y5RYYI0RJT9TSVF6VDTPPFG81XSS",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9988642930984497,
 "1": 0.0011321059428155422,
 "2": 9.025775398185942e-07,
 "3": 2.6832617550098803e-06
 }
 },
 {
 "id": "cosmosqa/validation/3Z33IC0JC20J0PR16LU4O9N48E1V9J##34FNN24DCPOKQGIOY8A98CILWK05YE##A2KYOM2ZGYDJGT##Blog_1098471##q1_a1##3V7ICJJAZCU3U5NJK97D6PIHJZB4B4",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.2513699150294997e-05,
 "1": 1.3594601568911457e-06,
 "2": 0.9999693632125854,
 "3": 6.755436970706796e-06
 }
 },
 {
 "id": "cosmosqa/validation/3W0KKJIARR7QCKF8IRYJU6T4FSQK8Z##3F6KKYWMNCZ6N5UWMF7HXAKXXI0NDT##AXJ1PE87PD0KK##Blog_728874##q1_a2##3D1UCPY6GINZAB68HJIK5I0AXFV83X",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.5251459899445763e-06,
 "1": 0.00018651281425263733,
 "2": 3.368605661080437e-08,
 "3": 0.9998119473457336
 }
 },
 {
 "id": "cosmosqa/validation/302U8RURJZZPV2WONX19ECUGKS2VN4##3XC1O3LBOTKNGUN8WSRHPMYUW7FLTW##AJ7NSZOAFP05U##Blog_828600##q1_a2##3R15W654VF7MWJWT9J3KNVFRREOLQT",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "0": 0.020099060609936714,
 "1": 6.975680298637599e-05,
 "2": 0.00033111858647316694,
 "3": 0.9795000553131104
 }
 },
 {
 "id": "cosmosqa/validation/38LRF35D5NAX88DLKMJP1AIZTHMU3J##39O5D9O87W749YUFOX4RWOSIDL33CU##A2KYOM2ZGYDJGT##Blog_763994##q1_a1##38RHULDV90TPQFK90P5DN45LYDOWIB",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 5.114595751365414e-07,
 "1": 2.1471671516337665e-06,
 "2": 2.8480263836172526e-07,
 "3": 0.9999970197677612
 }
 },
 {
 "id": "cosmosqa/validation/3OND0WXMHYTVDKNY97TM1MX6PYXHEI##33LK57MYLWKL18Q46QP0FU1FBTASZD##A2KYOM2ZGYDJGT##Blog_1179477##q1_a1##3J9UN9O9J56SD03UP0XSSABHAGRJ0N",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.8506242440707865e-06,
 "1": 6.111511902417988e-05,
 "2": 8.624668225820642e-06,
 "3": 0.99992835521698
 }
 },
 {
 "id": "cosmosqa/validation/3Q7TKIAPOVOS7ASO4XX0NW8S8WKDLV##3STRJBFXOZ6RRD4XVPRG2K9ODNSTK6##A2KYOM2ZGYDJGT##Blog_963939##q1_a1##3I7KR83SNCR5KEXP7HGXM0CNSKRK9R",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 4.751662345370278e-05,
 "1": 2.0362192287848302e-07,
 "2": 0.9999510049819946,
 "3": 1.2627562000488979e-06
 }
 },
 {
 "id": "cosmosqa/validation/3TTPFEFXCTIC6GVCJRYOX0HLHL26HB##3OE22WJIGJMD74IAO2PNB4JQODFQUU##A394JO4NEPCY3M##Blog_789630##q1_a1##3NBFJK3IOJW3PF5CJFE0LYFJGT5GOP",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9996240139007568,
 "1": 0.0003476256679277867,
 "2": 1.5412464563269168e-05,
 "3": 1.2993580639886204e-05
 }
 },
 {
 "id": "cosmosqa/validation/3TTPFEFXCVYR05UJ3PAOPY5T9FN6HV##3DHE4R9OCZQS5SE1MJM5358545D2GL##A2KYOM2ZGYDJGT##Blog_990065##q1_a1##3EKTG13IZWHVB4Y1HQW36MVMHCPLM2",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 2.0170763548321702e-07,
 "1": 0.9996893405914307,
 "2": 6.73031763653853e-07,
 "3": 0.0003098096640314907
 }
 },
 {
 "id": "cosmosqa/validation/3L4YG5VW9P699Y4WRGZC4J0NZL4DDX##3FE7TXL1LL2JIH8FPFRHXJC7C622QQ##A2KYOM2ZGYDJGT##Blog_1417857##q1_a1##3KA7IJSNW8JPJY92ITZ19NTCATPPBU",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 9.477341933461503e-08,
 "1": 0.9999474287033081,
 "2": 3.897403928476706e-07,
 "3": 5.209715891396627e-05
 }
 },
 {
 "id": "cosmosqa/validation/3O2Y2UIUCS8ELOO2A3GPWWYYVCCFKS##3OLQQLKKNV4URAGIN9292J9AWYXEJF##A2KYOM2ZGYDJGT##Blog_1532372##q1_a1##3P4ZBJFX2XH5GOIJZHZSK7STIQFWFD",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.07525786012411118,
 "1": 0.0022911850828677416,
 "2": 0.9124293327331543,
 "3": 0.010021611116826534
 }
 },
 {
 "id": "cosmosqa/validation/379OL9DBSSCLAUNZRY5O3A3QXTXY99##3I33IC7ZWG0C0C2RGMIVFL9T3VGA2F##A2KYOM2ZGYDJGT##Blog_1412614##q1_a1##3SV8KD29L66KQ5F2BFAWZOEG7LYKZL",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 5.633077648781182e-07,
 "1": 2.5300236302427948e-05,
 "2": 2.6176254053211778e-08,
 "3": 0.9999741315841675
 }
 },
 {
 "id": "cosmosqa/validation/3D06DR5225HZLW8Q01VD5AR3UT8AM1##3AUQQEL7U6RPWQ2BCU5DSKURQY5V06##A2LTOS0AZ3B28A##Blog_619627##q2_a1##3WUVMVA7ODHEES6GZOX75ABK26OZAZ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.012988134287297726,
 "1": 0.03774354234337807,
 "2": 0.04685899615287781,
 "3": 0.9024093747138977
 }
 },
 {
 "id": "cosmosqa/validation/382GHPVPHSPJXLKZLIG7W8WX2YP43A##3ZPBJO59KQZED98LJHC4SQN1D2UDH7##AXJ1PE87PD0KK##Blog_298872##q1_a2##3ZG552ORAOI80562LCFPEMFQ2822V9",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.438679757688078e-06,
 "1": 2.7158853299624752e-06,
 "2": 0.9999927282333374,
 "3": 2.1346502308006166e-06
 }
 },
 {
 "id": "cosmosqa/validation/3R5OYNIC2C7ALV4CYHEMHJBQNTXPT2##30BUDKLTXET7PM2QEAMF9B2W84CE5Q##AJ7NSZOAFP05U##Blog_655627##q1_a1##3WYZV0QBFLRUS7BT4K9I30FFCPMXB8",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "0": 1.1907673069799785e-06,
 "1": 0.017539072781801224,
 "2": 0.9824427366256714,
 "3": 1.706151851976756e-05
 }
 },
 {
 "id": "cosmosqa/validation/35O6H0UNLSECEIQO28R4H7J0YR65JS##3IAEQB9FMFIWAZ3AEGY4TTXOSQWWDF##AXJ1PE87PD0KK##Blog_1681##q1_a2##35YHTYFL1IHS1ZIE97UWY59FBC7FV6",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 8.009393059182912e-05,
 "1": 0.0070707425475120544,
 "2": 0.006968157831579447,
 "3": 0.985880970954895
 }
 },
 {
 "id": "cosmosqa/validation/368IUKXGA564JXT0246T69JC3ED6PL##32Q90QCQ1TJA75NTLXQLB0LE4VFKEQ##A2KYOM2ZGYDJGT##Blog_958341##q1_a1##3JVP4ZJHDR6WIPJO7ICSG9WHX6WI0E",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9998904466629028,
 "1": 6.059888983145356e-05,
 "2": 4.5455781219061464e-05,
 "3": 3.490529024929856e-06
 }
 },
 {
 "id": "cosmosqa/validation/3L1EFR8WWVJU4S5B66P06NKIKD5F9T##33FBRBDW6REQAV0L0EZNN50TUPNC8O##A2KYOM2ZGYDJGT##Blog_179342##q1_a1##3S829FDFT4FN749DIG0OYQ3FFDADXD",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9963874816894531,
 "1": 0.00010355238191550598,
 "2": 0.001117986161261797,
 "3": 0.0023910379968583584
 }
 },
 {
 "id": "cosmosqa/validation/3CVBMEMMXBEVAG9H213G3EHHCV17HO##34T446B1C1CTXHWBLDA82S6243OC0A##A2KYOM2ZGYDJGT##Blog_1034814##q1_a1##3KWGG5KP6LG2804V4D34335ZN1UMC2",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9889400601387024,
 "1": 0.0013472047867253423,
 "2": 0.0006020480650477111,
 "3": 0.009110712446272373
 }
 },
 {
 "id": "cosmosqa/validation/3B9J25CZ27R56VH0OAZQFC45FKUCSA##3KKG4CDWKLDNTW6GCRZJ43JGMAF493##AJ7NSZOAFP05U##Blog_1539728##q1_a1##3IKMEYR0LY9UKUCQM81ICD49MDX2KV",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 7.945839024614543e-05,
 "1": 1.1089455256296787e-06,
 "2": 0.999882698059082,
 "3": 3.6728659324580804e-05
 }
 },
 {
 "id": "cosmosqa/validation/351S7I5UG9UG47WU68NQBOJMP36NJI##345LHZDEDYQL03C8QRDRYWQFFPS3U4##AJ7NSZOAFP05U##Blog_795595##q1_a1##31YWE12TE2Q7QV83R3X2YYTJTWVX7G",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.0012767250882461667,
 "1": 1.4445282431552187e-05,
 "2": 0.9986923336982727,
 "3": 1.6466961824335158e-05
 }
 },
 {
 "id": "cosmosqa/validation/3J5XXLQDHM9IRY3E5EHU5YWAFCRV36##39JEC7537VZ9QIXROAYXNLWSZSKCV8##AO33H4GL9KZX9##Blog_884565##q1_a1##3P888QFVX58U24ZQMBJCFCJLMK7QOV",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999750852584839,
 "1": 9.343000783701427e-06,
 "2": 1.2986818546778522e-05,
 "3": 2.5633412406023126e-06
 }
 },
 {
 "id": "cosmosqa/validation/391FPZIE4E0C2GKKZ78R54J9C14HU7##3QILPRALQ8A91ZXCP4813GU5QM18NL##A2KYOM2ZGYDJGT##Blog_62238##q1_a1##306996CF6YYM26Q0XROB8RBE77FB1W",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.003439784748479724,
 "1": 0.09398829936981201,
 "2": 0.023047463968396187,
 "3": 0.8795244693756104
 }
 },
 {
 "id": "cosmosqa/validation/3MDKGGG61S1H5ELE3JRD3H8HLJK6TS##39DD6S19JSQKRPLM5V5VQDX0E4GZE6##A394JO4NEPCY3M##Blog_860847##q1_a2##388CL5C1RL19JQXQQTGQEHNQ0WPHL0",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.04211769625544548,
 "1": 0.08641420304775238,
 "2": 0.8310366868972778,
 "3": 0.0404314361512661
 }
 },
 {
 "id": "cosmosqa/validation/3WUVMVA7ODHEES6GZOX75ABK19TZA9##3CCZ6YKWRAY5PFAGQWIT9VPUKK759W##AJ7NSZOAFP05U##Blog_602590##q1_a1##375VSR8FVYN6D9SX1ZJT000U2BBZR8",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.008697698824107647,
 "1": 1.4987946997280233e-05,
 "2": 0.9901542067527771,
 "3": 0.0011331377318128943
 }
 },
 {
 "id": "cosmosqa/validation/3XDSWAMB246TTF2NW6ALKCP4XEUCQ6##3BV8HQ2ZZZGFD2KZNL6YXWEEQJOA6W##A2KYOM2ZGYDJGT##Blog_83046##q1_a1##3M0556243UYFTU957G85OOF3QTMNFL",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9944789409637451,
 "1": 0.0027659302577376366,
 "2": 0.001962060574442148,
 "3": 0.0007930790307000279
 }
 },
 {
 "id": "cosmosqa/validation/3DW3BNF1GHG4SV9KHVCQWF4LV9WV8X##33CID57105RIHDPEHDLQ79UW3QEL3S##A33PQXKQHFVF3V##Blog_1489836##q1_a1##359AP8GAGIYJNXX8071FKL89SJ0C7W",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0008690086542628706,
 "1": 0.9895117878913879,
 "2": 5.0834903959184885e-05,
 "3": 0.009568408131599426
 }
 },
 {
 "id": "cosmosqa/validation/33W1NHWFYHJ4ZZ6ILQCO8HD4XXNTZK##3LRLIPTPER77HS6H5SOZFHVNK23AKG##A2MHM7JK1T9XHR##Blog_429737##q2_a1##3VMHWJRYHXUK5ON99MVEMQ60RP6XFM",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.011477875523269176,
 "1": 0.0019427216611802578,
 "2": 0.00020211466471664608,
 "3": 0.9863772392272949
 }
 },
 {
 "id": "cosmosqa/validation/36QZ6V1589BMY0ZTQ38QHK23TRBSUC##3K5TEWLKGW903LJRGKUJYOUNDTBVIT##A2KYOM2ZGYDJGT##Blog_495387##q1_a1##3I7SHAD35OAWWJ9ZDWVTNHDLMVC7MD",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9989079236984253,
 "1": 1.5653861282771686e-06,
 "2": 0.0010268439073115587,
 "3": 6.370308983605355e-05
 }
 },
 {
 "id": "cosmosqa/validation/3YD0MU1NC2ZG0T13WVTOMSUDZ3X7A5##3FFJ6VRIL2MKAKIBDN6UDV4XIQM0IF##A2KYOM2ZGYDJGT##Blog_645273##q1_a1##3WUVMVA7ODHEES6GZOX75ABK25AZAJ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 4.57276837551035e-05,
 "1": 4.644138698495226e-06,
 "2": 4.813574196305126e-05,
 "3": 0.999901533126831
 }
 },
 {
 "id": "cosmosqa/validation/3IHWR4LC7FRE0EOR2MP79O2ZG4AI83##32EYX73OY3OSEUWHLEQN8SH4S40RUI##AJ7NSZOAFP05U##Blog_52147##q1_a1##3AQN9REUTHUC79ZNNCMQH4AOHLZYD3",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 2.3327152121055406e-06,
 "1": 0.9999957084655762,
 "2": 6.088571353757288e-07,
 "3": 1.2683912018474075e-06
 }
 },
 {
 "id": "cosmosqa/validation/372AGES0I432EPTBGJJ8BHWB1XERX0##3G0WWMR1UWI0CEY1R1D1X9RWGGMQN0##A1OPGPTQJ48XOY##Blog_742807##q1_a1##359AP8GAGIYJNXX8071FKL89SC07CD",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.6639467048662482e-06,
 "1": 1.223939034389332e-05,
 "2": 2.2324613382807e-05,
 "3": 0.9999637603759766
 }
 },
 {
 "id": "cosmosqa/validation/3OB6JN3A9QNRKDU72CHZ00GDU3PRMV##3DBQWDE4Y7WBJSF379PYZYW5MW8N5J##A2KYOM2ZGYDJGT##Blog_311600##q1_a1##3YOAVL4CA2VZC3QZQNNGS7PKAX44Z7",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 7.255410764628323e-06,
 "1": 0.9998914003372192,
 "2": 1.4912799088051543e-05,
 "3": 8.638002327643335e-05
 }
 },
 {
 "id": "cosmosqa/validation/3U18MJKL1W08QIPVGEI4GIUZS14NCN##3TUI152ZZE2CTWQJ6BR3S022MLGQ1X##A2KYOM2ZGYDJGT##Blog_643890##q1_a1##3BAKUKE49JQ9ID7PTOA186GJQGG1RA",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.021231314167380333,
 "1": 0.0037492334377020597,
 "2": 0.239744171500206,
 "3": 0.7352753281593323
 }
 },
 {
 "id": "cosmosqa/validation/32W3UF2EZQZM4AFPY938H9OZ1ADC41##3FIJLY1B6XJIJZJCDXRZHXGMGTCPFR##A394JO4NEPCY3M##Blog_355255##q1_a1##3DTJ4WT8BFTH45NSFTZT1T1BNQLEZI",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.2144828360760584e-06,
 "1": 1.991920726140961e-05,
 "2": 8.703691634082134e-08,
 "3": 0.99997878074646
 }
 },
 {
 "id": "cosmosqa/validation/3JYPJ2TAYI602QMB3X9X0FWPWBEPF1##3WAKVUDHUXE31MDVS3GHBD2VJNMU7Z##A394JO4NEPCY3M##Blog_1062922##q1_a1##3NKW03WTLOL27D8SDXAMW6W7PIDQW6",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9037656188011169,
 "1": 0.0949261412024498,
 "2": 0.0004068689013365656,
 "3": 0.0009014513343572617
 }
 },
 {
 "id": "cosmosqa/validation/3XEDXEGFX3M8UVA5NNL41RNMW8V0K1##3GU1KF0O4JZDBT8POYT3691SW8FPB4##A2KYOM2ZGYDJGT##Blog_495809##q1_a1##32XN26MTX1XY8QRUSL958PO9ECGL0P",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.00034913833951577544,
 "1": 5.584151585935615e-05,
 "2": 0.9972768425941467,
 "3": 0.0023181219585239887
 }
 },
 {
 "id": "cosmosqa/validation/3XUSYT70ITZ0GP4VV6ZHXESWCVYD0T##3ZQIG0FLQFEI4G3W51YYZF3VRF1WVE##AJ7NSZOAFP05U##Blog_174977##q1_a1##3SSN80MU8E2VLAFNAXMB23Y5ZJ1KXL",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.0005338348564691842,
 "1": 0.0032104619313031435,
 "2": 0.9961405396461487,
 "3": 0.00011528308823471889
 }
 },
 {
 "id": "cosmosqa/validation/33BFF6QPI3PA0C2QFMYQJY3WFMZW31##3TYCR1GOTFYYXVVEMHSZ38NKN1XZL9##A33PQXKQHFVF3V##Blog_796482##q1_a1##3RWO3EJELJNO60RAGW6C3BG3OHTP16",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.00045896790106780827,
 "1": 0.000420332740759477,
 "2": 0.9990798234939575,
 "3": 4.076835830346681e-05
 }
 },
 {
 "id": "cosmosqa/validation/3O71U79SRBN3LSUKH1H4QAL9GPTMS3##3DI28L7YXBC81417DNK36LPFUN5E12##AO33H4GL9KZX9##Blog_1001422##q1_a1##3YGYP13643NCWKEEYB8Z5DTLSPSRNS",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 1.8611433461046545e-06,
 "1": 2.2599101612286177e-07,
 "2": 0.9999972581863403,
 "3": 6.574535404979542e-07
 }
 },
 {
 "id": "cosmosqa/validation/3MIVREZQVHWWIOF44LFJGXE49HBKQB##3JBT3HLQF909TRB5UC96SR8VNWHPZW##AJ7NSZOAFP05U##Blog_375911##q1_a1##3D5G8J4N5CI2K40F4RZLF9OG2N6TVN",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 6.03026890644287e-08,
 "1": 0.9999912977218628,
 "2": 8.488405001116917e-06,
 "3": 1.3692582001567644e-07
 }
 },
 {
 "id": "cosmosqa/validation/3R868ACW4SA0UUKC1QDH9YYF7O1ZG9##3VE8AYVF8NVIIILO7JDO3DXNKJA8FW##A1OPGPTQJ48XOY##Blog_747293##q1_a1##31ANT7FQNAGVH1U5Y60JXSRTEYA5H5",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9994144439697266,
 "1": 6.64134495309554e-05,
 "2": 4.421913035912439e-05,
 "3": 0.0004749921499751508
 }
 },
 {
 "id": "cosmosqa/validation/3PA41K45VPI2HM6C5CNL4L7GRPW7PW##3QAPZX2QN7SVUO3G8HX2IDLPZR4024##AJ7NSZOAFP05U##Blog_184536##q1_a1##3X0EMNLXER34SC78NAQ1AZNJDLWPVR",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 3.114681749138981e-05,
 "1": 1.4694513993163127e-05,
 "2": 0.9999514818191528,
 "3": 2.7221963136980776e-06
 }
 },
 {
 "id": "cosmosqa/validation/3N5YJ55YXG153ETQC2ADIPG3ZJIAN1##3K5TEWLKGW903LJRGKUJYOUNCS4IV6##AJ7NSZOAFP05U##Blog_1219011##q1_a1##3YGYP13643NCWKEEYB8Z5DTLSW4RNI",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "0",
 "probabilities": {
 "0": 0.9026973843574524,
 "1": 0.08826474845409393,
 "2": 0.00846133939921856,
 "3": 0.000576470629312098
 }
 },
 {
 "id": "cosmosqa/validation/38O9DZ0A62L8RQSU06VHAWBJS7V26A##35GCEFQ6I6M2M26IWECZFO5UHSL3ZD##A2KYOM2ZGYDJGT##Blog_1302100##q1_a1##3IKDQS3DQGEGCHE4JY5CBAYI6XOIC6",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "0": 0.041473641991615295,
 "1": 0.00199219211935997,
 "2": 0.34448403120040894,
 "3": 0.6120501756668091
 }
 },
 {
 "id": "cosmosqa/validation/32FESTC2NHO6RJAXTHIQ2FYL78ACUR##338JKRMM27XGF259Z7L22PFVPI7AHB##A29RROPO0Z90XV##Blog_921892##q1_a2##3JMQI2OLF1JIVAV9GADF0OCHDO1DN8",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0887865200638771,
 "1": 0.4749443531036377,
 "2": 0.42757999897003174,
 "3": 0.008689179085195065
 }
 },
 {
 "id": "cosmosqa/validation/32FESTC2NJ4LL894DFUQUDMT0J1UCY##3B2X28YI3ZUKWCPRTZ6US7YK64GB63##A2KYOM2ZGYDJGT##Blog_368912##q1_a1##3XJOUITW8W5AFW4YIS5B9MP8LW7TQB",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999619722366333,
 "1": 3.5464858228806406e-05,
 "2": 1.113844405153941e-06,
 "3": 1.5132518456084654e-06
 }
 },
 {
 "id": "cosmosqa/validation/30EV7DWJTX9DM0K8DPECBY0WZW06Y0##3HPZF4IVNP85P1R3DIXHMIOVLACYCK##AJ7NSZOAFP05U##Blog_1366992##q1_a1##373L46LKP9KUQQAYTT7ABUQQOL4JKE",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9995005130767822,
 "1": 0.00020360192866064608,
 "2": 0.00023606258037034422,
 "3": 5.9813381085405126e-05
 }
 },
 {
 "id": "cosmosqa/validation/3ZQX1VYFTFJOW0ZU0H2CTLYLSFN8O6##35L9RVQFCRXFP0IEP9ETUOFY56WUH4##AJ7NSZOAFP05U##Blog_776120##q1_a1##3E24UO25Q141SMG6725E972UMP06OV",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 4.6823282673358335e-07,
 "1": 1.614024199625419e-06,
 "2": 0.9999948740005493,
 "3": 2.9896293654019246e-06
 }
 },
 {
 "id": "cosmosqa/validation/3M93N4X8HM1SE9N6Y2GC2GH05FJSJ6##35GMH2SV3HW8SL7F3MUEAFQWV3TOEG##A2KYOM2ZGYDJGT##Blog_1396501##q1_a1##3YZ7A3YHR77K7A8L90CI16YVNJRS5X",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9994031190872192,
 "1": 0.0005911373882554471,
 "2": 3.4372353638900677e-06,
 "3": 2.2994565824774327e-06
 }
 },
 {
 "id": "cosmosqa/validation/32XN26MTX1XY8QRUSL958PO9E5XL0S##3OLF68YTNCGBWVDAOE92R2RRHQKFAB##A2KYOM2ZGYDJGT##Blog_1218698##q1_a1##3R868ACW4UQFOJJJLOPH1WMN05OGZN",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9997133612632751,
 "1": 1.1061802069889382e-05,
 "2": 1.8629099940881133e-05,
 "3": 0.00025698976241983473
 }
 },
 {
 "id": "cosmosqa/validation/3ECKRY5B1QUEQLQ7REDKUABVM2PZIZ##339ANSOTR60MNAPFHGGCWFJ6WH2IKZ##A2KYOM2ZGYDJGT##Blog_856806##q1_a1##31HLTCK4BN9YFZE9KR0EGQ9RCO6GVZ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 3.575413984435727e-06,
 "1": 3.841586658381857e-05,
 "2": 8.16606018361199e-08,
 "3": 0.9999579191207886
 }
 },
 {
 "id": "cosmosqa/validation/33BFF6QPI3PA0C2QFMYQJY3WFOL3WY##31UV0MXWNTRY0G3JUQO6MOOX54VI5J##A2KYOM2ZGYDJGT##Blog_1384143##q1_a1##32LAQ1JNTB327PBFZ1PGJNMXOHHTUK",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.999805748462677,
 "1": 6.168453546706587e-05,
 "2": 0.00012797676026821136,
 "3": 4.53426355306874e-06
 }
 },
 {
 "id": "cosmosqa/validation/3PUV2Q8SV6IGU16CDK869TCNWE5BDT##33ISQZVXPS1SMYQYHQFSPRWCM91CCB##A33PQXKQHFVF3V##Blog_434338##q1_a1##39TX062QX32PPC7GPRUCLHH4PHRX36",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 6.081093602006149e-07,
 "1": 4.951315872858686e-07,
 "2": 0.9999979734420776,
 "3": 9.508643756817037e-07
 }
 },
 {
 "id": "race/validation/4448",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 9.35309799388051e-06,
 "B": 2.6724212602857733e-06,
 "C": 1.3153541658539325e-05,
 "D": 0.9999748468399048
 }
 },
 {
 "id": "cosmosqa/validation/3IH9TRB0FDDWPE4631GSUEFS3B21IT##3EICBYG647BFUCW4RIWECV97C6TCJY##AJ7NSZOAFP05U##Blog_1423500##q1_a1##3MVY4USGB81WJY30GKVCQF21TKPSIW",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "1",
 "probabilities": {
 "0": 0.25576525926589966,
 "1": 0.6830384135246277,
 "2": 0.06027877330780029,
 "3": 0.0009175298037007451
 }
 },
 {
 "id": "cosmosqa/validation/30F94FBDNRI8H132FCWLMZ616PJBT4##3J4Q2Z4UTZ1QZ87FJ24OTS4NB62WQ3##AXJ1PE87PD0KK##Blog_131850##q1_a2##3S1L4CQSFZJ89BFGYC302IV2Q26FA5",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 1.0257926078338642e-05,
 "1": 0.9992479681968689,
 "2": 0.000506654498167336,
 "3": 0.00023513621999882162
 }
 },
 {
 "id": "cosmosqa/validation/3SCKNODZ0XENKCVS5ZYT8YEDY1RN7I##30ZX6P7VF9TN155VASV88KCCQSE2J8##A2KYOM2ZGYDJGT##Blog_1559772##q1_a1##3UY4PIS8QTZO3FFX0HTKZMB8ORZN1T",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.006526751443743706,
 "1": 0.980405330657959,
 "2": 0.010237259790301323,
 "3": 0.002830721437931061
 }
 },
 {
 "id": "cosmosqa/validation/3VJ4PFXFJ35IY4DZ93MZWK8X77PAUR##3GGAI1SQEWWQ0V3IAIX60PDF7VWMCG##AO33H4GL9KZX9##Blog_129230##q1_a1##3OEWW2KGQLP0DVVLHPDSFDCM3XODO3",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.242871960334014e-05,
 "1": 1.890346084110206e-06,
 "2": 0.9999727010726929,
 "3": 3.01285263049067e-06
 }
 },
 {
 "id": "cosmosqa/validation/3D1TUISJWKECYBV45PQC5GKODMYUI9##3EA3QWIZ4LA0LIE9VM64P4UBLZYTIM##A2KYOM2ZGYDJGT##Blog_263486##q1_a1##3XJOUITW8W5AFW4YIS5B9MP8LW1TQ5",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9812814593315125,
 "1": 0.006292679812759161,
 "2": 0.00813865102827549,
 "3": 0.004287285264581442
 }
 },
 {
 "id": "cosmosqa/validation/389A2A304QW55MXOF8G656YMKWJC03##3AZHRG4CU7ZKF6ZZCHB71NQH5FT03O##A29RROPO0Z90XV##Blog_131452##q1_a1##3FHTJGYT8PELAFHDYFRFAYK9Q6HGP2",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0006366978632286191,
 "1": 0.9717718958854675,
 "2": 0.0002125317696481943,
 "3": 0.027378806844353676
 }
 },
 {
 "id": "cosmosqa/validation/3G57RS03HH3C8MSGVNLCIQXJWGK525##36W0OB37HXCHGADHBZ11JI641J7HZB##A2KYOM2ZGYDJGT##Blog_1586309##q1_a1##34O39PNDK8MAVBPHVL3QEITRB81BRQ",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9991651773452759,
 "1": 0.0007668480975553393,
 "2": 6.746313738403842e-05,
 "3": 5.471345048135845e-07
 }
 },
 {
 "id": "cosmosqa/validation/3CIS7GGG65HLOHUBP8OLS3YOSP6UEV##3IAS3U3I0GEHJM7JM0IRAWTZJBRB2V##AO33H4GL9KZX9##Blog_1365914##q1_a1##3NKW03WTLOL27D8SDXAMW6W7PHBQW2",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 1.644704934733454e-05,
 "1": 0.011766234412789345,
 "2": 0.9881792068481445,
 "3": 3.803638173849322e-05
 }
 },
 {
 "id": "cosmosqa/validation/3WGZLY9VCHKIYE9YQACWQTYEV2S8DB##3ZGVPD4G6UF7LJ4CBTIQX19SCYFZTM##AO33H4GL9KZX9##Blog_898103##q1_a1##3QHITW7OYQN5005PCSEUEHZGX66QA5",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "1",
 "probabilities": {
 "0": 0.15254074335098267,
 "1": 0.6699817776679993,
 "2": 0.025585057213902473,
 "3": 0.15189246833324432
 }
 },
 {
 "id": "cosmosqa/validation/3UQVX1UPFSFDDF6FYHF01V1SFYE020##3T111IHZ5FOFZD9736JWKY6H7C69RI##AO33H4GL9KZX9##Blog_521656##q1_a1##3R4QIDVOJRQJTAWWJ6LGGJ0TZBHEEZ",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0642036646604538,
 "1": 0.9356842637062073,
 "2": 6.119133468018845e-05,
 "3": 5.085170050733723e-05
 }
 },
 {
 "id": "cosmosqa/validation/3WGZLY9VCJ0XS385A8OWIRMMOSM8DX##3DIP6YHAPF757TKZW55IDT0WFY98EG##A2KYOM2ZGYDJGT##Blog_111938##q1_a1##3WJGKMRWVKNZ46YIRK41KDI5EVICD6",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 7.947526682983153e-06,
 "1": 1.4616551879953477e-07,
 "2": 2.8900672077725176e-07,
 "3": 0.9999916553497314
 }
 },
 {
 "id": "cosmosqa/validation/3WJGKMRWVI7KAHZB7MS1SFUXLQVDCY##3ZWFC4W1UV5O0N0K6SJOUUSSC1YRF9##A394JO4NEPCY3M##Blog_276312##q1_a1##368IUKXGA7MJDMS7M2ITY77KWR7P60",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9917653203010559,
 "1": 0.008059638552367687,
 "2": 0.00010298701090505347,
 "3": 7.208478200482205e-05
 }
 },
 {
 "id": "cosmosqa/validation/3AFT28WXLHGHL6B793E36ASZVGTIOZ##3MD9PLUKKLTOLVYZYU8EG0R9UPBZND##AJ7NSZOAFP05U##Blog_1134186##q1_a1##33KGGVH24WVTF9T12NGTXO27RX0X11",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.45315316319465637,
 "1": 0.0011034124763682485,
 "2": 0.0008330357377417386,
 "3": 0.5449104309082031
 }
 },
 {
 "id": "cosmosqa/validation/302U8RURJ1F4PRVV7VD96AIODWANVO##369J354OFGPGNTX29LY8TIPHMNE6GX##A2LTOS0AZ3B28A##Blog_1206213##q1_a1##3UUSLRKAUNHTSP3M03WQ4RQRDKW7DD",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 4.106264896108769e-06,
 "1": 3.165243469993584e-05,
 "2": 0.9998868703842163,
 "3": 7.742377056274563e-05
 }
 },
 {
 "id": "cosmosqa/validation/31GN6YMHLPQZQ4OYL3NRZ8N0E7MSWB##3AZHRG4CU5I5KH0SSKZ79P2808F036##AXJ1PE87PD0KK##Blog_1264909##q1_a1##37SDSEDINBG308STUF41HPLDZ4N81T",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 6.513846892630681e-05,
 "1": 1.208196954394225e-05,
 "2": 8.784821693552658e-05,
 "3": 0.9998348951339722
 }
 },
 {
 "id": "cosmosqa/validation/3Q7TKIAPOT8DDLTHKZL0VYKKFT9LDA##3U84XHCDIDBN4YPMWSHIPTX0VWR4ZH##A1OPGPTQJ48XOY##Blog_1133050##q1_a1##3SD15I2WD48C6XNKO4LML44JRTH633",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.00012209097621962428,
 "1": 0.9992591738700867,
 "2": 0.00016892184794414788,
 "3": 0.00044980942038819194
 }
 },
 {
 "id": "cosmosqa/validation/3SR6AEG6W77TJP4P985G8PPM9AAHY7##3YMTUJH0DVV6DBFUAX40V3KSH10T4F##A3GVXFBYAU9HP##Blog_1283884##q1_a1##3OZ4VAIBEZT8614ASXLHD3174P0VJD",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.002653766656294465,
 "1": 8.08246259111911e-05,
 "2": 0.9961748123168945,
 "3": 0.0010905925882980227
 }
 },
 {
 "id": "cosmosqa/validation/3VP28W7DUNIUGUOHQ67ZRDY36JWZFP##3VNXK88KKDG7SKQOCQO6LVVKUV79VR##AJ7NSZOAFP05U##Blog_994777##q1_a1##3RQVKZ7ZRMX6PF563EC988ZNGQV72M",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.998060405254364,
 "1": 2.1018017832830083e-06,
 "2": 2.3976575903361663e-05,
 "3": 0.00191350351087749
 }
 },
 {
 "id": "cosmosqa/validation/3XD2A6FGFNSOE8BQSSDZLOFPG229SV##320DUZ38G8KDGZD297GXAO6D20JGJL##AJ7NSZOAFP05U##Blog_812607##q1_a1##3NBFJK3IOJW3PF5CJFE0LYFJGYFOGH",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9996005892753601,
 "1": 8.639303268864751e-05,
 "2": 8.508299652021378e-05,
 "3": 0.00022794333926867694
 }
 },
 {
 "id": "cosmosqa/validation/3511RHPADXSM8KBTMYSKSLPWN0SRL2##3LEP4MGT3JFJHL463ME8YD8COSWBDT##A2KYOM2ZGYDJGT##Blog_1243077##q1_a1##3UUSLRKAUNHTSP3M03WQ4RQRDMQD7H",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 6.0669910453725606e-05,
 "1": 0.06295261532068253,
 "2": 0.0006475939298979938,
 "3": 0.9363391399383545
 }
 },
 {
 "id": "cosmosqa/validation/3HUR21WDDW3Q5F44ECN2BBDGX0SYXG##3R2PKQ87NZNW8NOPMP5S3QZJSABIMM##AJ7NSZOAFP05U##Blog_402735##q1_a1##3CVBMEMMXDUA458OMZFGVC5P5FPH72",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0001316611305810511,
 "1": 0.999531626701355,
 "2": 9.063818288268521e-05,
 "3": 0.0002460945979692042
 }
 },
 {
 "id": "cosmosqa/validation/338GLSUI45PC971AFCPRNIP8CZ9FSF##3COPXFW7XERTZLBQE8IWXA8RO52PKY##A2KYOM2ZGYDJGT##Blog_804711##q1_a1##3T8DUCXY0PK4NXNC1QUPNRZJBW39T5",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.48031696677207947,
 "1": 0.5155342221260071,
 "2": 0.0014152862131595612,
 "3": 0.0027335514314472675
 }
 },
 {
 "id": "cosmosqa/validation/3TL87MO8CMNAB57DB5GQJYN5R4VFLH##324G5B4FB46NL01FUW2XM6EO4HI70C##A2KYOM2ZGYDJGT##Blog_789838##q1_a1##3MNJFORX8DIB353Z3GCJU7FGYNUF5M",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.0464998584657224e-08,
 "1": 3.7644772419298533e-06,
 "2": 0.9999942779541016,
 "3": 1.977832880584174e-06
 }
 },
 {
 "id": "cosmosqa/validation/3S829FDFT2Z8DFA6YIOO6SF7MWNDXG##39KFRKBFIOTR31PZJQU8PKU35VPYOZ##A1OPGPTQJ48XOY##Blog_1513301##q1_a1##3EQPA8A376APFL9KTUAKY9E3SPMZJB",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.005041089374572039,
 "1": 0.004279397893697023,
 "2": 0.02349771186709404,
 "3": 0.9671818614006042
 }
 },
 {
 "id": "cosmosqa/validation/3MD8CKRQZZL1J5OMZF47K8OWQ0KRJZ##3XUHV3NRVLWJ9WTS4BULUEZ9ZBA5HK##AJ7NSZOAFP05U##Blog_524139##q1_a1##375VMB7D4LXQH9KIJU0NVGG0RSRDII",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9994077682495117,
 "1": 0.0002855915226973593,
 "2": 0.0002638495643623173,
 "3": 4.278791311662644e-05
 }
 },
 {
 "id": "cosmosqa/validation/3VCK0Q0PO5C921V8K0DKTJ4QQS40NO##3XC1O3LBOTKNGUN8WSRHPMYUVJELTI##A2KYOM2ZGYDJGT##Blog_245504##q1_a1##3UEDKCTP9X416V9WSKID84LHXKAK72",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.9885566732446023e-07,
 "1": 3.5868545182893286e-06,
 "2": 0.9996718168258667,
 "3": 0.0003242638777010143
 }
 },
 {
 "id": "cosmosqa/validation/39N6W9XWRF1FJT9EP2NYNPDLU8YYGM##3HMVI3QICM7L2YH5XL8X5M54K7V1YY##A2KYOM2ZGYDJGT##Blog_307489##q1_a1##3I01FDIL6OMM6O6GURFV809J26RD2O",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "0": 0.00023430815781466663,
 "1": 0.0238335020840168,
 "2": 0.025405757129192352,
 "3": 0.9505264759063721
 }
 },
 {
 "id": "cosmosqa/validation/3RQVKZ7ZRKHRVQ6ZJG09GABFNBI723##352YTHGROWB1DDPSQ6BBPMTVY59H40##A1OPGPTQJ48XOY##Blog_320648##q1_a1##3DGDV62G7QNVVL96Z06Q8UQ8GEUP2S",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0033241144847124815,
 "1": 0.993703305721283,
 "2": 0.0016935201128944755,
 "3": 0.0012790036853402853
 }
 },
 {
 "id": "cosmosqa/validation/3PUOXASB59H099WB7IJ79F00VRC9Z5##3NLZY2D53S4PJ3UNZL9MCFBGHMLQLQ##A2KYOM2ZGYDJGT##Blog_113795##q1_a1##3K3G488TR4M8UUT8HWND66JU5Q25Q7",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9872485995292664,
 "1": 0.01213254127651453,
 "2": 0.0005587802152149379,
 "3": 6.005796603858471e-05
 }
 },
 {
 "id": "cosmosqa/validation/30OITAWPBQ1ZOZXAHY10HA8N7OB9HU##3TPWUS5F8AZM57XWEK3T9NF66Q4WCY##A2KYOM2ZGYDJGT##Blog_312674##q1_a1##371DNNCG46G49S42WYOBNV48RLOT8T",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9967067837715149,
 "1": 0.002466936130076647,
 "2": 0.00015895537217147648,
 "3": 0.0006673291209153831
 }
 },
 {
 "id": "cosmosqa/validation/3PA41K45VPI2HM6C5CNL4L7GRVI7PU##3EF8EXOTT4AV79M6BL7UVZQG6LRJ1Q##A394JO4NEPCY3M##Blog_861482##q1_a2##3SZYX62S5IEYOCOTLXE76F2Z04957G",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9932621121406555,
 "1": 0.0014329752884805202,
 "2": 0.0006831057253293693,
 "3": 0.004621842410415411
 }
 },
 {
 "id": "cosmosqa/validation/3UOMW19E6D4P64KISGAIU36589X5CI##33NF62TLXK0IIL9LZY1C8GY6AITKJO##A2KYOM2ZGYDJGT##Blog_423706##q1_a1##38RHULDV90TPQFK90P5DN45LYF5IWI",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.00036453193752095103,
 "1": 7.038464536890388e-05,
 "2": 3.650389430731593e-07,
 "3": 0.9995647072792053
 }
 },
 {
 "id": "cosmosqa/validation/3RHLQY6EDVIW92L3Y5YJ1F6HMKLD4I##3R8YZBNQ9KXQ4I5LVR8CTATFP9W7QV##AJ7NSZOAFP05U##Blog_726068##q1_a1##3AA88CN98RHKLF5XFT8DLH5F29UKYX",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "0": 1.9020246782019967e-06,
 "1": 1.3034659218646993e-07,
 "2": 0.001470787450671196,
 "3": 0.9985271692276001
 }
 },
 {
 "id": "cosmosqa/validation/3M93N4X8HKLDKKOZE44CAITSC23SJO##3SKRO2GZ72PBN4TH99V64G2KFC2K1E##AJ7NSZOAFP05U##Blog_1354557##q1_a1##3XUY87HIVRSFHMGMH2XI6ZABHF3MMP",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 1.9972836525994353e-05,
 "1": 1.1256567631789949e-05,
 "2": 0.9999676942825317,
 "3": 1.0070043572341092e-06
 }
 },
 {
 "id": "cosmosqa/validation/37SDSEDINBG308STUF41HPLE0FX81S##3TXMY6UCAH3WGZ0HM8GN9WLTQ23CQ9##A3GVXFBYAU9HP##Blog_125732##q1_a1##3O71U79SRD3IFHTR1ZT4I89IAT9MS6",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9497283697128296,
 "1": 0.00033410644391551614,
 "2": 0.049827076494693756,
 "3": 0.00011038452794309705
 }
 },
 {
 "id": "cosmosqa/validation/3OPLMF3EU5LMZXKU9TZ3R3WETSBNLK##39GHHAVOMGPX4JKIE6EW1Z9ACD8J4Z##AJ7NSZOAFP05U##Blog_19308##q1_a1##3T6SSHJUZHO7PK04B81YMDWDJZ2IIW",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.09780813008546829,
 "1": 0.24027372896671295,
 "2": 0.15066102147102356,
 "3": 0.5112571716308594
 }
 },
 {
 "id": "cosmosqa/validation/3X0EMNLXEPNPYN813CE1I1ZBKEDVPO##3MRNMEIQW64GZ5RBA2R2KIG8UIWLDD##A2KYOM2ZGYDJGT##Blog_1294197##q1_a1##3PKJ68EHDPA3O09QN84DY2AYW4CJHM",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 3.5643674323182495e-07,
 "1": 0.0031246505677700043,
 "2": 0.9967435598373413,
 "3": 0.00013140449300408363
 }
 },
 {
 "id": "cosmosqa/validation/375VSR8FVW7RJKTQH17T82CM9RSZR9##3A4TN5196LG48H27JLHT8K5JPR5CH4##A2KYOM2ZGYDJGT##Blog_69090##q1_a1##3BC9H1KCYWQBGFZE47RG9SZ9OOBWYQ",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999874830245972,
 "1": 9.191267963615246e-06,
 "2": 2.693502892725519e-06,
 "3": 5.858954637005809e-07
 }
 },
 {
 "id": "cosmosqa/validation/3XH7ZM9YX2SCDTBDD3DUVEATU6JR91##3IXEICO793H5X9K19OLF03GX7Q0T6Z##A2KYOM2ZGYDJGT##Blog_1205090##q1_a1##3WRKFXQBODLTFKF9FQN181Q19N5YIX",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0018480167491361499,
 "1": 0.9980676770210266,
 "2": 4.737342896987684e-05,
 "3": 3.704684422700666e-05
 }
 },
 {
 "id": "cosmosqa/validation/36AZSFEYZ4YNUT7RK7W0NHUM3WJBVC##36ZN444YTSW5D1AUF8853W0FF2YOI6##A1S1K7134S2VUC##Blog_448285##q1_a1##33N1S8XHHOZIK4TG354EU9IHY4P1Z4",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.014256046153604984,
 "1": 0.8001266121864319,
 "2": 0.029887013137340546,
 "3": 0.15573033690452576
 }
 },
 {
 "id": "cosmosqa/validation/3OWZNK3RYN38NQH2GM5KI0D8PC2U2A##3H7Z272LXAM4JRT8Z8UXCAL6FBAPLI##A33PQXKQHFVF3V##Blog_205280##q1_a1##3WUVMVA7ODHEES6GZOX75ABL3RLZA5",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.00021653641306329519,
 "1": 6.5253025240963325e-06,
 "2": 1.2672788329837203e-07,
 "3": 0.9997767806053162
 }
 },
 {
 "id": "cosmosqa/validation/3LN50BUKPV9M2IW7P8CVVS19TLULPO##3XIQGXAUMD6VIQ7QX8R4VK1ZF1G7XB##A1OPGPTQJ48XOY##Blog_695838##q1_a1##30ZKOOGW2YKHJWL8RDOFDGR8PUFA1V",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 3.4659972243389348e-06,
 "1": 1.1123542208224535e-05,
 "2": 0.008735344745218754,
 "3": 0.9912500381469727
 }
 },
 {
 "id": "cosmosqa/validation/3S1WOPCJFI7R4QIOX7W7DZDL35EEJS##3NC5L260MR10YZ7BYXEKSXBW4WQFOO##A2KYOM2ZGYDJGT##Blog_438974##q1_a1##3HEA4ZVWVF0J18AAX4ZA8VO39RZ558",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 2.0503218820522306e-06,
 "1": 9.930379746947438e-05,
 "2": 2.969313754874747e-07,
 "3": 0.9998983144760132
 }
 },
 {
 "id": "cosmosqa/validation/30P8I9JKOIJL5DQIJX5NKVUUTED5VD##3U5NZHP4LS0N26BBOQ3ZSK4QRLWHPO##A2MHM7JK1T9XHR##Blog_293009##q2_a1##3G57RS03HJJR2BRNFLXCAOLRPK0252",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 7.164778071455657e-05,
 "1": 2.9059294320177287e-05,
 "2": 0.9971911311149597,
 "3": 0.002708143787458539
 }
 },
 {
 "id": "cosmosqa/validation/3MGHRFQY2LN3E0ZMR2MVAZ2XUSN0YP##3R2UR8A0IBE23WZPEU8666NU5R6OX0##A1OPGPTQJ48XOY##Blog_669043##q1_a1##3L55D8AUFCB22HT9NGRFXYS6WI8YCI",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 3.168620651194942e-06,
 "1": 3.03104843624169e-06,
 "2": 1.3777472531728563e-06,
 "3": 0.9999924898147583
 }
 },
 {
 "id": "cosmosqa/validation/3T8DUCXY0N4PT8O5HSIPVTBBIG59TZ##3FPRZHYEPZ5LDI18VHNWUISYSSW3VF##A2KYOM2ZGYDJGT##Blog_896843##q1_a1##3PEG1BH7AG5O87CM51UTD67DGB3KBS",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.6398298740386963,
 "1": 0.314551442861557,
 "2": 0.020746398717164993,
 "3": 0.024872232228517532
 }
 },
 {
 "id": "cosmosqa/validation/3J94SKDEKK3E5LP3CAHT67CVMIJD5U##3OONKJ5DKFYALIVVTVSO97KBT1GOB7##AJ7NSZOAFP05U##Blog_1154002##q1_a1##379OL9DBSUS04JM6BWHOV8RZRFVY9U",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.0002194041881011799,
 "1": 0.12652957439422607,
 "2": 0.000756067456677556,
 "3": 0.8724949359893799
 }
 },
 {
 "id": "cosmosqa/validation/3YCT0L9OMM793AFKT2W43PA9LTVNSO##3COPXFW7XCAE4WCJUB6W5CKIJUZPKJ##A33PQXKQHFVF3V##Blog_1239641##q1_a1##3XH7ZM9YX48R7IAKX1PUNCY1N8V9RB",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "0",
 "probabilities": {
 "0": 0.6152780652046204,
 "1": 2.400243465672247e-05,
 "2": 0.38458728790283203,
 "3": 0.00011066674051107839
 }
 },
 {
 "id": "cosmosqa/validation/32TMVRKDGNW5YIQ5XZB3NP48I9K84N##32AT8R96GM7PKKXNG6ES64YR6PBUSA##AJ7NSZOAFP05U##Blog_998147##q1_a1##3MDKGGG61S1H5ELE3JRD3H8HLMHT6I",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 6.584975199075416e-05,
 "1": 3.450539907134953e-06,
 "2": 4.911045721200935e-07,
 "3": 0.9999301433563232
 }
 },
 {
 "id": "cosmosqa/validation/3N3WJQXELSOQABV1NA1JZFTGIX92LX##3DEL4X4EL7JEXA3RKHH48XLWH5YXYJ##A33PQXKQHFVF3V##Blog_149550##q1_a1##3XU9MCX6VQQG7YPLCSAFDPQNHXL2R2",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9988299012184143,
 "1": 3.5946657590102404e-05,
 "2": 0.0011123460717499256,
 "3": 2.1851315977983177e-05
 }
 },
 {
 "id": "cosmosqa/validation/3S37Y8CWIAEVI8LUF039T3V2TK94W4##3H0W84IWBNHBPYZVVEBI4AV15KUERJ##AJ7NSZOAFP05U##Blog_698241##q1_a1##3HXCEECSQO7FAA4JFUBN8WTR514ZYU",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 6.074617431295337e-06,
 "1": 8.417957360506989e-06,
 "2": 0.9999762773513794,
 "3": 9.266219421988353e-06
 }
 },
 {
 "id": "cosmosqa/validation/336OE47KI2IFIW52FYSWAV77D46VWV##36ZN444YTSW5D1AUF8853W0FGTQOIH##AO33H4GL9KZX9##Blog_853628##q1_a1##3DIIW4IV8R9DHD1PQJZUSCMUEICI4U",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.030188726261258125,
 "1": 0.8612430691719055,
 "2": 0.08949704468250275,
 "3": 0.019071273505687714
 }
 },
 {
 "id": "cosmosqa/validation/3UEDKCTP9X416V9WSKID84LHWSBK7I##3OS4RQUCRCUGF5I2US9YMDBXYQ9BF5##AJ7NSZOAFP05U##Blog_1347993##q1_a1##3VMV5CHJZATJ27AKFW0ZF8L6F6DTGP",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.00809777993708849,
 "1": 0.030700786039233208,
 "2": 0.9335407018661499,
 "3": 0.02766076661646366
 }
 },
 {
 "id": "cosmosqa/validation/3909MD9T21VBZOWSEH5MY1LTAYYEFQ##3EWIJTFFVRMNP4FW11S3KEUEUTUE0G##A2KYOM2ZGYDJGT##Blog_815344##q1_a1##3LB1BGHFL4AJOYVK92M5M8SAKZPYT8",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9999949932098389,
 "1": 1.7484851468907436e-06,
 "2": 3.1401766591443447e-06,
 "3": 1.793309110098562e-07
 }
 },
 {
 "id": "cosmosqa/validation/3ZCC2DXSD720BVV7ZAX42RWH651YYN##3WYGZ5XF3XDBHOSBMSAQ0HPT6AMKS2##AO33H4GL9KZX9##Blog_241726##q1_a1##3RSBJ6YZEE4XBF9H8V8I3DF7FL1OF8",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.00011201051529496908,
 "1": 4.872589124715887e-05,
 "2": 1.1307697889151314e-07,
 "3": 0.9998390674591064
 }
 },
 {
 "id": "cosmosqa/validation/3UXQ63NLAAKKYOVX63UU08WKC5NBLD##333U7HK6IADA4F3B7HFP433GPWHJD4##A1OPGPTQJ48XOY##Blog_650589##q1_a1##3XABXM4AJ3JRHCCAFUUGIQTL7EPQ8C",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 4.534982144832611e-05,
 "1": 0.999908447265625,
 "2": 4.246336538926698e-05,
 "3": 3.7319025523174787e-06
 }
 },
 {
 "id": "cosmosqa/validation/38VTL6WC4CRH3WKU8647JSMW3GOY55##3DY4FPOOA43SR9LHQ40S4EJN2HHRVN##AISNLDPD2DFEG##Blog_262078##q2_a1##3X4Q1O9UBJ0KWMUBQCA6FL2QHSY7OT",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.002637383760884404,
 "1": 0.9971548318862915,
 "2": 5.8096833527088165e-05,
 "3": 0.00014971192285884172
 }
 },
 {
 "id": "cosmosqa/validation/3LG268AV3AK82E115C5GFQZCD79ERR##3OS4RQUCRCUGF5I2US9YMDBXY8EBFA##A33PQXKQHFVF3V##Blog_1016062##q1_a1##3XD2A6FGFP838XAXCQPZDM3X9A5S99",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 8.922739880290464e-08,
 "1": 7.777929567964748e-06,
 "2": 4.650118171412032e-06,
 "3": 0.9999874830245972
 }
 },
 {
 "id": "cosmosqa/validation/3D4BBDG7ZHUMSPXEEXMPF68LUD6C3Z##3T3IWE1XG7LY7R3LOXZD68XO7QVTQD##AO33H4GL9KZX9##Blog_1088071##q1_a1##3RHLQY6EDVIW92L3Y5YJ1F6HMHZ4DH",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "3",
 "probabilities": {
 "0": 0.0027840123511850834,
 "1": 0.20919080078601837,
 "2": 0.00010020814806921408,
 "3": 0.7879250049591064
 }
 },
 {
 "id": "cosmosqa/validation/3G4VVJO6PZEBICEP480UGSOU42QKPJ##392CY0QWG2P1980CWOTWPYUAYADI43##AJ7NSZOAFP05U##Blog_6092##q1_a1##3R868ACW4UQFOJJJLOPH1WMN0T7ZG1",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9945017099380493,
 "1": 7.429063407471403e-05,
 "2": 0.00041008542757481337,
 "3": 0.005013927351683378
 }
 },
 {
 "id": "cosmosqa/validation/3YZ7A3YHR77K7A8L90CI16YVNLTS53##3OSWBBLG1HCQUO778I6QNAZ48QQDX1##AJ7NSZOAFP05U##Blog_220597##q1_a1##3EAWOID6MVB14V3O6PBBVYMC6UZ0VB",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 4.9596837925491855e-06,
 "1": 8.409543806919828e-05,
 "2": 4.730381419904006e-07,
 "3": 0.9999103546142578
 }
 },
 {
 "id": "cosmosqa/validation/3YLTXLH3DHKZABCOIL9XVYWA5W0PHW##37TRT2X24T6WK74YDI0J1B3PSOOBJP##AJ7NSZOAFP05U##Blog_386113##q1_a1##36MUZ9VAE8GE14IWNBAWA1I8Z7ZDE4",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9996848106384277,
 "1": 4.06145591114182e-05,
 "2": 1.8888378008341533e-06,
 "3": 0.0002726606326177716
 }
 },
 {
 "id": "cosmosqa/validation/3CVBMEMMXBEVAG9H213G3EHHCWM7HB##33FOTY3KENJT46HTHWE8GXPCA30C10##AO33H4GL9KZX9##Blog_1245107##q1_a1##3YGYP13643NCWKEEYB8Z5DTLSSBNRD",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.08666516840457916,
 "1": 0.006114616058766842,
 "2": 0.6894660592079163,
 "3": 0.21775418519973755
 }
 },
 {
 "id": "cosmosqa/validation/35XW21VSVGCJFACZHPGPQX6KGK3LS0##37UQDCYH6YT3E12VXTQM7Z4EX637VX##A2KYOM2ZGYDJGT##Blog_731078##q1_a1##3NSCTNUR211G7EMRS0QRG64P1055A9",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9992249011993408,
 "1": 0.00021941066370345652,
 "2": 0.0004790797538589686,
 "3": 7.663273572688922e-05
 }
 },
 {
 "id": "cosmosqa/validation/32W3UF2EZOJ7ALGIEBR8PB0R8GL4C1##3EF8EXOTT2TGCKNZROVU31271IH1J2##A2KYOM2ZGYDJGT##Blog_1165565##q1_a1##3DQYSJDTYNPSZANDEBB848L0FOSEXM",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 5.6724086761050785e-08,
 "1": 2.47463916025481e-08,
 "2": 0.9999077320098877,
 "3": 9.223623783327639e-05
 }
 },
 {
 "id": "cosmosqa/validation/3MYASTQBG9P3Y781W25W4I7MI27DQK##3IUZPWIU1RMJJU8RQUWRP4GNHHUWKJ##A33PQXKQHFVF3V##Blog_255787##q1_a1##3J6BHNX0UB6Q9G1J8RZ24177X98KNQ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 5.354191898732097e-07,
 "1": 1.4843629969618632e-07,
 "2": 6.162710519674874e-08,
 "3": 0.9999992847442627
 }
 },
 {
 "id": "cosmosqa/validation/375VMB7D4LXQH9KIJU0NVGG0QEOIDR##3XCC1ODXDOQ0M1P0456IYRKFLP9QRZ##AO33H4GL9KZX9##Blog_174449##q1_a1##3ULIZ0H1VCJKDQ9RWHXY55BMXBU51Q",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "0",
 "probabilities": {
 "0": 0.9673277139663696,
 "1": 0.0002956792013719678,
 "2": 0.030750926584005356,
 "3": 0.0016256991075351834
 }
 },
 {
 "id": "cosmosqa/validation/3Q7TKIAPOT8DDLTHKZL0VYKKFOYLDP##3A7Y0R2P2PMO2L8SYHG1YA6840QJXA##AJ7NSZOAFP05U##Blog_665743##q1_a1##31N9JPQXIRWZ7Q0B26W5R9RF5JDHNA",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "2",
 "probabilities": {
 "0": 3.026151898666285e-05,
 "1": 0.002800097456201911,
 "2": 0.9918596148490906,
 "3": 0.005310038570314646
 }
 },
 {
 "id": "cosmosqa/validation/3FO95NVK5EE9CX6J67PL473RFRISRT##3HQUKB7LNITAKECUPA7UB9Q8A97HHO##AJ7NSZOAFP05U##Blog_24696##q1_a2##3HKIF5DF60B0IS16HQVIO8KUL3B9GE",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.0006900756270624697,
 "1": 0.00013070380373392254,
 "2": 0.0014030757592990994,
 "3": 0.9977760910987854
 }
 },
 {
 "id": "cosmosqa/validation/3R0WOCG21ONVQ0FIDBP1EJ4BK9CDUV##3TE22NPXPERMWQ04VW3SNV80YE6442##AO33H4GL9KZX9##Blog_1442209##q1_a1##3AQN9REUTHUC79ZNNCMQH4AOHD1DY4",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 1.8586488295113668e-06,
 "1": 4.0646338561600714e-07,
 "2": 7.905052916612476e-05,
 "3": 0.9999186992645264
 }
 },
 {
 "id": "cosmosqa/validation/35XW21VSVGCJFACZHPGPQX6KGWJLS4##3QFUFYSY9ZDHZHYMVGNBM1HIJTIF4T##A1OPGPTQJ48XOY##Blog_337539##q1_a1##3Z56AA6EK6EDSJVUUMQ1H3DPRKY6MJ",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9984683394432068,
 "1": 0.00012271992454770952,
 "2": 0.0004971851594746113,
 "3": 0.0009117599693126976
 }
 },
 {
 "id": "cosmosqa/validation/3S8APUMBJZXDSLK84Q3WXTF889ZBFI##36V4Q8R5ZNF9PLC8ZXA1CSQ5I1UQMW##A2KYOM2ZGYDJGT##Blog_1376968##q1_a1##3FJ2RVH251KA3YTZIBATM4QTBJL928",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 8.039963722694665e-05,
 "1": 9.276366199628683e-07,
 "2": 3.753206101464457e-07,
 "3": 0.9999182224273682
 }
 },
 {
 "id": "cosmosqa/validation/3CIS7GGG67X0I6TI960LK1MWK6DEUV##32AT8R96GOO4F9WU03QSY2M0C1TSU5##AO33H4GL9KZX9##Blog_1125178##q1_a1##371QPA24C42MYSKHXDFW8LH0DHVT1Q",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0009758514352142811,
 "1": 0.9990222454071045,
 "2": 1.8174113165514427e-06,
 "3": 1.2742462729420367e-07
 }
 },
 {
 "id": "multirc/validation/2136",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.4368184506893158,
 "no": 0.5631815195083618
 }
 },
 {
 "id": "cosmosqa/validation/3L2OEKSTWBO0QE3W64BAVCWN6N7Y8O##3YGXWBAF73WPRUDJOB9A6TKOTTSC4Z##AJ7NSZOAFP05U##Blog_292897##q1_a1##3QHITW7OYQN5005PCSEUEHZGX9LQAQ",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 5.03088904224569e-06,
 "1": 0.999518632888794,
 "2": 0.00045047298772260547,
 "3": 2.590316034911666e-05
 }
 },
 {
 "id": "cosmosqa/validation/3OID399FXG5RDTJLYOI20LF0IRRDFP##3QFUFYSY9ZDHZHYMVGNBM1HIIUTF45##A1OPGPTQJ48XOY##Blog_638929##q1_a1##30EV7DWJTX9DM0K8DPECBY0W0PY6YL",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.6655539870262146,
 "1": 0.14923039078712463,
 "2": 0.011539670638740063,
 "3": 0.17367598414421082
 }
 },
 {
 "id": "cosmosqa/validation/3YD0MU1NC2ZG0T13WVTOMSUDZ1WA73##3Q5ZZ9ZEVPDQGLSZ170FH16AVQ885R##AO33H4GL9KZX9##Blog_1157234##q1_a1##3OEWW2KGQLP0DVVLHPDSFDCM31PODN",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 2.4340620257135015e-06,
 "1": 0.9824087023735046,
 "2": 0.01758386939764023,
 "3": 5.074259206594434e-06
 }
 },
 {
 "id": "cosmosqa/validation/35A1YQPVFEEGJ76IKQ645640LCH5IG##3RSDURM96BK5RACAPCRCK80P3DOYE6##A394JO4NEPCY3M##Blog_134992##q1_a1##3H4IKZHALDWRHMMM54RIWTOBR0QNN3",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9986151456832886,
 "1": 0.0012412364594638348,
 "2": 7.632275810465217e-05,
 "3": 6.724696140736341e-05
 }
 },
 {
 "id": "cosmosqa/validation/3AC6MFV69MWTOFEU5U7ZMWYOEJJHZW##3N8OEVH1FU5U4CK4X0XAFRKAVZOOOT##A2KYOM2ZGYDJGT##Blog_376376##q1_a1##307FVKVSYTTGSTKT7SVX1NWL01Y47G",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 9.633629815652966e-06,
 "1": 2.9460437644956983e-07,
 "2": 1.0599100050967536e-06,
 "3": 0.9999890327453613
 }
 },
 {
 "id": "cosmosqa/validation/35F6NGNVM8HLFP0C2VK5HXK5NTY7T8##352YTHGROWB1DDPSQ6BBPMTVZYS4HT##AO33H4GL9KZX9##Blog_193555##q1_a1##3VZYA8PITQCK61O90YTYR0SCSL5053",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.999747097492218,
 "1": 0.00022028913372196257,
 "2": 2.7655425583361648e-05,
 "3": 4.969601832272019e-06
 }
 },
 {
 "id": "cosmosqa/validation/338GLSUI439XFI23VEDRVK10JVXSFW##3TR2532VIQSBJ6OWXU7BYXTO82V6JK##AO33H4GL9KZX9##Blog_246177##q1_a1##3SR6AEG6W77TJP4P985G8PPM95LHY8",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0001079261492122896,
 "1": 0.9998592138290405,
 "2": 2.189919541706331e-05,
 "3": 1.100526424124837e-05
 }
 },
 {
 "id": "cosmosqa/validation/39WICJI5AV6EU77VQ9IXI2XEWZVZ38##3EJPLAJKEPVGEAS7ZDMXEP5NYQQ6ZV##A3GVXFBYAU9HP##Blog_375803##q1_a2##3OB6JN3A9S36E2TEMATZSY4LNDIMRF",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 3.2604532407276565e-06,
 "1": 2.913067476129072e-08,
 "2": 0.9999473094940186,
 "3": 4.946822446072474e-05
 }
 },
 {
 "id": "cosmosqa/validation/3CMIQF80GNOPJ9UF2HLI6E9M8H96Q0##3E337GFOLA69ZP4NO58P3VNUCB0GNK##A2KYOM2ZGYDJGT##Blog_818773##q1_a1##3SCKNODZ0ZU2E1UZPXAT0W2LRLMN7T",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0019524152157828212,
 "1": 0.9978654980659485,
 "2": 3.3992386306636035e-05,
 "3": 0.00014811681467108428
 }
 },
 {
 "id": "cosmosqa/validation/3JYPJ2TAYI602QMB3X9X0FWPW9TFP2##3UWN2HHPUZ3CPUDEJ526S96XYXTSNZ##A394JO4NEPCY3M##Blog_952552##q1_a1##30Z7M1Q8U0YZUW5TQYXUSQEG3SMA80",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.001186884706839919,
 "1": 0.9985754489898682,
 "2": 0.00019200889801140875,
 "3": 4.565963536151685e-05
 }
 },
 {
 "id": "cosmosqa/validation/3VLL1PIENQMOM71AP9QPE77HYASOZL##3EG49X351VACEQTLEYMCP613ISLX6T##AO33H4GL9KZX9##Blog_907159##q1_a1##334ZEL5JX8T6FKE3EF8HACYM44WOSE",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.800376751110889e-05,
 "1": 0.00010743288294179365,
 "2": 0.9996547698974609,
 "3": 0.00020978662359993905
 }
 },
 {
 "id": "multirc/validation/2137",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.12604950368404388,
 "no": 0.8739505410194397
 }
 },
 {
 "id": "cosmosqa/validation/3WGZLY9VCJ0XS385A8OWIRMMO8KD8W##3FIJLY1B6XJIJZJCDXRZHXGMI4MPFP##A2KYOM2ZGYDJGT##Blog_467009##q1_a1##36JW4WBR08YNJYNU4IDSJ4I74HDHF6",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 7.478858066178873e-08,
 "1": 2.848248215059357e-08,
 "2": 8.063643122113717e-07,
 "3": 0.9999991655349731
 }
 },
 {
 "id": "cosmosqa/validation/3MGHRFQY2LN3E0ZMR2MVAZ2XUGU0Y8##3XM0HYN6NLXBIWK9YPVT7H7DY4DPEM##A33PQXKQHFVF3V##Blog_1337069##q1_a1##3MZ3TAMYTN1KIJ3N1VV705X46WWIRD",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 4.689000263624621e-07,
 "1": 7.448504391049937e-08,
 "2": 0.9999949932098389,
 "3": 4.514412921707844e-06
 }
 },
 {
 "id": "cosmosqa/validation/3TCFMTM8HEMDK4RXU0J0F0K2K0F12J##345LHZDEDYQL03C8QRDRYWQFGWNU35##AJ7NSZOAFP05U##Blog_958069##q1_a1##3SBX2M1TKF1QYBP83R431EDG5UJ4QD",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.0001565827988088131,
 "1": 0.005247338674962521,
 "2": 0.00023158414114732295,
 "3": 0.9943645000457764
 }
 },
 {
 "id": "cosmosqa/validation/38Z7YZ2SB5GK6IONOG67VFNZAE8QI6##37TD41K0AKO8TFFUEC5S4W0U7URSCO##A1OPGPTQJ48XOY##Blog_1577853##q1_a1##3IVKZBIBJ2NWN3SGA2DRQ31LST2SH0",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.009522161446511745,
 "1": 0.0005211959360167384,
 "2": 0.00026723393239080906,
 "3": 0.9896893501281738
 }
 },
 {
 "id": "cosmosqa/validation/31GN6YMHLR6EKTN551ZRR6B879EWSN##34HJIJKLP8BLQ3JJFS4FWL8DEWD4VN##A2KYOM2ZGYDJGT##Blog_146776##q1_a1##3UY4PIS8QTZO3FFX0HTKZMB8O4T1NR",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.14687922596931458,
 "1": 0.8520650863647461,
 "2": 0.000591700547374785,
 "3": 0.0004639750986825675
 }
 },
 {
 "id": "race/validation/3721",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 5.338060304893588e-07,
 "B": 6.64618710288778e-07,
 "C": 4.239701354435965e-07,
 "D": 0.9999983310699463
 }
 },
 {
 "id": "cosmosqa/validation/3IH9TRB0FDDWPE4631GSUEFR2P1I1Y##3VW04L3ZLWL4SUCOZI72HH2WM72XXJ##A33PQXKQHFVF3V##Blog_26990##q1_a1##3GONHBMNHXCFGTF5WRNCE16KWRQZM4",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "3",
 "probabilities": {
 "0": 0.265909343957901,
 "1": 0.00032207166077569127,
 "2": 7.030582764855353e-06,
 "3": 0.7337615489959717
 }
 },
 {
 "id": "cosmosqa/validation/3FTID4TN8LWGBWO8GV8E8JULXLFLYQ##36TFCYNS458SBFD2AHDGMXQTTNMXHF##A1I72NHC21347A##Blog_831152##q1_a1##3D17ECOUOG9HZBMNIY95Q8D2JIV13U",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9241178035736084,
 "1": 0.07394897937774658,
 "2": 0.00142049347050488,
 "3": 0.0005128168850205839
 }
 },
 {
 "id": "cosmosqa/validation/38G0E1M85OJHKHF44WPVK5AIKGFVUB##354P56DE9NI2HYKYNVXHOA8G2157SB##A2KYOM2ZGYDJGT##Blog_1291301##q1_a1##3L84EBDQ39GP1AGGBE7MC9Q337PKKT",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.00011503620044095442,
 "1": 0.9988125562667847,
 "2": 0.001055134809575975,
 "3": 1.7323665815638378e-05
 }
 },
 {
 "id": "race/validation/4157",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00019445105863269418,
 "B": 1.2371163393254392e-05,
 "C": 0.9996090531349182,
 "D": 0.00018415733939036727
 }
 },
 {
 "id": "cosmosqa/validation/3U18MJKL1W08QIPVGEI4GIUZSY1NCE##3KB8R4ZV1HMMT5EX5CFG4YB91Z8BGT##A3GVXFBYAU9HP##Blog_1168650##q1_a2##3APP19WN73SD09Z8JJS64YTSV4T6GU",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "0": 0.0002115765237249434,
 "1": 0.9799172282218933,
 "2": 0.010247763246297836,
 "3": 0.009623522870242596
 }
 },
 {
 "id": "cosmosqa/validation/338GLSUI439XFI23VEDRVK10J9FFST##3AMYWKA6YCKPKHFTD7ZG6TAA8MVO6S##A33PQXKQHFVF3V##Blog_1363740##q1_a1##3VGET1QSZ2DS1V3DHY1GWFFCKZHW7O",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 2.2360195828241558e-07,
 "1": 1.0296245278595961e-07,
 "2": 0.9999885559082031,
 "3": 1.1091088708781172e-05
 }
 },
 {
 "id": "cosmosqa/validation/3LG268AV3AK82E115C5GFQZCE55ERK##3Z2R0DQ0JKTULEIAGOWH32NTU2D2EC##A1OPGPTQJ48XOY##Blog_1204207##q1_a1##3ZXV7Q5FJD2W2QCEIQZR8DLQQAKFCA",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9989773035049438,
 "1": 0.0007360613089986145,
 "2": 0.0002496562956366688,
 "3": 3.707735595526174e-05
 }
 },
 {
 "id": "multirc/validation/2899",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.000705420330632478,
 "no": 0.9992945194244385
 }
 },
 {
 "id": "cosmosqa/validation/39WSF6KUV2JQBCAETMCCTX6ZA3IEO2##3YMU66OBIO6HPTICL7DCYBQYO7THGS##AO33H4GL9KZX9##Blog_863061##q1_a1##35NNO802AXAZ3LBV8OEQRLSU5VQIN0",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9992749094963074,
 "1": 0.0001760896120686084,
 "2": 2.455537651258055e-05,
 "3": 0.0005245264037512243
 }
 },
 {
 "id": "cosmosqa/validation/3QXFBUZ4ZKEJ8VQ2V3E9S6J05L0UGN##338JKRMM27XGF259Z7L22PFVSKHHAZ##A1OPGPTQJ48XOY##Blog_39999##q1_a1##3DZKABX2ZKJLYNY4I54VQZOCE17CVO",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "3",
 "probabilities": {
 "0": 0.13871468603610992,
 "1": 0.013094265945255756,
 "2": 0.28532111644744873,
 "3": 0.5628699660301208
 }
 },
 {
 "id": "cosmosqa/validation/3HFWPF5AKBXHTVLHVHOPM90B8Z1S3V##3VW6495TLMFIS4S89AF6J9GENLHYYO##AJ7NSZOAFP05U##Blog_1522633##q1_a2##30UZJB2POJQGNTN6JLBHTYQJVV853I",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.00035150963230989873,
 "1": 2.8402295356499963e-05,
 "2": 0.9991194605827332,
 "3": 0.0005006862338632345
 }
 },
 {
 "id": "cosmosqa/validation/3DW3BNF1GJWJMK8R1TOQODSTO3OV8P##3TEM0PF1Q8CIXY1W56HJEWCTW23D04##A2KYOM2ZGYDJGT##Blog_1416997##q1_a1##3BA7SXOG1L4RTDRAS9JGNG3Q363R82",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.06264932453632355,
 "1": 0.00031756452517583966,
 "2": 0.9369142055511475,
 "3": 0.00011895911302417517
 }
 },
 {
 "id": "multirc/validation/2871",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001752563752233982,
 "no": 0.998247504234314
 }
 },
 {
 "id": "cosmosqa/validation/37M4O367VJG2FQW9554WXZBH2N1M5H##37Z929RLGA6AK7IYG1OI18CPPF7STU##AXJ1PE87PD0KK##Blog_173738##q1_a2##3D0LPO3EADD3B6QCDL06SYMNLW5OYJ",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.00010851156548596919,
 "1": 9.132945706369355e-07,
 "2": 0.9998422861099243,
 "3": 4.8313399020116776e-05
 }
 },
 {
 "id": "cosmosqa/validation/3KL228NDMVK55ZLQ1PA7MJA23JQGKE##3Q8GYXHFEQ0SSOICIJGKJN2TM045C8##AXJ1PE87PD0KK##Blog_297053##q1_a1##3NCN4N1H1IVPGDXP2EM95YOHAA1BNP",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.000366921944078058,
 "1": 0.9980820417404175,
 "2": 0.0014702487969771028,
 "3": 8.077242819126695e-05
 }
 },
 {
 "id": "cosmosqa/validation/3KTZHH2ONKT0WXEIN1NZ1H9GXMC8M9##3LOTDFNYAAE838GDPJ5U9ROIB1NFW8##A2KYOM2ZGYDJGT##Blog_1132046##q1_a1##3EKTG13IZWHVB4Y1HQW36MVNIZYMLP",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 7.628162165929098e-06,
 "1": 6.162343197502196e-05,
 "2": 0.9728959798812866,
 "3": 0.027034709230065346
 }
 },
 {
 "id": "cosmosqa/validation/37ZQELHEQ2CLZ41MT3CIJEZBJBRNMD##3WRFBPLXRD3N02R6DND5CZMU7NAN3G##AO33H4GL9KZX9##Blog_1176058##q1_a1##3K8CQCU3KGFH4GV0X5I5SC12K1BNWP",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9991981387138367,
 "1": 0.00012996638542972505,
 "2": 0.0006656642653979361,
 "3": 6.286541065492202e-06
 }
 },
 {
 "id": "cosmosqa/validation/3A520CCNWNY198GDJ37US2MULOCAE1##3G0WWMR1UWI0CEY1R1D1X9RWHMQQNH##A2LTOS0AZ3B28A##Blog_985120##q2_a1##34OWYT6U3YVEEJ717EVL9KXA09F9I4",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0008351539145223796,
 "1": 0.9985999464988708,
 "2": 0.0005514205549843609,
 "3": 1.3482382200891152e-05
 }
 },
 {
 "id": "race/validation/4170",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9990664124488831,
 "B": 0.0005834865150973201,
 "C": 0.00027184851933270693,
 "D": 7.82284841989167e-05
 }
 },
 {
 "id": "cosmosqa/validation/3VMV5CHJZ8D48IBDVYOZNAXYM2BGTQ##34YB12FSQZM3H7JNC4TPQGYU7HUGMG##A1OPGPTQJ48XOY##Blog_252934##q1_a1##3S1WOPCJFI7R4QIOX7W7DZDL4OKEJ1",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 0.0011068895692005754,
 "1": 3.586715683923103e-05,
 "2": 0.998778760433197,
 "3": 7.844132778700441e-05
 }
 },
 {
 "id": "multirc/validation/1993",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9881979823112488,
 "no": 0.011801992543041706
 }
 },
 {
 "id": "cosmosqa/validation/3U18MJKL1UKTWTQOWG64OK6RZJXCNT##3HMIGG0U4M4OI92JC95CSY43R1B8YJ##A1OPGPTQJ48XOY##Blog_841160##q1_a1##3BDORL6HKMRT5GHN9LZLP2Z9TQ1CRS",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "1",
 "probabilities": {
 "0": 0.0016304438468068838,
 "1": 0.6212226748466492,
 "2": 0.11905483901500702,
 "3": 0.25809207558631897
 }
 },
 {
 "id": "cosmosqa/validation/3KQC8JMJGE6S3M71KNCQIVT9ZOKH3Z##3IO1LGZLKCC1UERKQNMK158JVUZ86U##A2KYOM2ZGYDJGT##Blog_1313225##q1_a1##3DGDV62G7QNVVL96Z06Q8UQ8GH8P2C",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 1.4125621419225354e-06,
 "1": 0.00018246524268761277,
 "2": 0.9997852444648743,
 "3": 3.08721064357087e-05
 }
 },
 {
 "id": "cosmosqa/validation/3P520RYKCH4LSY1IE7VIO9SBFCM5UG##3IAS3U3I0GEHJM7JM0IRAWTZLJFB21##AJ7NSZOAFP05U##Blog_345454##q1_a2##3RDTX9JRT0FHPZQ5ONROIV71MYX97C",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.0030631814152002335,
 "1": 0.9949642419815063,
 "2": 0.00034352130023762584,
 "3": 0.0016290743369609118
 }
 },
 {
 "id": "multirc/validation/2028",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9955548644065857,
 "no": 0.00444513326510787
 }
 },
 {
 "id": "cosmosqa/validation/36FFXPMST9M2VM1GJNKIE03C2W7HO8##3TPWUS5F8AZM57XWEK3T9NF675GCWL##A2KYOM2ZGYDJGT##Blog_638832##q1_a1##3M93N4X8HM1SE9N6Y2GC2GH05HZJSH",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 6.786166341044009e-05,
 "1": 6.154371931188507e-06,
 "2": 2.7787241378973704e-06,
 "3": 0.9999231100082397
 }
 },
 {
 "id": "cosmosqa/validation/3S1WOPCJFI7R4QIOX7W7DZDL4VFJEF##3I02618YA3L72HG28MMY3XHPSRKUPC##AJ7NSZOAFP05U##Blog_1483896##q1_a1##3RKHNXPHGYA346LS5SBOIWZ26LWKUC",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 0.00042348113493062556,
 "1": 0.9982168078422546,
 "2": 0.0013537148479372263,
 "3": 5.962935119896429e-06
 }
 },
 {
 "id": "multirc/validation/3514",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9769245386123657,
 "no": 0.023075425997376442
 }
 },
 {
 "id": "cosmosqa/validation/338GLSUI439XFI23VEDRVK10JVXFSJ##3YMTUJH0DTERIMGNQ0S035WJACH4T1##A1OPGPTQJ48XOY##Blog_1389538##q1_a1##3IV1AEQ4DTRAVP8XK4A7LPHZTIM8JQ",
 "source": "cosmosqa",
 "expected": "3",
 "choice": "3",
 "probabilities": {
 "0": 0.00391255971044302,
 "1": 0.0032130032777786255,
 "2": 7.453902071574703e-05,
 "3": 0.9927998781204224
 }
 },
 {
 "id": "multirc/validation/3501",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.4642350673675537,
 "no": 0.5357648730278015
 }
 },
 {
 "id": "cosmosqa/validation/39N6W9XWRDL0P4A754BYVRPD1QCGY6##3C8HJ7UOP8S38OES2WHEBNE0GC8ZMG##AO33H4GL9KZX9##Blog_1114758##q1_a1##3D1UCPY6GINZAB68HJIK5I0AXFW38T",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.6364780068397522,
 "1": 0.3527103066444397,
 "2": 0.0013628056040033698,
 "3": 0.009448966942727566
 }
 },
 {
 "id": "multirc/validation/3552",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9977766871452332,
 "no": 0.0022232523187994957
 }
 },
 {
 "id": "cosmosqa/validation/3HKIF5DF60B0IS16HQVIO8KUL1C9GB##3IRIK4HM3DZ36HHX7Z2P3R42P576CT##A1OPGPTQJ48XOY##Blog_623206##q1_a1##3PIOQ99R7004CC6V2534AOG5Z2WNUK",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 7.828805973986164e-05,
 "1": 0.9995183944702148,
 "2": 0.00031007645884528756,
 "3": 9.32921320782043e-05
 }
 },
 {
 "id": "multirc/validation/2278",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0011798491468653083,
 "no": 0.9988201260566711
 }
 },
 {
 "id": "multirc/validation/4111",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.06065484508872032,
 "no": 0.9393451809883118
 }
 },
 {
 "id": "multirc/validation/4096",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0002704693761188537,
 "no": 0.999729573726654
 }
 },
 {
 "id": "multirc/validation/2312",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.998069703578949,
 "no": 0.001930364640429616
 }
 },
 {
 "id": "multirc/validation/1983",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9983623623847961,
 "no": 0.0016376138664782047
 }
 },
 {
 "id": "cosmosqa/validation/3P520RYKCJK0MN0PY57IG7GJ82FU5Q##3P529IW9K10SSE4EL5YS0G72BXVLF1##A2KYOM2ZGYDJGT##Blog_831927##q1_a1##31YWE12TE2Q7QV83R3X2YYTJT23X70",
 "source": "cosmosqa",
 "expected": "2",
 "choice": "2",
 "probabilities": {
 "0": 3.215913011445082e-07,
 "1": 1.292360281013316e-07,
 "2": 0.9999945163726807,
 "3": 4.980084213457303e-06
 }
 },
 {
 "id": "cosmosqa/validation/3UEBBGULPH2SOKTE4HKR2J7XYG8FUG##3TYCR1GOTFYYXVVEMHSZ38NKPDWZLY##A3GVXFBYAU9HP##Blog_615658##q1_a2##3VMV5CHJZATJ27AKFW0ZF8L7GQCTGV",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.991963267326355,
 "1": 0.0004775024426635355,
 "2": 0.00032946362625807524,
 "3": 0.00722977751865983
 }
 },
 {
 "id": "multirc/validation/421",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.3508681058883667,
 "no": 0.6491318941116333
 }
 },
 {
 "id": "cosmosqa/validation/3HRWUH63QWGNRXGG14VW2COP0405NJ##3ERET4BTVPO87YVJXJMZBK8CJW39KW##AJ7NSZOAFP05U##Blog_1157846##q1_a1##3VQTAXTYN5Z2B3HYZFYVJTGBIA7UBL",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 6.671338041996933e-07,
 "1": 0.9999443292617798,
 "2": 5.288542160997167e-05,
 "3": 2.159234554710565e-06
 }
 },
 {
 "id": "multirc/validation/448",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9987450838088989,
 "no": 0.0012549435487017035
 }
 },
 {
 "id": "multirc/validation/420",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.996601939201355,
 "no": 0.0033980412408709526
 }
 },
 {
 "id": "cosmosqa/validation/3BJKPTD2QEQARXJYV7UBECZDH4ATRL##3E1QT0TDFSOHNYMLSOV9Y8YO7LJI8Q##AJ7NSZOAFP05U##Blog_975359##q1_a1##3CO05SML7XJ9BOM7X561U1X8K770R2",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 3.865658015911322e-07,
 "1": 0.9999911785125732,
 "2": 6.771728777721364e-08,
 "3": 8.360988431377336e-06
 }
 },
 {
 "id": "multirc/validation/3260",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0020870384760200977,
 "no": 0.9979129433631897
 }
 },
 {
 "id": "multirc/validation/17",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9973192811012268,
 "no": 0.002680761506780982
 }
 },
 {
 "id": "multirc/validation/4463",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9937214851379395,
 "no": 0.0062784855253994465
 }
 },
 {
 "id": "race/validation/3877",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9998412132263184,
 "B": 6.844370363978669e-05,
 "C": 2.8125748940510675e-05,
 "D": 6.221430521691218e-05
 }
 },
 {
 "id": "multirc/validation/3272",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0007308634230867028,
 "no": 0.9992691874504089
 }
 },
 {
 "id": "multirc/validation/4492",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9918917417526245,
 "no": 0.008108310401439667
 }
 },
 {
 "id": "cosmosqa/validation/3YGYP13643NCWKEEYB8Z5DTLSVSNR0##3MMN5BL1W2JHIK661MY5ZKXU98T3MI##A1OPGPTQJ48XOY##Blog_71022##q1_a1##3KLL7H3EGFFCFV9K4XIW1QLSRWRHVN",
 "source": "cosmosqa",
 "expected": "1",
 "choice": "1",
 "probabilities": {
 "0": 2.1736536837124731e-07,
 "1": 0.9999963045120239,
 "2": 1.958444499905454e-06,
 "3": 1.604658336873399e-06
 }
 },
 {
 "id": "multirc/validation/4658",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.994556188583374,
 "no": 0.005443830508738756
 }
 },
 {
 "id": "multirc/validation/0",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00040872034151107073,
 "no": 0.9995912909507751
 }
 },
 {
 "id": "race/validation/3482",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0016152389580383897,
 "B": 0.9971605539321899,
 "C": 1.7669695807853714e-05,
 "D": 0.0012065906776115298
 }
 },
 {
 "id": "multirc/validation/2441",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0039204880595207214,
 "no": 0.9960795044898987
 }
 },
 {
 "id": "race/validation/2163",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00016382418107241392,
 "B": 3.204663983069622e-07,
 "C": 0.9998292922973633,
 "D": 6.633740667894017e-06
 }
 },
 {
 "id": "race/validation/853",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9754102230072021,
 "B": 0.0005871574976481497,
 "C": 0.001281341421417892,
 "D": 0.0227211806923151
 }
 },
 {
 "id": "multirc/validation/2461",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9952632188796997,
 "no": 0.004736816510558128
 }
 },
 {
 "id": "multirc/validation/3226",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006642541848123074,
 "no": 0.9993357062339783
 }
 },
 {
 "id": "race/validation/2728",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 4.257352429704042e-06,
 "B": 0.004670789465308189,
 "C": 0.9953020811080933,
 "D": 2.289748954353854e-05
 }
 },
 {
 "id": "multirc/validation/2453",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00028779092826880515,
 "no": 0.9997121691703796
 }
 },
 {
 "id": "race/validation/3995",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0001058825509971939,
 "B": 1.2985972261958523e-06,
 "C": 0.9997921586036682,
 "D": 0.00010068525443784893
 }
 },
 {
 "id": "multirc/validation/479",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.8437830805778503,
 "no": 0.15621693432331085
 }
 },
 {
 "id": "multirc/validation/4439",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006291221361607313,
 "no": 0.9993708729743958
 }
 },
 {
 "id": "multirc/validation/4653",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006134710856713355,
 "no": 0.9993865489959717
 }
 },
 {
 "id": "multirc/validation/3243",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.029266882687807083,
 "no": 0.9707331657409668
 }
 },
 {
 "id": "multirc/validation/108",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9976329803466797,
 "no": 0.0023669637739658356
 }
 },
 {
 "id": "race/validation/3545",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 2.7845218937727623e-05,
 "B": 1.6199254559978726e-06,
 "C": 0.9999126195907593,
 "D": 5.7875960919773206e-05
 }
 },
 {
 "id": "multirc/validation/1492",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0007527669076807797,
 "no": 0.9992473125457764
 }
 },
 {
 "id": "cosmosqa/validation/359AP8GAGGI4T8Y1G9PFSNK1Z6IC7C##32Z9ZLUT1MI5H6ZA9QQK3KZ0GIKOH1##A1OPGPTQJ48XOY##Blog_470723##q1_a1##3M93N4X8HM1SE9N6Y2GC2GH05D3SJM",
 "source": "cosmosqa",
 "expected": "0",
 "choice": "0",
 "probabilities": {
 "0": 0.9993380904197693,
 "1": 0.00033626778167672455,
 "2": 0.0002511264756321907,
 "3": 7.454221486113966e-05
 }
 },
 {
 "id": "multirc/validation/662",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0005900382529944181,
 "no": 0.9994099140167236
 }
 },
 {
 "id": "multirc/validation/3197",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.0019376056734472513,
 "no": 0.9980624318122864
 }
 },
 {
 "id": "multirc/validation/101",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9941354990005493,
 "no": 0.005864470731467009
 }
 },
 {
 "id": "multirc/validation/1491",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0014874159824103117,
 "no": 0.9985125660896301
 }
 },
 {
 "id": "multirc/validation/486",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9622051119804382,
 "no": 0.03779495507478714
 }
 },
 {
 "id": "multirc/validation/1477",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009354171343147755,
 "no": 0.9990646243095398
 }
 },
 {
 "id": "multirc/validation/653",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.996487021446228,
 "no": 0.0035129650495946407
 }
 },
 {
 "id": "race/validation/3576",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9968872666358948,
 "B": 0.0012285900302231312,
 "C": 0.001378714689053595,
 "D": 0.0005053841159678996
 }
 },
 {
 "id": "race/validation/3936",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9995545744895935,
 "B": 0.00012090480595361441,
 "C": 0.00017472985200583935,
 "D": 0.0001498515484854579
 }
 },
 {
 "id": "multirc/validation/9",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009374701185151935,
 "no": 0.9990625977516174
 }
 },
 {
 "id": "multirc/validation/690",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0001409475808031857,
 "no": 0.9998589754104614
 }
 },
 {
 "id": "multirc/validation/2630",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0002525840245652944,
 "no": 0.9997474551200867
 }
 },
 {
 "id": "multirc/validation/2627",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9978253841400146,
 "no": 0.0021746577695012093
 }
 },
 {
 "id": "multirc/validation/2620",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006881385925225914,
 "no": 0.9993118047714233
 }
 },
 {
 "id": "multirc/validation/3471",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009180559427477419,
 "no": 0.9990819692611694
 }
 },
 {
 "id": "race/validation/3105",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9502168297767639,
 "B": 0.003610153216868639,
 "C": 0.0007684877491556108,
 "D": 0.04540460556745529
 }
 },
 {
 "id": "multirc/validation/2258",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9985622763633728,
 "no": 0.001437789062038064
 }
 },
 {
 "id": "multirc/validation/3440",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9971285462379456,
 "no": 0.002871492877602577
 }
 },
 {
 "id": "race/validation/4503",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9986159801483154,
 "B": 0.0006549723329953849,
 "C": 0.00034312857314944267,
 "D": 0.0003859550633933395
 }
 },
 {
 "id": "race/validation/3503",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.003310930682346225,
 "B": 5.668392986990511e-06,
 "C": 0.9966758489608765,
 "D": 7.470820037269732e-06
 }
 },
 {
 "id": "multirc/validation/2256",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9986520409584045,
 "no": 0.001347972545772791
 }
 },
 {
 "id": "race/validation/4121",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 8.638332360533241e-07,
 "B": 1.0957555787172168e-06,
 "C": 0.9999707937240601,
 "D": 2.7309510187478736e-05
 }
 },
 {
 "id": "multirc/validation/3466",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.307835191488266,
 "no": 0.6921647787094116
 }
 },
 {
 "id": "race/validation/3897",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 7.195641228463501e-05,
 "B": 2.8527041649795137e-05,
 "C": 0.9998613595962524,
 "D": 3.813384682871401e-05
 }
 },
 {
 "id": "multirc/validation/1560",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9650066494941711,
 "no": 0.034993357956409454
 }
 },
 {
 "id": "race/validation/529",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9997219443321228,
 "B": 6.708730506943539e-05,
 "C": 8.117881225189194e-05,
 "D": 0.00012974566197954118
 }
 },
 {
 "id": "race/validation/1377",
 "source": "race",
 "expected": "D",
 "choice": "B",
 "probabilities": {
 "A": 0.15033309161663055,
 "B": 0.4104487895965576,
 "C": 0.335287868976593,
 "D": 0.10393030196428299
 }
 },
 {
 "id": "multirc/validation/1569",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.8812980055809021,
 "no": 0.11870195716619492
 }
 },
 {
 "id": "race/validation/3656",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.00013575675257015973,
 "B": 3.392237658772501e-06,
 "C": 7.727476969421332e-08,
 "D": 0.9998607635498047
 }
 },
 {
 "id": "race/validation/2821",
 "source": "race",
 "expected": "D",
 "choice": "B",
 "probabilities": {
 "A": 0.0960468277335167,
 "B": 0.7892670631408691,
 "C": 0.028115583583712578,
 "D": 0.08657050877809525
 }
 },
 {
 "id": "race/validation/1282",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 3.282702436990803e-06,
 "B": 5.837679509568261e-06,
 "C": 0.9999483823776245,
 "D": 4.240431735524908e-05
 }
 },
 {
 "id": "race/validation/4636",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00020673393737524748,
 "B": 0.9921020865440369,
 "C": 0.007384497672319412,
 "D": 0.0003066939825657755
 }
 },
 {
 "id": "race/validation/3463",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.2823525139538106e-06,
 "B": 0.0007249435293488204,
 "C": 0.999261200428009,
 "D": 1.2499468539317604e-05
 }
 },
 {
 "id": "race/validation/1224",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 8.366655674763024e-05,
 "B": 0.9996345043182373,
 "C": 9.816890087677166e-05,
 "D": 0.0001836159499362111
 }
 },
 {
 "id": "race/validation/3816",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.9250855984864756e-06,
 "B": 6.24244478331093e-07,
 "C": 0.999961256980896,
 "D": 3.615931927924976e-05
 }
 },
 {
 "id": "race/validation/1240",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.00028072873828932643,
 "B": 0.0002846108400262892,
 "C": 0.0001211567796417512,
 "D": 0.999313473701477
 }
 },
 {
 "id": "multirc/validation/4235",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9987838864326477,
 "no": 0.0012161765480414033
 }
 },
 {
 "id": "multirc/validation/2387",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0013110869331285357,
 "no": 0.9986888766288757
 }
 },
 {
 "id": "race/validation/4380",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 2.3316348233493045e-05,
 "B": 8.131766662700102e-05,
 "C": 0.9996484518051147,
 "D": 0.0002468553138896823
 }
 },
 {
 "id": "race/validation/1869",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0002781036891974509,
 "B": 0.9996764659881592,
 "C": 2.7209949621465057e-05,
 "D": 1.8177874153479934e-05
 }
 },
 {
 "id": "race/validation/4695",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 5.49698938812071e-07,
 "B": 6.561890586453956e-07,
 "C": 0.999992847442627,
 "D": 5.925650839344598e-06
 }
 },
 {
 "id": "multirc/validation/2345",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0046433997340500355,
 "no": 0.9953566193580627
 }
 },
 {
 "id": "multirc/validation/4238",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9934968948364258,
 "no": 0.006503064651042223
 }
 },
 {
 "id": "race/validation/3397",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.377861735818442e-06,
 "B": 3.2118310627993196e-05,
 "C": 0.9999630451202393,
 "D": 3.461714868535637e-06
 }
 },
 {
 "id": "multirc/validation/2926",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0023668084759265184,
 "no": 0.9976332187652588
 }
 },
 {
 "id": "race/validation/3765",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.830635250778869e-05,
 "B": 0.0004894465091638267,
 "C": 1.438496656192001e-05,
 "D": 0.9994578957557678
 }
 },
 {
 "id": "race/validation/506",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9947726130485535,
 "B": 0.004265872295945883,
 "C": 0.0003137978201266378,
 "D": 0.0006477872957475483
 }
 },
 {
 "id": "race/validation/4368",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.4337539141706657e-05,
 "B": 0.9999480247497559,
 "C": 1.59409119078191e-05,
 "D": 2.1646705135935917e-05
 }
 },
 {
 "id": "race/validation/3870",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9973354935646057,
 "B": 0.0024210591800510883,
 "C": 0.0001409361866535619,
 "D": 0.00010250450577586889
 }
 },
 {
 "id": "multirc/validation/1410",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0019996624905616045,
 "no": 0.9980003237724304
 }
 },
 {
 "id": "race/validation/4194",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.0334777471143752e-05,
 "B": 3.1975278034224175e-06,
 "C": 4.204091237625107e-05,
 "D": 0.9999444484710693
 }
 },
 {
 "id": "multirc/validation/3485",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0008499289397150278,
 "no": 0.9991500377655029
 }
 },
 {
 "id": "multirc/validation/2945",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001699696178548038,
 "no": 0.9983003735542297
 }
 },
 {
 "id": "multirc/validation/3478",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.914948582649231,
 "no": 0.08505148440599442
 }
 },
 {
 "id": "multirc/validation/1416",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001729710609652102,
 "no": 0.9982702732086182
 }
 },
 {
 "id": "multirc/validation/846",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.4752455949783325,
 "no": 0.5247544646263123
 }
 },
 {
 "id": "multirc/validation/1322",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.046587906777858734,
 "no": 0.9534121155738831
 }
 },
 {
 "id": "multirc/validation/851",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9783985018730164,
 "no": 0.021601419895887375
 }
 },
 {
 "id": "race/validation/4551",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 3.339851173222996e-05,
 "B": 0.9998912811279297,
 "C": 7.308655312954215e-06,
 "D": 6.799580296501517e-05
 }
 },
 {
 "id": "multirc/validation/3642",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.015585635788738728,
 "no": 0.9844143390655518
 }
 },
 {
 "id": "race/validation/1122",
 "source": "race",
 "expected": "C",
 "choice": "A",
 "probabilities": {
 "A": 0.999866247177124,
 "B": 7.366140926023945e-05,
 "C": 5.297582902130671e-05,
 "D": 7.109100806701463e-06
 }
 },
 {
 "id": "multirc/validation/843",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.003336753463372588,
 "no": 0.9966632723808289
 }
 },
 {
 "id": "race/validation/633",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.942150354385376,
 "B": 0.0238445196300745,
 "C": 0.01788947917521,
 "D": 0.016115646809339523
 }
 },
 {
 "id": "multirc/validation/3611",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0004109358706045896,
 "no": 0.9995890259742737
 }
 },
 {
 "id": "race/validation/4591",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 2.6335983420722187e-05,
 "B": 6.337922542343222e-08,
 "C": 0.9999631643295288,
 "D": 1.047668956744019e-05
 }
 },
 {
 "id": "race/validation/2454",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 2.9628143238369375e-05,
 "B": 0.9999662637710571,
 "C": 2.3816478460503276e-06,
 "D": 1.6500102901773062e-06
 }
 },
 {
 "id": "multirc/validation/3602",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006228994461707771,
 "no": 0.9993770718574524
 }
 },
 {
 "id": "race/validation/3619",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 6.440694164666638e-07,
 "B": 1.3514635611500125e-05,
 "C": 0.9999765157699585,
 "D": 9.409774065716192e-06
 }
 },
 {
 "id": "multirc/validation/2074",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9989323019981384,
 "no": 0.0010676957899704576
 }
 },
 {
 "id": "multirc/validation/204",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001492527429945767,
 "no": 0.9985074400901794
 }
 },
 {
 "id": "race/validation/4150",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 5.362353405757858e-08,
 "B": 3.465763498411434e-08,
 "C": 5.10350837146234e-08,
 "D": 0.9999998807907104
 }
 },
 {
 "id": "multirc/validation/3654",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9846532940864563,
 "no": 0.015346652828156948
 }
 },
 {
 "id": "race/validation/4024",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 9.172714499072754e-08,
 "B": 2.442436937144521e-07,
 "C": 0.9999973773956299,
 "D": 2.2394119696400594e-06
 }
 },
 {
 "id": "multirc/validation/2054",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9917226433753967,
 "no": 0.00827733799815178
 }
 },
 {
 "id": "race/validation/3150",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00029831560095772147,
 "B": 6.729352753609419e-05,
 "C": 0.9994514584541321,
 "D": 0.0001828791428124532
 }
 },
 {
 "id": "multirc/validation/1305",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.011899863369762897,
 "no": 0.9881001710891724
 }
 },
 {
 "id": "race/validation/28",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.015351769514381886,
 "B": 0.9576277136802673,
 "C": 0.012888279743492603,
 "D": 0.014132224023342133
 }
 },
 {
 "id": "race/validation/3421",
 "source": "race",
 "expected": "C",
 "choice": "D",
 "probabilities": {
 "A": 0.048096660524606705,
 "B": 0.007648704573512077,
 "C": 0.2203287035226822,
 "D": 0.7239258885383606
 }
 },
 {
 "id": "multirc/validation/184",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.03943054378032684,
 "no": 0.960569441318512
 }
 },
 {
 "id": "multirc/validation/2189",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009730068850331008,
 "no": 0.9990270137786865
 }
 },
 {
 "id": "multirc/validation/1225",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9952051639556885,
 "no": 0.004794809967279434
 }
 },
 {
 "id": "race/validation/1095",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0013145452830940485,
 "B": 0.0021102679893374443,
 "C": 0.9964832067489624,
 "D": 9.2034024419263e-05
 }
 },
 {
 "id": "race/validation/1848",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00020597042748704553,
 "B": 0.00015031012299004942,
 "C": 0.9986646175384521,
 "D": 0.0009790969779714942
 }
 },
 {
 "id": "multirc/validation/193",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0033122918102890253,
 "no": 0.9966877102851868
 }
 },
 {
 "id": "multirc/validation/2237",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00033604796044528484,
 "no": 0.9996639490127563
 }
 },
 {
 "id": "race/validation/385",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.000615069002378732,
 "B": 0.9983475208282471,
 "C": 0.0003347939345985651,
 "D": 0.0007026238017715514
 }
 },
 {
 "id": "race/validation/2210",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9998152852058411,
 "B": 0.00011686169455060735,
 "C": 4.9553807912161574e-05,
 "D": 1.8306813217350282e-05
 }
 },
 {
 "id": "race/validation/4722",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 7.743975771745681e-08,
 "B": 1.0017459999289713e-06,
 "C": 4.3579129282989015e-07,
 "D": 0.9999985694885254
 }
 },
 {
 "id": "race/validation/1476",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 3.879152245644946e-06,
 "B": 0.9979662895202637,
 "C": 0.0019223382696509361,
 "D": 0.00010757584095699713
 }
 },
 {
 "id": "race/validation/4822",
 "source": "race",
 "expected": "D",
 "choice": "B",
 "probabilities": {
 "A": 0.000123411780805327,
 "B": 0.8363147974014282,
 "C": 0.0004536090709734708,
 "D": 0.16310818493366241
 }
 },
 {
 "id": "race/validation/2775",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00035082941758446395,
 "B": 0.0004427565436344594,
 "C": 0.9988378882408142,
 "D": 0.00036859113606624305
 }
 },
 {
 "id": "multirc/validation/1173",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0011821109801530838,
 "no": 0.9988178610801697
 }
 },
 {
 "id": "multirc/validation/3696",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9965600371360779,
 "no": 0.003439994528889656
 }
 },
 {
 "id": "race/validation/4145",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.5380584954982623e-05,
 "B": 0.999916672706604,
 "C": 7.901805474830326e-06,
 "D": 6.004481838317588e-05
 }
 },
 {
 "id": "multirc/validation/1169",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0013096356997266412,
 "no": 0.9986904263496399
 }
 },
 {
 "id": "multirc/validation/1179",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9968963861465454,
 "no": 0.0031035521533340216
 }
 },
 {
 "id": "race/validation/1455",
 "source": "race",
 "expected": "D",
 "choice": "C",
 "probabilities": {
 "A": 0.00019285173038952053,
 "B": 5.518485340871848e-05,
 "C": 0.9524655342102051,
 "D": 0.047286421060562134
 }
 },
 {
 "id": "multirc/validation/209",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9968541264533997,
 "no": 0.0031459403689950705
 }
 },
 {
 "id": "race/validation/3487",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.367281067312433e-07,
 "B": 0.014294502325356007,
 "C": 0.00022272412024904042,
 "D": 0.9854824542999268
 }
 },
 {
 "id": "race/validation/4353",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9954381585121155,
 "B": 0.000383844570023939,
 "C": 0.00401320168748498,
 "D": 0.0001648754987400025
 }
 },
 {
 "id": "multirc/validation/2087",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009671766893006861,
 "no": 0.9990328550338745
 }
 },
 {
 "id": "multirc/validation/2089",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9973465204238892,
 "no": 0.0026534912176430225
 }
 },
 {
 "id": "multirc/validation/3673",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00025549583369866014,
 "no": 0.9997444748878479
 }
 },
 {
 "id": "multirc/validation/929",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.13776719570159912,
 "no": 0.8622328042984009
 }
 },
 {
 "id": "multirc/validation/2085",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.000874202698469162,
 "no": 0.9991257786750793
 }
 },
 {
 "id": "multirc/validation/1876",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9975835084915161,
 "no": 0.0024164668284356594
 }
 },
 {
 "id": "multirc/validation/167",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0020660879090428352,
 "no": 0.9979339838027954
 }
 },
 {
 "id": "multirc/validation/3941",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9937123656272888,
 "no": 0.006287613417953253
 }
 },
 {
 "id": "race/validation/4649",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 7.949378777993843e-05,
 "B": 0.9997816681861877,
 "C": 9.314385533798486e-05,
 "D": 4.567556970869191e-05
 }
 },
 {
 "id": "multirc/validation/4031",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00045341820805333555,
 "no": 0.9995465874671936
 }
 },
 {
 "id": "race/validation/4315",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.999366819858551,
 "B": 0.00030155476997606456,
 "C": 0.00015340649406425655,
 "D": 0.00017819205822888762
 }
 },
 {
 "id": "multirc/validation/913",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9159284830093384,
 "no": 0.08407149463891983
 }
 },
 {
 "id": "race/validation/3096",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 4.0163482481148094e-05,
 "B": 1.1608502063609194e-05,
 "C": 0.9999270439147949,
 "D": 2.1222553186817095e-05
 }
 },
 {
 "id": "multirc/validation/2806",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.011991008184850216,
 "no": 0.988008975982666
 }
 },
 {
 "id": "multirc/validation/168",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0008457601652480662,
 "no": 0.9991542100906372
 }
 },
 {
 "id": "race/validation/4770",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 2.6356901798862964e-06,
 "B": 8.357195042663079e-07,
 "C": 0.9996848106384277,
 "D": 0.00031174509786069393
 }
 },
 {
 "id": "race/validation/4876",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 8.777853508945554e-06,
 "B": 3.0778917334828293e-06,
 "C": 8.025769784580916e-05,
 "D": 0.9999078512191772
 }
 },
 {
 "id": "multirc/validation/3595",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9986664056777954,
 "no": 0.001333584077656269
 }
 },
 {
 "id": "race/validation/3201",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 2.1264834231260465e-06,
 "B": 0.9999809265136719,
 "C": 7.784513400110882e-06,
 "D": 9.178392247122247e-06
 }
 },
 {
 "id": "multirc/validation/1005",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.003342891111969948,
 "no": 0.996657133102417
 }
 },
 {
 "id": "race/validation/3907",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 2.040529579971917e-05,
 "B": 0.9999140501022339,
 "C": 5.4248936066869646e-05,
 "D": 1.1333443580952007e-05
 }
 },
 {
 "id": "multirc/validation/1874",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.04439326375722885,
 "no": 0.9556066989898682
 }
 },
 {
 "id": "multirc/validation/2828",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9950882792472839,
 "no": 0.004911722149699926
 }
 },
 {
 "id": "race/validation/2975",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 4.030316176795168e-06,
 "B": 4.73062073069741e-06,
 "C": 0.9998458623886108,
 "D": 0.00014529298641718924
 }
 },
 {
 "id": "race/validation/1587",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.0438169510962325e-06,
 "B": 1.0285694997946848e-06,
 "C": 3.120014753221767e-06,
 "D": 0.9999947547912598
 }
 },
 {
 "id": "race/validation/640",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.009438606910407543,
 "B": 0.9722672700881958,
 "C": 0.01629769057035446,
 "D": 0.0019964787643402815
 }
 },
 {
 "id": "race/validation/4885",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0039675235748291016,
 "B": 0.9957948923110962,
 "C": 8.737800817470998e-05,
 "D": 0.00015018238627817482
 }
 },
 {
 "id": "multirc/validation/984",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001779726124368608,
 "no": 0.9982202649116516
 }
 },
 {
 "id": "race/validation/2452",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.3521274922823068e-05,
 "B": 6.063306727810414e-07,
 "C": 0.9999822378158569,
 "D": 3.731647666427307e-06
 }
 },
 {
 "id": "race/validation/2786",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 2.369049070694018e-06,
 "B": 0.9999771118164062,
 "C": 1.1978405382251367e-05,
 "D": 8.557617547921836e-06
 }
 },
 {
 "id": "multirc/validation/967",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.002228607889264822,
 "no": 0.9977713823318481
 }
 },
 {
 "id": "race/validation/1734",
 "source": "race",
 "expected": "D",
 "choice": "C",
 "probabilities": {
 "A": 0.00012693752069026232,
 "B": 5.285679435473867e-06,
 "C": 0.6084805130958557,
 "D": 0.3913872539997101
 }
 },
 {
 "id": "multirc/validation/1858",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0020687489304691553,
 "no": 0.9979312419891357
 }
 },
 {
 "id": "race/validation/256",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 7.351220006057702e-07,
 "B": 1.4949081332815695e-06,
 "C": 0.9999748468399048,
 "D": 2.2927259124116972e-05
 }
 },
 {
 "id": "race/validation/4558",
 "source": "race",
 "expected": "C",
 "choice": "A",
 "probabilities": {
 "A": 0.7429482340812683,
 "B": 0.04503205791115761,
 "C": 0.21184414625167847,
 "D": 0.0001755785633577034
 }
 },
 {
 "id": "multirc/validation/1769",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.005496708210557699,
 "no": 0.9945032596588135
 }
 },
 {
 "id": "race/validation/2694",
 "source": "race",
 "expected": "B",
 "choice": "D",
 "probabilities": {
 "A": 1.691306351858657e-05,
 "B": 2.3642101950827055e-05,
 "C": 2.468585989845451e-05,
 "D": 0.9999347925186157
 }
 },
 {
 "id": "race/validation/80",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 5.12746482854709e-05,
 "B": 0.9990047812461853,
 "C": 0.0008622457389719784,
 "D": 8.168238855432719e-05
 }
 },
 {
 "id": "multirc/validation/1795",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9964026212692261,
 "no": 0.003597381291911006
 }
 },
 {
 "id": "race/validation/3178",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9998879432678223,
 "B": 3.26303270412609e-05,
 "C": 7.817635196261108e-05,
 "D": 1.2671157492150087e-06
 }
 },
 {
 "id": "multirc/validation/1451",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9958985447883606,
 "no": 0.0041014328598976135
 }
 },
 {
 "id": "race/validation/1544",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.3973510249343235e-05,
 "B": 3.7165220874157967e-06,
 "C": 0.9999213218688965,
 "D": 6.103576743043959e-05
 }
 },
 {
 "id": "race/validation/587",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 2.845854396582581e-06,
 "B": 6.430499297493952e-07,
 "C": 9.537833648209926e-06,
 "D": 0.999987006187439
 }
 },
 {
 "id": "multirc/validation/313",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.004016705323010683,
 "no": 0.995983362197876
 }
 },
 {
 "id": "multirc/validation/1453",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.996651828289032,
 "no": 0.0033481998834758997
 }
 },
 {
 "id": "race/validation/4114",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9995983242988586,
 "B": 0.00014895277854520828,
 "C": 0.00016375900304410607,
 "D": 8.906403672881424e-05
 }
 },
 {
 "id": "race/validation/3310",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.998914361000061,
 "B": 0.00012071392848156393,
 "C": 4.738297138828784e-05,
 "D": 0.000917496217880398
 }
 },
 {
 "id": "race/validation/2884",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.212226123925575e-07,
 "B": 2.356485708787659e-07,
 "C": 7.172086952778045e-06,
 "D": 0.9999924898147583
 }
 },
 {
 "id": "multirc/validation/832",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9802659153938293,
 "no": 0.019734112545847893
 }
 },
 {
 "id": "multirc/validation/1455",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0007295446703210473,
 "no": 0.999270498752594
 }
 },
 {
 "id": "multirc/validation/3585",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9341541528701782,
 "no": 0.06584587693214417
 }
 },
 {
 "id": "multirc/validation/789",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.8680403828620911,
 "no": 0.13195954263210297
 }
 },
 {
 "id": "multirc/validation/282",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.000913269177544862,
 "no": 0.9990867376327515
 }
 },
 {
 "id": "race/validation/925",
 "source": "race",
 "expected": "D",
 "choice": "A",
 "probabilities": {
 "A": 0.7583421468734741,
 "B": 0.000640281243249774,
 "C": 0.12435253709554672,
 "D": 0.11666508764028549
 }
 },
 {
 "id": "race/validation/690",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9987949132919312,
 "B": 0.0003676008200272918,
 "C": 0.0007669679471291602,
 "D": 7.048487896099687e-05
 }
 },
 {
 "id": "multirc/validation/3329",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.004102109000086784,
 "no": 0.9958978295326233
 }
 },
 {
 "id": "multirc/validation/3844",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0028173644095659256,
 "no": 0.9971826076507568
 }
 },
 {
 "id": "multirc/validation/1092",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0038291190285235643,
 "no": 0.9961709380149841
 }
 },
 {
 "id": "multirc/validation/2170",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9969876408576965,
 "no": 0.003012419445440173
 }
 },
 {
 "id": "race/validation/2019",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 7.904781341494527e-06,
 "B": 6.893117188155884e-07,
 "C": 1.589180283190217e-05,
 "D": 0.9999754428863525
 }
 },
 {
 "id": "multirc/validation/2500",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0007913543377071619,
 "no": 0.9992086291313171
 }
 },
 {
 "id": "multirc/validation/3388",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.006786793936043978,
 "no": 0.9932131767272949
 }
 },
 {
 "id": "race/validation/3291",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.011074353009462357,
 "B": 0.9525395631790161,
 "C": 0.0003922542091459036,
 "D": 0.035993851721286774
 }
 },
 {
 "id": "race/validation/2483",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.00037247047293931246,
 "B": 1.1866761269629933e-05,
 "C": 0.396026074886322,
 "D": 0.6035895347595215
 }
 },
 {
 "id": "multirc/validation/1113",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0044905757531523705,
 "no": 0.9955093860626221
 }
 },
 {
 "id": "multirc/validation/2160",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.06818605214357376,
 "no": 0.9318140149116516
 }
 },
 {
 "id": "race/validation/2040",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 5.999871518724831e-06,
 "B": 0.00013313829549588263,
 "C": 0.982653021812439,
 "D": 0.0172079149633646
 }
 },
 {
 "id": "race/validation/47",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 2.0208321984682698e-06,
 "B": 4.4755410044672317e-07,
 "C": 8.78059012165977e-08,
 "D": 0.9999973773956299
 }
 },
 {
 "id": "multirc/validation/2482",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.002165549900382757,
 "no": 0.9978345036506653
 }
 },
 {
 "id": "multirc/validation/3832",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.06265143305063248,
 "no": 0.9373486042022705
 }
 },
 {
 "id": "race/validation/1851",
 "source": "race",
 "expected": "A",
 "choice": "D",
 "probabilities": {
 "A": 0.0776267945766449,
 "B": 1.7761993149179034e-05,
 "C": 1.843136669776868e-05,
 "D": 0.922336995601654
 }
 },
 {
 "id": "race/validation/1380",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.737394995871e-05,
 "B": 4.941866336594103e-06,
 "C": 0.9998819828033447,
 "D": 9.568941459292546e-05
 }
 },
 {
 "id": "multirc/validation/529",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9930843114852905,
 "no": 0.0069157425314188
 }
 },
 {
 "id": "race/validation/1327",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9991182684898376,
 "B": 0.000607042689807713,
 "C": 2.781657167361118e-05,
 "D": 0.0002467602025717497
 }
 },
 {
 "id": "race/validation/3820",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 9.552310984872747e-06,
 "B": 6.356235644489061e-06,
 "C": 0.9999463558197021,
 "D": 3.769011527765542e-05
 }
 },
 {
 "id": "race/validation/3239",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.44775567605393e-05,
 "B": 0.998771607875824,
 "C": 0.0011431374587118626,
 "D": 7.073170854710042e-05
 }
 },
 {
 "id": "multirc/validation/4182",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0010733839590102434,
 "no": 0.99892657995224
 }
 },
 {
 "id": "race/validation/4032",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.03229602798819542,
 "B": 0.9605592489242554,
 "C": 0.006943197455257177,
 "D": 0.0002014806232182309
 }
 },
 {
 "id": "race/validation/423",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.999715268611908,
 "B": 0.00011844893742818385,
 "C": 1.2321598660491873e-05,
 "D": 0.00015394913498312235
 }
 },
 {
 "id": "multirc/validation/2525",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.0010106258559972048,
 "no": 0.9989894032478333
 }
 },
 {
 "id": "race/validation/539",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.4580576419830322,
 "B": 0.02625417709350586,
 "C": 0.005340239964425564,
 "D": 0.5103479027748108
 }
 },
 {
 "id": "multirc/validation/2685",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.10542960464954376,
 "no": 0.8945704698562622
 }
 },
 {
 "id": "race/validation/1703",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 9.007083576761943e-07,
 "B": 3.9874996105027094e-07,
 "C": 4.2267416233698896e-08,
 "D": 0.9999986886978149
 }
 },
 {
 "id": "multirc/validation/4188",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9966821074485779,
 "no": 0.0033178327139467
 }
 },
 {
 "id": "race/validation/2127",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9979296922683716,
 "B": 0.00018902264127973467,
 "C": 0.0017583111766725779,
 "D": 0.0001230161578860134
 }
 },
 {
 "id": "race/validation/318",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00363618740811944,
 "B": 0.7909623384475708,
 "C": 0.015495048835873604,
 "D": 0.18990644812583923
 }
 },
 {
 "id": "race/validation/3018",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.009218852035701275,
 "B": 0.4835435152053833,
 "C": 0.5045450329780579,
 "D": 0.002692649606615305
 }
 },
 {
 "id": "multirc/validation/4132",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9963818788528442,
 "no": 0.003618132323026657
 }
 },
 {
 "id": "multirc/validation/2112",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0004715945105999708,
 "no": 0.9995284080505371
 }
 },
 {
 "id": "multirc/validation/2114",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0009687888086773455,
 "no": 0.9990311861038208
 }
 },
 {
 "id": "race/validation/1339",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.094549356319476e-06,
 "B": 2.4946170015027747e-05,
 "C": 1.4891394130245317e-05,
 "D": 0.9999570846557617
 }
 },
 {
 "id": "race/validation/911",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00030716948094777763,
 "B": 0.9909404516220093,
 "C": 0.0003524769563227892,
 "D": 0.008399982936680317
 }
 },
 {
 "id": "multirc/validation/3762",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.002190772444009781,
 "no": 0.9978092312812805
 }
 },
 {
 "id": "race/validation/31",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9984599351882935,
 "B": 0.0010640040272846818,
 "C": 0.00022304107551462948,
 "D": 0.0002530152560211718
 }
 },
 {
 "id": "multirc/validation/2662",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006474241381511092,
 "no": 0.9993525147438049
 }
 },
 {
 "id": "race/validation/1966",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.7980289459228516,
 "B": 0.0034525319933891296,
 "C": 0.0030483861919492483,
 "D": 0.19547013938426971
 }
 },
 {
 "id": "race/validation/1743",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 2.7110465453006327e-05,
 "B": 0.054306164383888245,
 "C": 0.00028529620612971485,
 "D": 0.9453814625740051
 }
 },
 {
 "id": "race/validation/922",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.0706578905228525e-05,
 "B": 9.349298125016503e-06,
 "C": 0.00021228028344921768,
 "D": 0.9997677206993103
 }
 },
 {
 "id": "multirc/validation/4146",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9974052309989929,
 "no": 0.002594730583950877
 }
 },
 {
 "id": "race/validation/1979",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.322302887274418e-05,
 "B": 1.4915263818693347e-05,
 "C": 3.901465333910892e-06,
 "D": 0.9999679327011108
 }
 },
 {
 "id": "race/validation/977",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9979276657104492,
 "B": 0.000994431902654469,
 "C": 0.00105828617233783,
 "D": 1.962984970305115e-05
 }
 },
 {
 "id": "multirc/validation/2115",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9984513521194458,
 "no": 0.0015486609190702438
 }
 },
 {
 "id": "race/validation/4314",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 3.9938593545230106e-05,
 "B": 0.9999138116836548,
 "C": 3.10429823002778e-05,
 "D": 1.5237449588312302e-05
 }
 },
 {
 "id": "race/validation/608",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.01358464453369379,
 "B": 0.006002491805702448,
 "C": 0.9803473949432373,
 "D": 6.547832163050771e-05
 }
 },
 {
 "id": "multirc/validation/3737",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00527202570810914,
 "no": 0.9947280287742615
 }
 },
 {
 "id": "race/validation/1825",
 "source": "race",
 "expected": "D",
 "choice": "C",
 "probabilities": {
 "A": 0.0015055750263854861,
 "B": 0.0022931809071451426,
 "C": 0.9472014307975769,
 "D": 0.04899981990456581
 }
 },
 {
 "id": "race/validation/4402",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00014761784404981881,
 "B": 0.006507830694317818,
 "C": 0.9933057427406311,
 "D": 3.873615060001612e-05
 }
 },
 {
 "id": "multirc/validation/3752",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0023563671857118607,
 "no": 0.9976436495780945
 }
 },
 {
 "id": "multirc/validation/4154",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.000606971443630755,
 "no": 0.9993929862976074
 }
 },
 {
 "id": "race/validation/1927",
 "source": "race",
 "expected": "A",
 "choice": "C",
 "probabilities": {
 "A": 0.21847222745418549,
 "B": 0.0018617060268297791,
 "C": 0.776586651802063,
 "D": 0.003079497255384922
 }
 },
 {
 "id": "race/validation/2043",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9959050416946411,
 "B": 0.0002479411195963621,
 "C": 0.0002866474096663296,
 "D": 0.003560408717021346
 }
 },
 {
 "id": "race/validation/2061",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.998869001865387,
 "B": 0.0006949196686036885,
 "C": 0.00041196763049811125,
 "D": 2.4069711798802018e-05
 }
 },
 {
 "id": "race/validation/1793",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.005248815752565861,
 "B": 0.005355257075279951,
 "C": 0.9864820241928101,
 "D": 0.0029139097314327955
 }
 },
 {
 "id": "multirc/validation/586",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00016582962416578084,
 "no": 0.9998341798782349
 }
 },
 {
 "id": "multirc/validation/628",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.998080849647522,
 "no": 0.0019191689789295197
 }
 },
 {
 "id": "race/validation/591",
 "source": "race",
 "expected": "B",
 "choice": "C",
 "probabilities": {
 "A": 0.000431093096267432,
 "B": 0.0684187039732933,
 "C": 0.930263340473175,
 "D": 0.0008869132725521922
 }
 },
 {
 "id": "multirc/validation/52",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001053701271302998,
 "no": 0.9989463686943054
 }
 },
 {
 "id": "race/validation/399",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00035156848025508225,
 "B": 0.9995145797729492,
 "C": 9.211041469825432e-05,
 "D": 4.168432133155875e-05
 }
 },
 {
 "id": "multirc/validation/4813",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.014306262135505676,
 "no": 0.9856938123703003
 }
 },
 {
 "id": "multirc/validation/37",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0003305829595774412,
 "no": 0.9996694326400757
 }
 },
 {
 "id": "race/validation/1841",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00022108452685642987,
 "B": 0.99964439868927,
 "C": 5.614414840238169e-05,
 "D": 7.844666833989322e-05
 }
 },
 {
 "id": "race/validation/382",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.5152301784837618e-05,
 "B": 4.114425610168837e-05,
 "C": 2.2516428543895017e-06,
 "D": 0.9999414682388306
 }
 },
 {
 "id": "multirc/validation/259",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.001369793084450066,
 "no": 0.9986302852630615
 }
 },
 {
 "id": "race/validation/3333",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.03238804638385773,
 "B": 0.005735926330089569,
 "C": 0.009294036775827408,
 "D": 0.9525819420814514
 }
 },
 {
 "id": "multirc/validation/4736",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0008936563390307128,
 "no": 0.9991063475608826
 }
 },
 {
 "id": "multirc/validation/86",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.21452316641807556,
 "no": 0.785476803779602
 }
 },
 {
 "id": "multirc/validation/4827",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0012841913849115372,
 "no": 0.998715877532959
 }
 },
 {
 "id": "multirc/validation/4733",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9726173877716064,
 "no": 0.027382569387555122
 }
 },
 {
 "id": "multirc/validation/3076",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.014992795884609222,
 "no": 0.9850072264671326
 }
 },
 {
 "id": "multirc/validation/3080",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0002209534723078832,
 "no": 0.9997790455818176
 }
 },
 {
 "id": "race/validation/3075",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.27154362201690674,
 "B": 0.19844269752502441,
 "C": 0.5132364630699158,
 "D": 0.016777217388153076
 }
 },
 {
 "id": "race/validation/163",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.007181424181908369,
 "B": 0.9925116300582886,
 "C": 0.00023086817236617208,
 "D": 7.60878247092478e-05
 }
 },
 {
 "id": "multirc/validation/261",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.003078075824305415,
 "no": 0.996921956539154
 }
 },
 {
 "id": "race/validation/832",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 1.0298927008989267e-06,
 "B": 2.5615615868446184e-06,
 "C": 0.9999891519546509,
 "D": 7.160430413932772e-06
 }
 },
 {
 "id": "race/validation/1305",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9992081522941589,
 "B": 0.0007486905669793487,
 "C": 2.9957234801258892e-05,
 "D": 1.3288979971548542e-05
 }
 },
 {
 "id": "race/validation/2",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.004069404676556587,
 "B": 0.44272205233573914,
 "C": 0.5519668459892273,
 "D": 0.0012417305260896683
 }
 },
 {
 "id": "race/validation/371",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9914665222167969,
 "B": 0.006699902471154928,
 "C": 0.0015389310428872705,
 "D": 0.0002945860615000129
 }
 },
 {
 "id": "race/validation/3232",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 5.922771038058272e-07,
 "B": 0.004905102774500847,
 "C": 0.9950867295265198,
 "D": 7.559177447546972e-06
 }
 },
 {
 "id": "race/validation/2751",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0011786296963691711,
 "B": 0.0029728994704782963,
 "C": 0.9892282485961914,
 "D": 0.006620142143219709
 }
 },
 {
 "id": "race/validation/1815",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.39175182580947876,
 "B": 0.35501208901405334,
 "C": 0.0011111675994470716,
 "D": 0.2521248459815979
 }
 },
 {
 "id": "race/validation/2262",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 3.772380296140909e-05,
 "B": 0.9996905326843262,
 "C": 0.0002472696069162339,
 "D": 2.457850132486783e-05
 }
 },
 {
 "id": "race/validation/1628",
 "source": "race",
 "expected": "A",
 "choice": "B",
 "probabilities": {
 "A": 0.2054833024740219,
 "B": 0.4442935883998871,
 "C": 0.0016396158607676625,
 "D": 0.34858348965644836
 }
 },
 {
 "id": "race/validation/1415",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9996305704116821,
 "B": 0.00024208816466853023,
 "C": 3.8222282455535606e-05,
 "D": 8.907073060981929e-05
 }
 },
 {
 "id": "race/validation/2907",
 "source": "race",
 "expected": "B",
 "choice": "D",
 "probabilities": {
 "A": 0.0022346586920320988,
 "B": 0.005169857293367386,
 "C": 1.4572046893590596e-05,
 "D": 0.9925808906555176
 }
 },
 {
 "id": "race/validation/653",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.024042075499892235,
 "B": 0.9106132984161377,
 "C": 0.0010700344573706388,
 "D": 0.06427451223134995
 }
 },
 {
 "id": "multirc/validation/4637",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0003531142428983003,
 "no": 0.9996469020843506
 }
 },
 {
 "id": "multirc/validation/4753",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9947538375854492,
 "no": 0.005246157757937908
 }
 },
 {
 "id": "multirc/validation/263",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.007664279080927372,
 "no": 0.9923357367515564
 }
 },
 {
 "id": "multirc/validation/4765",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.03907714784145355,
 "no": 0.96092289686203
 }
 },
 {
 "id": "race/validation/960",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 4.712733698397642e-06,
 "B": 4.6360622718566447e-07,
 "C": 1.384524011882604e-06,
 "D": 0.9999934434890747
 }
 },
 {
 "id": "race/validation/1040",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 4.889486149295408e-07,
 "B": 4.866314498030988e-07,
 "C": 0.9999191761016846,
 "D": 7.990644371602684e-05
 }
 },
 {
 "id": "multirc/validation/4574",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.02829592674970627,
 "no": 0.9717041254043579
 }
 },
 {
 "id": "race/validation/205",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9990947246551514,
 "B": 0.0006225284887477756,
 "C": 7.9679986811243e-05,
 "D": 0.00020306774240452796
 }
 },
 {
 "id": "race/validation/761",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.705598771572113,
 "B": 0.00019785118638537824,
 "C": 0.2936496138572693,
 "D": 0.000553710910025984
 }
 },
 {
 "id": "race/validation/2893",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.09883786737918854,
 "B": 0.14619120955467224,
 "C": 0.028282854706048965,
 "D": 0.7266880869865417
 }
 },
 {
 "id": "race/validation/1477",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9979643821716309,
 "B": 0.001457784092053771,
 "C": 0.0003654337197076529,
 "D": 0.0002123620652128011
 }
 },
 {
 "id": "race/validation/2332",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9965044260025024,
 "B": 0.0005417391075752676,
 "C": 9.180021152133122e-06,
 "D": 0.0029447004199028015
 }
 },
 {
 "id": "multirc/validation/1638",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.003526533953845501,
 "no": 0.996473491191864
 }
 },
 {
 "id": "race/validation/2530",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.00015047207125462592,
 "B": 0.9934677481651306,
 "C": 0.000503465358633548,
 "D": 0.005878275725990534
 }
 },
 {
 "id": "multirc/validation/1631",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.08311031013727188,
 "no": 0.9168896675109863
 }
 },
 {
 "id": "race/validation/1193",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9972838163375854,
 "B": 0.0003360181872267276,
 "C": 0.0004049897543154657,
 "D": 0.0019750576466321945
 }
 },
 {
 "id": "multirc/validation/371",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.998004138469696,
 "no": 0.001995815895497799
 }
 },
 {
 "id": "race/validation/1918",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.00017407584527973086,
 "B": 0.0010894057340919971,
 "C": 2.722035969782155e-05,
 "D": 0.9987093210220337
 }
 },
 {
 "id": "multirc/validation/381",
 "source": "multirc",
 "expected": "yes",
 "choice": "no",
 "probabilities": {
 "yes": 0.34027400612831116,
 "no": 0.6597260236740112
 }
 },
 {
 "id": "race/validation/2351",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9650680422782898,
 "B": 0.00031268977909348905,
 "C": 0.033253252506256104,
 "D": 0.0013660972472280264
 }
 },
 {
 "id": "race/validation/657",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9355920553207397,
 "B": 0.0020813944283872843,
 "C": 0.004448732361197472,
 "D": 0.057877834886312485
 }
 },
 {
 "id": "multirc/validation/3164",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9965659976005554,
 "no": 0.0034340478014200926
 }
 },
 {
 "id": "multirc/validation/3099",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9915722608566284,
 "no": 0.008427808992564678
 }
 },
 {
 "id": "race/validation/3165",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.2501576520662638e-06,
 "B": 0.9999804496765137,
 "C": 1.7594265955267474e-05,
 "D": 6.820209250690823e-07
 }
 },
 {
 "id": "race/validation/2237",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9868525862693787,
 "B": 0.002693942515179515,
 "C": 0.008600855246186256,
 "D": 0.0018526819767430425
 }
 },
 {
 "id": "race/validation/1777",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.59397711740894e-06,
 "B": 0.9999133348464966,
 "C": 8.412167517235503e-05,
 "D": 9.148797630587069e-07
 }
 },
 {
 "id": "multirc/validation/3091",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00021826809097547084,
 "no": 0.9997817873954773
 }
 },
 {
 "id": "multirc/validation/3137",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9956063628196716,
 "no": 0.004393673501908779
 }
 },
 {
 "id": "race/validation/1453",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.61951919103376e-07,
 "B": 7.814623614876837e-08,
 "C": 2.331779000996903e-07,
 "D": 0.9999992847442627
 }
 },
 {
 "id": "multirc/validation/1526",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.002074792981147766,
 "no": 0.9979251623153687
 }
 },
 {
 "id": "multirc/validation/1529",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9973645806312561,
 "no": 0.0026354514993727207
 }
 },
 {
 "id": "race/validation/67",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0002445818099658936,
 "B": 0.9996472597122192,
 "C": 7.161642770370236e-06,
 "D": 0.00010093733726534992
 }
 },
 {
 "id": "race/validation/4848",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.004486673511564732,
 "B": 0.9554028511047363,
 "C": 0.02064669132232666,
 "D": 0.01946386881172657
 }
 },
 {
 "id": "race/validation/885",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0001446832320652902,
 "B": 5.292809532875253e-07,
 "C": 0.9992969036102295,
 "D": 0.0005578327691182494
 }
 },
 {
 "id": "race/validation/906",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 3.205056054866873e-05,
 "B": 0.9998800754547119,
 "C": 3.797802492044866e-05,
 "D": 4.9803147703642026e-05
 }
 },
 {
 "id": "multirc/validation/3159",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00025593320606276393,
 "no": 0.9997441172599792
 }
 },
 {
 "id": "race/validation/483",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.27509528398513794,
 "B": 0.00011581442231545225,
 "C": 0.0049004461616277695,
 "D": 0.7198883891105652
 }
 },
 {
 "id": "race/validation/719",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 5.0273585657123476e-05,
 "B": 0.003751548705622554,
 "C": 9.062302706297487e-05,
 "D": 0.9961075186729431
 }
 },
 {
 "id": "race/validation/3380",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 6.302862311713398e-05,
 "B": 5.320835612110386e-07,
 "C": 0.1561017632484436,
 "D": 0.8438346982002258
 }
 },
 {
 "id": "race/validation/3011",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.001029877457767725,
 "B": 0.9977523684501648,
 "C": 0.0006682789535261691,
 "D": 0.0005495682125911117
 }
 },
 {
 "id": "race/validation/2359",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 8.64320172695443e-05,
 "B": 1.231960737868576e-07,
 "C": 1.0363061164753162e-06,
 "D": 0.9999123811721802
 }
 },
 {
 "id": "race/validation/1335",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 1.1063040801673196e-05,
 "B": 0.9996250867843628,
 "C": 3.8312249671434984e-05,
 "D": 0.00032559013925492764
 }
 },
 {
 "id": "multirc/validation/4538",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0011409916914999485,
 "no": 0.9988590478897095
 }
 },
 {
 "id": "multirc/validation/3815",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0004464885569177568,
 "no": 0.9995535016059875
 }
 },
 {
 "id": "race/validation/2640",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.2169922229077201e-05,
 "B": 3.010456168794917e-07,
 "C": 3.325350235172664e-07,
 "D": 0.9999871253967285
 }
 },
 {
 "id": "race/validation/2672",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 4.7472894948441535e-06,
 "B": 0.9999310970306396,
 "C": 6.0638700233539566e-05,
 "D": 3.4272777611477068e-06
 }
 },
 {
 "id": "race/validation/3042",
 "source": "race",
 "expected": "B",
 "choice": "D",
 "probabilities": {
 "A": 0.002402221318334341,
 "B": 0.3793596625328064,
 "C": 5.5362615967169404e-05,
 "D": 0.6181827187538147
 }
 },
 {
 "id": "race/validation/969",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9685091376304626,
 "B": 0.013488640077412128,
 "C": 0.012777219526469707,
 "D": 0.005225073080509901
 }
 },
 {
 "id": "race/validation/985",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 8.288152457680553e-06,
 "B": 4.19240478777283e-07,
 "C": 5.530822363652987e-06,
 "D": 0.9999856948852539
 }
 },
 {
 "id": "race/validation/1769",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.0005379117792472243,
 "B": 7.186949460447067e-06,
 "C": 3.100271715084091e-05,
 "D": 0.9994239807128906
 }
 },
 {
 "id": "multirc/validation/4546",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00034663573023863137,
 "no": 0.9996533393859863
 }
 },
 {
 "id": "multirc/validation/2693",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0018296657362952828,
 "no": 0.9981703758239746
 }
 },
 {
 "id": "race/validation/660",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0005507050082087517,
 "B": 0.9982957243919373,
 "C": 0.00025140316574834287,
 "D": 0.000902156752999872
 }
 },
 {
 "id": "multirc/validation/3820",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9870573878288269,
 "no": 0.012942573055624962
 }
 },
 {
 "id": "race/validation/2334",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 5.430006240203511e-06,
 "B": 0.9997798800468445,
 "C": 0.00017318241589237005,
 "D": 4.144811100559309e-05
 }
 },
 {
 "id": "multirc/validation/1513",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00013282886357046664,
 "no": 0.9998672008514404
 }
 },
 {
 "id": "contractnli/validation/456/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9965811371803284,
 "Contradiction": 0.0003880851436406374,
 "NotMentioned": 0.0030307937413454056
 }
 },
 {
 "id": "race/validation/17",
 "source": "race",
 "expected": "D",
 "choice": "B",
 "probabilities": {
 "A": 0.10918726772069931,
 "B": 0.8729928731918335,
 "C": 0.0063325753435492516,
 "D": 0.011487231589853764
 }
 },
 {
 "id": "multirc/validation/2733",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9804975986480713,
 "no": 0.0195024311542511
 }
 },
 {
 "id": "race/validation/2752",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 8.3371669461485e-06,
 "B": 0.00011929423635592684,
 "C": 0.001434285193681717,
 "D": 0.9984380602836609
 }
 },
 {
 "id": "contractnli/validation/456/nda-13",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 3.165235830238089e-05,
 "Contradiction": 3.134813960059546e-05,
 "NotMentioned": 0.9999369382858276
 }
 },
 {
 "id": "race/validation/2700",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.7045970202598255e-07,
 "B": 7.767631586830248e-07,
 "C": 7.881510100560263e-05,
 "D": 0.9999200105667114
 }
 },
 {
 "id": "contractnli/validation/456/nda-16",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0001507936103735119,
 "Contradiction": 7.597448711749166e-05,
 "NotMentioned": 0.9997733235359192
 }
 },
 {
 "id": "multirc/validation/2742",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.000523228314705193,
 "no": 0.9994768500328064
 }
 },
 {
 "id": "multirc/validation/3805",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9979493021965027,
 "no": 0.0020507301669567823
 }
 },
 {
 "id": "race/validation/2409",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0006941634928807616,
 "B": 1.295979643600731e-07,
 "C": 0.999302864074707,
 "D": 2.8973777261853684e-06
 }
 },
 {
 "id": "race/validation/557",
 "source": "race",
 "expected": "D",
 "choice": "B",
 "probabilities": {
 "A": 0.003499878803268075,
 "B": 0.5640116333961487,
 "C": 0.41004428267478943,
 "D": 0.02244424633681774
 }
 },
 {
 "id": "race/validation/2255",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.862678050994873,
 "B": 0.009676054120063782,
 "C": 0.04453659430146217,
 "D": 0.08310935646295547
 }
 },
 {
 "id": "race/validation/100",
 "source": "race",
 "expected": "A",
 "choice": "D",
 "probabilities": {
 "A": 0.14800965785980225,
 "B": 0.005747355055063963,
 "C": 0.06288663297891617,
 "D": 0.7833563089370728
 }
 },
 {
 "id": "race/validation/2817",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9100548624992371,
 "B": 0.011426461860537529,
 "C": 0.0005372281302697957,
 "D": 0.07798144966363907
 }
 },
 {
 "id": "race/validation/1143",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.00013891089474782348,
 "B": 1.041305608850962e-06,
 "C": 0.00013586354907602072,
 "D": 0.9997240900993347
 }
 },
 {
 "id": "race/validation/1465",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.002397629665210843,
 "B": 0.9961322546005249,
 "C": 0.0008283344213850796,
 "D": 0.000641843129415065
 }
 },
 {
 "id": "multirc/validation/4395",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0022723544389009476,
 "no": 0.9977276921272278
 }
 },
 {
 "id": "multirc/validation/4390",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.7761244773864746,
 "no": 0.22387546300888062
 }
 },
 {
 "id": "race/validation/982",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.9998132586479187,
 "B": 1.3839015991834458e-05,
 "C": 0.00016114243771880865,
 "D": 1.1767241630877834e-05
 }
 },
 {
 "id": "multirc/validation/499",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.03236280009150505,
 "no": 0.9676372408866882
 }
 },
 {
 "id": "multirc/validation/511",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0007121323724277318,
 "no": 0.9992879033088684
 }
 },
 {
 "id": "race/validation/1515",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.0042247893288731575,
 "B": 0.9823984503746033,
 "C": 0.0030620680190622807,
 "D": 0.010314698331058025
 }
 },
 {
 "id": "multirc/validation/3874",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0006189053528942168,
 "no": 0.9993810653686523
 }
 },
 {
 "id": "multirc/validation/3307",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.31762784719467163,
 "no": 0.6823721528053284
 }
 },
 {
 "id": "multirc/validation/3304",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.33744966983795166,
 "no": 0.6625503301620483
 }
 },
 {
 "id": "multirc/validation/3713",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9961394667625427,
 "no": 0.0038605055306106806
 }
 },
 {
 "id": "multirc/validation/3706",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00040938222082331777,
 "no": 0.9995905756950378
 }
 },
 {
 "id": "multirc/validation/3708",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.006122618447989225,
 "no": 0.9938773512840271
 }
 },
 {
 "id": "multirc/validation/1730",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0008747418760322034,
 "no": 0.9991253018379211
 }
 },
 {
 "id": "multirc/validation/3893",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.993104875087738,
 "no": 0.006895055528730154
 }
 },
 {
 "id": "race/validation/3206",
 "source": "race",
 "expected": "A",
 "choice": "A",
 "probabilities": {
 "A": 0.7240117192268372,
 "B": 0.006122963968664408,
 "C": 0.2689655125141144,
 "D": 0.0008998332195915282
 }
 },
 {
 "id": "multirc/validation/3891",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.00152165989857167,
 "no": 0.998478353023529
 }
 },
 {
 "id": "multirc/validation/498",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9978736639022827,
 "no": 0.002126375213265419
 }
 },
 {
 "id": "race/validation/838",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 1.3910799907534965e-06,
 "B": 1.232460363098653e-06,
 "C": 2.5019819531735266e-06,
 "D": 0.9999948740005493
 }
 },
 {
 "id": "race/validation/1213",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 3.838039447146002e-06,
 "B": 3.2065010600490496e-05,
 "C": 0.00046965351793915033,
 "D": 0.9994944334030151
 }
 },
 {
 "id": "multirc/validation/1723",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.0023954291827976704,
 "no": 0.9976045489311218
 }
 },
 {
 "id": "race/validation/2738",
 "source": "race",
 "expected": "D",
 "choice": "C",
 "probabilities": {
 "A": 0.4867320954799652,
 "B": 0.007449783850461245,
 "C": 0.4991115629673004,
 "D": 0.006706551183015108
 }
 },
 {
 "id": "multirc/validation/1718",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.973453938961029,
 "no": 0.026545995846390724
 }
 },
 {
 "id": "multirc/validation/1940",
 "source": "multirc",
 "expected": "no",
 "choice": "yes",
 "probabilities": {
 "yes": 0.8675079941749573,
 "no": 0.13249197602272034
 }
 },
 {
 "id": "race/validation/3132",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 8.451202120340895e-07,
 "B": 0.9327335357666016,
 "C": 9.730573765409645e-06,
 "D": 0.067255899310112
 }
 },
 {
 "id": "race/validation/852",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.014822081662714481,
 "B": 0.9696958661079407,
 "C": 0.015287650749087334,
 "D": 0.00019446818623691797
 }
 },
 {
 "id": "multirc/validation/1934",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.004276244435459375,
 "no": 0.9957237243652344
 }
 },
 {
 "id": "race/validation/2978",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.0001301271258853376,
 "B": 2.3581681034556823e-06,
 "C": 0.999841570854187,
 "D": 2.5901177650666796e-05
 }
 },
 {
 "id": "race/validation/2963",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00022638421796727926,
 "B": 0.14131110906600952,
 "C": 0.8566492199897766,
 "D": 0.0018132581608369946
 }
 },
 {
 "id": "multirc/validation/1931",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9808628559112549,
 "no": 0.019137099385261536
 }
 },
 {
 "id": "race/validation/3254",
 "source": "race",
 "expected": "B",
 "choice": "A",
 "probabilities": {
 "A": 0.7435370087623596,
 "B": 0.2281307727098465,
 "C": 0.028317857533693314,
 "D": 1.4361596186063252e-05
 }
 },
 {
 "id": "multirc/validation/4305",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9949144124984741,
 "no": 0.005085616838186979
 }
 },
 {
 "id": "multirc/validation/4352",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.010053394362330437,
 "no": 0.9899466633796692
 }
 },
 {
 "id": "race/validation/1624",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 0.00018346463912166655,
 "B": 0.003026586724445224,
 "C": 0.6869723200798035,
 "D": 0.3098176121711731
 }
 },
 {
 "id": "contractnli/validation/71/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0012212529545649886,
 "Contradiction": 0.0007221737178042531,
 "NotMentioned": 0.9980564713478088
 }
 },
 {
 "id": "contractnli/validation/71/nda-10",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.996793806552887,
 "Contradiction": 0.00031230784952640533,
 "NotMentioned": 0.002893900964409113
 }
 },
 {
 "id": "contractnli/validation/71/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9985210299491882,
 "Contradiction": 0.00039074852247722447,
 "NotMentioned": 0.0010882255155593157
 }
 },
 {
 "id": "multirc/validation/725",
 "source": "multirc",
 "expected": "yes",
 "choice": "yes",
 "probabilities": {
 "yes": 0.9940046668052673,
 "no": 0.005995406303554773
 }
 },
 {
 "id": "multirc/validation/724",
 "source": "multirc",
 "expected": "no",
 "choice": "no",
 "probabilities": {
 "yes": 0.01109037734568119,
 "no": 0.9889096617698669
 }
 },
 {
 "id": "race/validation/2681",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 4.802384864888154e-05,
 "B": 0.0025009820237755775,
 "C": 4.206666289974237e-06,
 "D": 0.9974468946456909
 }
 },
 {
 "id": "race/validation/2304",
 "source": "race",
 "expected": "B",
 "choice": "B",
 "probabilities": {
 "A": 0.1888645440340042,
 "B": 0.8100541234016418,
 "C": 0.0005590380169451237,
 "D": 0.0005222075269557536
 }
 },
 {
 "id": "race/validation/2003",
 "source": "race",
 "expected": "D",
 "choice": "D",
 "probabilities": {
 "A": 0.005358066409826279,
 "B": 3.335379233249114e-06,
 "C": 7.091784937074408e-06,
 "D": 0.9946315288543701
 }
 },
 {
 "id": "race/validation/2196",
 "source": "race",
 "expected": "C",
 "choice": "C",
 "probabilities": {
 "A": 2.129019776475616e-06,
 "B": 1.8386284637017525e-06,
 "C": 0.999984622001648,
 "D": 1.1388916391297244e-05
 }
 },
 {
 "id": "contractnli/validation/488/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.997462272644043,
 "Contradiction": 0.00031722537823952734,
 "NotMentioned": 0.002220487454906106
 }
 },
 {
 "id": "contractnli/validation/488/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00037500052712857723,
 "Contradiction": 0.000551376782823354,
 "NotMentioned": 0.9990735054016113
 }
 },
 {
 "id": "contractnli/validation/488/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0015429443446919322,
 "Contradiction": 0.13570807874202728,
 "NotMentioned": 0.8627490401268005
 }
 },
 {
 "id": "contractnli/validation/488/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9895880222320557,
 "Contradiction": 0.0003672668244689703,
 "NotMentioned": 0.010044755414128304
 }
 },
 {
 "id": "contractnli/validation/15/nda-3",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9996223449707031,
 "Contradiction": 8.31313882372342e-05,
 "NotMentioned": 0.00029445410473272204
 }
 },
 {
 "id": "contractnli/validation/15/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.33140403032302856,
 "Contradiction": 0.0019812670070677996,
 "NotMentioned": 0.6666147708892822
 }
 },
 {
 "id": "contractnli/validation/15/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990091323852539,
 "Contradiction": 0.0001793853734852746,
 "NotMentioned": 0.0008115207892842591
 }
 },
 {
 "id": "contractnli/validation/15/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 7.028235268080607e-05,
 "Contradiction": 5.64973470318364e-06,
 "NotMentioned": 0.9999240636825562
 }
 },
 {
 "id": "contractnli/validation/482/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.998884379863739,
 "Contradiction": 0.0002010147727560252,
 "NotMentioned": 0.0009145927615463734
 }
 },
 {
 "id": "contractnli/validation/482/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9995307922363281,
 "Contradiction": 9.208283881889656e-05,
 "NotMentioned": 0.0003771478950511664
 }
 },
 {
 "id": "contractnli/validation/482/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0089107109233737,
 "Contradiction": 0.250326931476593,
 "NotMentioned": 0.7407623529434204
 }
 },
 {
 "id": "contractnli/validation/14/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0035004932433366776,
 "Contradiction": 0.0007698963745497167,
 "NotMentioned": 0.9957296252250671
 }
 },
 {
 "id": "contractnli/validation/14/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0006633502198383212,
 "Contradiction": 0.0019403654150664806,
 "NotMentioned": 0.9973963499069214
 }
 },
 {
 "id": "contractnli/validation/14/nda-7",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.008146451786160469,
 "Contradiction": 0.33503904938697815,
 "NotMentioned": 0.656814455986023
 }
 },
 {
 "id": "contractnli/validation/29/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991047978401184,
 "Contradiction": 0.0001853104040492326,
 "NotMentioned": 0.0007098987698554993
 }
 },
 {
 "id": "contractnli/validation/29/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 3.542157355695963e-05,
 "Contradiction": 6.983820640016347e-06,
 "NotMentioned": 0.9999575614929199
 }
 },
 {
 "id": "contractnli/validation/29/nda-13",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9780744314193726,
 "Contradiction": 0.0024490272626280785,
 "NotMentioned": 0.019476531073451042
 }
 },
 {
 "id": "contractnli/validation/82/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00016279022383969277,
 "Contradiction": 0.0003128974058199674,
 "NotMentioned": 0.9995243549346924
 }
 },
 {
 "id": "contractnli/validation/82/nda-18",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9985359907150269,
 "Contradiction": 0.00014283259224612266,
 "NotMentioned": 0.0013212155317887664
 }
 },
 {
 "id": "contractnli/validation/82/nda-20",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0008204326150007546,
 "Contradiction": 0.9947721362113953,
 "NotMentioned": 0.004407443106174469
 }
 },
 {
 "id": "contractnli/validation/39/nda-3",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00041341775795444846,
 "Contradiction": 0.00011911648471141234,
 "NotMentioned": 0.9994674324989319
 }
 },
 {
 "id": "contractnli/validation/39/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0006254288600757718,
 "Contradiction": 0.00042518190457485616,
 "NotMentioned": 0.9989494681358337
 }
 },
 {
 "id": "contractnli/validation/44/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0002952871145680547,
 "Contradiction": 0.9944650530815125,
 "NotMentioned": 0.005239663179963827
 }
 },
 {
 "id": "contractnli/validation/44/nda-19",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9973294734954834,
 "Contradiction": 0.00047274911776185036,
 "NotMentioned": 0.0021978369913995266
 }
 },
 {
 "id": "contractnli/validation/39/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.004096286837011576,
 "Contradiction": 0.0019738152623176575,
 "NotMentioned": 0.9939298629760742
 }
 },
 {
 "id": "contractnli/validation/39/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 8.739210898056626e-05,
 "Contradiction": 1.3194247003411874e-05,
 "NotMentioned": 0.9998993873596191
 }
 },
 {
 "id": "contractnli/validation/44/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 4.7972869651857764e-05,
 "Contradiction": 1.0397821824881248e-05,
 "NotMentioned": 0.9999417066574097
 }
 },
 {
 "id": "contractnli/validation/435/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0007479495834559202,
 "Contradiction": 0.9915722608566284,
 "NotMentioned": 0.007679781876504421
 }
 },
 {
 "id": "contractnli/validation/435/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.001966954907402396,
 "Contradiction": 0.00014397143968380988,
 "NotMentioned": 0.9978891015052795
 }
 },
 {
 "id": "contractnli/validation/435/nda-16",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0006516944267787039,
 "Contradiction": 0.0001788125664461404,
 "NotMentioned": 0.9991695880889893
 }
 },
 {
 "id": "contractnli/validation/435/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.010295779444277287,
 "Contradiction": 0.015234855003654957,
 "NotMentioned": 0.9744693040847778
 }
 },
 {
 "id": "contractnli/validation/70/nda-19",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992928504943848,
 "Contradiction": 0.0001320904411841184,
 "NotMentioned": 0.0005750022828578949
 }
 },
 {
 "id": "contractnli/validation/70/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0003811088972724974,
 "Contradiction": 3.2035939511843026e-05,
 "NotMentioned": 0.9995868802070618
 }
 },
 {
 "id": "contractnli/validation/81/nda-1",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.004860596731305122,
 "Contradiction": 0.989376425743103,
 "NotMentioned": 0.005763036198914051
 }
 },
 {
 "id": "contractnli/validation/81/nda-13",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992049336433411,
 "Contradiction": 0.00016913044964894652,
 "NotMentioned": 0.0006259277579374611
 }
 },
 {
 "id": "contractnli/validation/81/nda-16",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.17440341413021088,
 "Contradiction": 0.012224261648952961,
 "NotMentioned": 0.8133723139762878
 }
 },
 {
 "id": "contractnli/validation/70/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 4.155438000452705e-05,
 "Contradiction": 3.2705411285860464e-06,
 "NotMentioned": 0.9999551773071289
 }
 },
 {
 "id": "contractnli/validation/81/nda-7",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.049095433205366135,
 "Contradiction": 0.9273714423179626,
 "NotMentioned": 0.02353312075138092
 }
 },
 {
 "id": "contractnli/validation/45/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 9.883009624900296e-05,
 "Contradiction": 9.517985017737374e-05,
 "NotMentioned": 0.9998059868812561
 }
 },
 {
 "id": "contractnli/validation/45/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.998755693435669,
 "Contradiction": 0.00040878431173041463,
 "NotMentioned": 0.0008355248137377203
 }
 },
 {
 "id": "contractnli/validation/45/nda-8",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0006122682243585587,
 "Contradiction": 0.00014457227371167392,
 "NotMentioned": 0.9992431402206421
 }
 },
 {
 "id": "contractnli/validation/152/nda-3",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.001501785241998732,
 "Contradiction": 0.00020980479894205928,
 "NotMentioned": 0.9982884526252747
 }
 },
 {
 "id": "contractnli/validation/152/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0004044778470415622,
 "Contradiction": 0.9984117746353149,
 "NotMentioned": 0.0011837456841021776
 }
 },
 {
 "id": "contractnli/validation/152/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9989805817604065,
 "Contradiction": 0.00026917996001429856,
 "NotMentioned": 0.0007503111846745014
 }
 },
 {
 "id": "contractnli/validation/20/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9988665580749512,
 "Contradiction": 0.00021033230586908758,
 "NotMentioned": 0.0009231711737811565
 }
 },
 {
 "id": "contractnli/validation/20/nda-17",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9224168062210083,
 "Contradiction": 0.07100124657154083,
 "NotMentioned": 0.006581959780305624
 }
 },
 {
 "id": "contractnli/validation/20/nda-16",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9987150430679321,
 "Contradiction": 0.0003008991770911962,
 "NotMentioned": 0.000984146841801703
 }
 },
 {
 "id": "contractnli/validation/20/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0012252918677404523,
 "Contradiction": 0.9905361533164978,
 "NotMentioned": 0.008238565176725388
 }
 },
 {
 "id": "contractnli/validation/547/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 4.53415050287731e-05,
 "Contradiction": 2.927294553956017e-05,
 "NotMentioned": 0.9999253749847412
 }
 },
 {
 "id": "contractnli/validation/507/nda-12",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0015874725067988038,
 "Contradiction": 0.0005780116771347821,
 "NotMentioned": 0.9978345036506653
 }
 },
 {
 "id": "contractnli/validation/507/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 6.281973764998838e-05,
 "Contradiction": 0.00018107421055901796,
 "NotMentioned": 0.9997561573982239
 }
 },
 {
 "id": "contractnli/validation/507/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 6.854314415249974e-05,
 "Contradiction": 9.830264389165677e-06,
 "NotMentioned": 0.9999216794967651
 }
 },
 {
 "id": "contractnli/validation/547/nda-7",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.8972631096839905,
 "Contradiction": 0.05133073776960373,
 "NotMentioned": 0.05140607804059982
 }
 },
 {
 "id": "contractnli/validation/547/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990761280059814,
 "Contradiction": 0.00017320729966741055,
 "NotMentioned": 0.0007507175905629992
 }
 },
 {
 "id": "contractnli/validation/79/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.17060528695583344,
 "Contradiction": 0.14009416103363037,
 "NotMentioned": 0.6893005967140198
 }
 },
 {
 "id": "contractnli/validation/79/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9927825927734375,
 "Contradiction": 0.0011101594427600503,
 "NotMentioned": 0.006107315421104431
 }
 },
 {
 "id": "contractnli/validation/79/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00011573889059945941,
 "Contradiction": 0.0003506735956761986,
 "NotMentioned": 0.9995336532592773
 }
 },
 {
 "id": "contractnli/validation/486/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.00025397847639396787,
 "Contradiction": 0.9988735318183899,
 "NotMentioned": 0.0008725234074518085
 }
 },
 {
 "id": "contractnli/validation/486/nda-13",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00819859653711319,
 "Contradiction": 0.004146002233028412,
 "NotMentioned": 0.9876554608345032
 }
 },
 {
 "id": "contractnli/validation/486/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0002720006159506738,
 "Contradiction": 0.0008717261371202767,
 "NotMentioned": 0.9988563060760498
 }
 },
 {
 "id": "contractnli/validation/486/nda-7",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9959278702735901,
 "Contradiction": 0.001557601965032518,
 "NotMentioned": 0.002514422405511141
 }
 },
 {
 "id": "contractnli/validation/54/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9974563717842102,
 "Contradiction": 0.0005863930564373732,
 "NotMentioned": 0.0019572493620216846
 }
 },
 {
 "id": "contractnli/validation/54/nda-12",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00015452346997335553,
 "Contradiction": 0.00013625169231090695,
 "NotMentioned": 0.9997091889381409
 }
 },
 {
 "id": "contractnli/validation/54/nda-11",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.8269205093383789,
 "Contradiction": 0.00017177921836264431,
 "NotMentioned": 0.1729077249765396
 }
 },
 {
 "id": "contractnli/validation/75/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 7.665920566068962e-05,
 "Contradiction": 0.0001577367220306769,
 "NotMentioned": 0.9997655749320984
 }
 },
 {
 "id": "contractnli/validation/75/nda-16",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.42156338691711426,
 "Contradiction": 0.006398885976523161,
 "NotMentioned": 0.5720377564430237
 }
 },
 {
 "id": "contractnli/validation/75/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 3.6852998164249584e-05,
 "Contradiction": 4.3475388338265475e-06,
 "NotMentioned": 0.999958872795105
 }
 },
 {
 "id": "contractnli/validation/12/nda-3",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0003922322648577392,
 "Contradiction": 6.751613545930013e-05,
 "NotMentioned": 0.9995402097702026
 }
 },
 {
 "id": "contractnli/validation/12/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00010380763706052676,
 "Contradiction": 8.895558494259603e-06,
 "NotMentioned": 0.999887228012085
 }
 },
 {
 "id": "contractnli/validation/12/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 7.063372322591022e-05,
 "Contradiction": 2.7971113013336435e-05,
 "NotMentioned": 0.999901294708252
 }
 },
 {
 "id": "contractnli/validation/12/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990994930267334,
 "Contradiction": 0.00018922470917459577,
 "NotMentioned": 0.0007112661842256784
 }
 },
 {
 "id": "contractnli/validation/406/nda-19",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.030343493446707726,
 "Contradiction": 0.0012831002241000533,
 "NotMentioned": 0.9683733582496643
 }
 },
 {
 "id": "contractnli/validation/406/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 6.146272789919749e-05,
 "Contradiction": 9.999125722970348e-06,
 "NotMentioned": 0.9999284744262695
 }
 },
 {
 "id": "contractnli/validation/19/nda-3",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9996323585510254,
 "Contradiction": 7.729172648396343e-05,
 "NotMentioned": 0.00029027508571743965
 }
 },
 {
 "id": "contractnli/validation/16/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9978220462799072,
 "Contradiction": 0.0003461452724877745,
 "NotMentioned": 0.0018318280344828963
 }
 },
 {
 "id": "contractnli/validation/406/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00036654502036981285,
 "Contradiction": 1.985421113204211e-05,
 "NotMentioned": 0.9996135830879211
 }
 },
 {
 "id": "contractnli/validation/16/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 3.616134563344531e-05,
 "Contradiction": 2.8186543659103336e-06,
 "NotMentioned": 0.9999610185623169
 }
 },
 {
 "id": "contractnli/validation/19/nda-13",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00013074408343527466,
 "Contradiction": 7.46028745197691e-05,
 "NotMentioned": 0.9997946619987488
 }
 },
 {
 "id": "contractnli/validation/16/nda-4",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9988743662834167,
 "Contradiction": 0.0003616347967181355,
 "NotMentioned": 0.0007640065159648657
 }
 },
 {
 "id": "contractnli/validation/19/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9993605017662048,
 "Contradiction": 0.0001942118542501703,
 "NotMentioned": 0.0004453076689969748
 }
 },
 {
 "id": "contractnli/validation/28/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9967136383056641,
 "Contradiction": 0.0017690841341391206,
 "NotMentioned": 0.0015173200517892838
 }
 },
 {
 "id": "contractnli/validation/28/nda-16",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9987567663192749,
 "Contradiction": 0.00023934489581733942,
 "NotMentioned": 0.001003915793262422
 }
 },
 {
 "id": "contractnli/validation/28/nda-20",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0005052703781984746,
 "Contradiction": 0.9974997639656067,
 "NotMentioned": 0.0019950089044868946
 }
 },
 {
 "id": "contractnli/validation/28/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.06863702833652496,
 "Contradiction": 0.0007247644243761897,
 "NotMentioned": 0.9306381940841675
 }
 },
 {
 "id": "contractnli/validation/411/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0002375720941927284,
 "Contradiction": 1.624887590878643e-05,
 "NotMentioned": 0.9997462630271912
 }
 },
 {
 "id": "contractnli/validation/411/nda-5",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.01025762502104044,
 "Contradiction": 0.009900362230837345,
 "NotMentioned": 0.9798420667648315
 }
 },
 {
 "id": "contractnli/validation/411/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991914629936218,
 "Contradiction": 0.00017879073857329786,
 "NotMentioned": 0.0006297414074651897
 }
 },
 {
 "id": "contractnli/validation/575/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.998999297618866,
 "Contradiction": 0.0002520877751521766,
 "NotMentioned": 0.0007486501708626747
 }
 },
 {
 "id": "contractnli/validation/575/nda-16",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9951847195625305,
 "Contradiction": 0.0007953098975121975,
 "NotMentioned": 0.004019950982183218
 }
 },
 {
 "id": "contractnli/validation/575/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9985664486885071,
 "Contradiction": 0.0004977877251803875,
 "NotMentioned": 0.000935746356844902
 }
 },
 {
 "id": "contractnli/validation/3/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0003358681278768927,
 "Contradiction": 0.9992966651916504,
 "NotMentioned": 0.00036742317024618387
 }
 },
 {
 "id": "contractnli/validation/63/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990864992141724,
 "Contradiction": 0.00029940009699203074,
 "NotMentioned": 0.0006140631739981472
 }
 },
 {
 "id": "contractnli/validation/3/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9981086254119873,
 "Contradiction": 0.00029691754025407135,
 "NotMentioned": 0.0015945371706038713
 }
 },
 {
 "id": "contractnli/validation/63/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 5.502131534740329e-05,
 "Contradiction": 7.976621418492869e-06,
 "NotMentioned": 0.9999369382858276
 }
 },
 {
 "id": "contractnli/validation/63/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991069436073303,
 "Contradiction": 0.0002541531575843692,
 "NotMentioned": 0.0006388258188962936
 }
 },
 {
 "id": "contractnli/validation/3/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992536902427673,
 "Contradiction": 0.00019121853983961046,
 "NotMentioned": 0.0005550553905777633
 }
 },
 {
 "id": "contractnli/validation/440/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990422129631042,
 "Contradiction": 0.00018538717995397747,
 "NotMentioned": 0.0007723545422777534
 }
 },
 {
 "id": "contractnli/validation/440/nda-20",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.014901434071362019,
 "Contradiction": 0.6371889710426331,
 "NotMentioned": 0.3479095697402954
 }
 },
 {
 "id": "contractnli/validation/440/nda-11",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9988671541213989,
 "Contradiction": 0.0002001713146455586,
 "NotMentioned": 0.000932728813495487
 }
 },
 {
 "id": "contractnli/validation/56/nda-3",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9994431138038635,
 "Contradiction": 0.0001467236434109509,
 "NotMentioned": 0.0004102631064597517
 }
 },
 {
 "id": "contractnli/validation/598/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9994538426399231,
 "Contradiction": 0.00010616254439810291,
 "NotMentioned": 0.0004399673198349774
 }
 },
 {
 "id": "contractnli/validation/598/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992031455039978,
 "Contradiction": 0.0001992914330912754,
 "NotMentioned": 0.0005975568201392889
 }
 },
 {
 "id": "contractnli/validation/598/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 4.019035986857489e-05,
 "Contradiction": 5.560078534472268e-06,
 "NotMentioned": 0.9999542236328125
 }
 },
 {
 "id": "contractnli/validation/56/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0003600049240048975,
 "Contradiction": 0.002075626514852047,
 "NotMentioned": 0.997564435005188
 }
 },
 {
 "id": "contractnli/validation/56/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0015825730515643954,
 "Contradiction": 0.00010634720820235088,
 "NotMentioned": 0.9983110427856445
 }
 },
 {
 "id": "contractnli/validation/61/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00030640579643659294,
 "Contradiction": 0.0012519150041043758,
 "NotMentioned": 0.998441755771637
 }
 },
 {
 "id": "contractnli/validation/61/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 6.71193702146411e-05,
 "Contradiction": 5.823882020195015e-06,
 "NotMentioned": 0.9999270439147949
 }
 },
 {
 "id": "contractnli/validation/61/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9989911913871765,
 "Contradiction": 0.00014921363617759198,
 "NotMentioned": 0.0008596346015110612
 }
 },
 {
 "id": "contractnli/validation/563/nda-12",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992904663085938,
 "Contradiction": 0.00014873275358695537,
 "NotMentioned": 0.0005607791827060282
 }
 },
 {
 "id": "contractnli/validation/563/nda-18",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991135001182556,
 "Contradiction": 0.00020902158576063812,
 "NotMentioned": 0.0006775206420570612
 }
 },
 {
 "id": "contractnli/validation/563/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9988425374031067,
 "Contradiction": 0.0004726495244540274,
 "NotMentioned": 0.000684870348777622
 }
 },
 {
 "id": "contractnli/validation/563/nda-7",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9965223073959351,
 "Contradiction": 0.0017427499406039715,
 "NotMentioned": 0.0017349623376503587
 }
 },
 {
 "id": "contractnli/validation/605/nda-19",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.997252881526947,
 "Contradiction": 0.0007693846710026264,
 "NotMentioned": 0.001977713545784354
 }
 },
 {
 "id": "contractnli/validation/605/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 5.8861496654571965e-05,
 "Contradiction": 8.193018402380403e-06,
 "NotMentioned": 0.9999328851699829
 }
 },
 {
 "id": "contractnli/validation/605/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00011054577043978497,
 "Contradiction": 4.5031800254946575e-05,
 "NotMentioned": 0.9998444318771362
 }
 },
 {
 "id": "contractnli/validation/27/nda-3",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.999727189540863,
 "Contradiction": 7.691651990171522e-05,
 "NotMentioned": 0.00019596904166974127
 }
 },
 {
 "id": "contractnli/validation/605/nda-7",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9503779411315918,
 "Contradiction": 0.017501166090369225,
 "NotMentioned": 0.03212082013487816
 }
 },
 {
 "id": "contractnli/validation/27/nda-1",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.001311584492214024,
 "Contradiction": 0.0588676817715168,
 "NotMentioned": 0.9398207664489746
 }
 },
 {
 "id": "contractnli/validation/27/nda-17",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.09433075785636902,
 "Contradiction": 0.5907195210456848,
 "NotMentioned": 0.31494972109794617
 }
 },
 {
 "id": "contractnli/validation/478/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 5.821326340083033e-05,
 "Contradiction": 5.9029439398727845e-06,
 "NotMentioned": 0.9999358654022217
 }
 },
 {
 "id": "contractnli/validation/478/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.18230092525482178,
 "Contradiction": 0.8031083941459656,
 "NotMentioned": 0.014590674079954624
 }
 },
 {
 "id": "contractnli/validation/478/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 7.156498759286478e-05,
 "Contradiction": 3.6119247397436993e-06,
 "NotMentioned": 0.999924898147583
 }
 },
 {
 "id": "contractnli/validation/478/nda-20",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.005647844634950161,
 "Contradiction": 0.8917930722236633,
 "NotMentioned": 0.10255907475948334
 }
 },
 {
 "id": "contractnli/validation/10/nda-1",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.00515627721324563,
 "Contradiction": 0.991396963596344,
 "NotMentioned": 0.003446840913966298
 }
 },
 {
 "id": "contractnli/validation/10/nda-10",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9969711303710938,
 "Contradiction": 0.0004097595810890198,
 "NotMentioned": 0.0026190674398094416
 }
 },
 {
 "id": "contractnli/validation/10/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9857984781265259,
 "Contradiction": 0.005594219546765089,
 "NotMentioned": 0.008607237599790096
 }
 },
 {
 "id": "contractnli/validation/10/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9994679093360901,
 "Contradiction": 0.00012648825941141695,
 "NotMentioned": 0.0004055367608089
 }
 },
 {
 "id": "contractnli/validation/33/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00022182757675182074,
 "Contradiction": 0.0011198221473023295,
 "NotMentioned": 0.9986584186553955
 }
 },
 {
 "id": "contractnli/validation/33/nda-13",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991596937179565,
 "Contradiction": 0.00017770222621038556,
 "NotMentioned": 0.0006626327522099018
 }
 },
 {
 "id": "contractnli/validation/33/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992994070053101,
 "Contradiction": 0.00018928118515759706,
 "NotMentioned": 0.0005112970829941332
 }
 },
 {
 "id": "contractnli/validation/582/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 8.687972149346024e-05,
 "Contradiction": 0.00018515100236982107,
 "NotMentioned": 0.9997279047966003
 }
 },
 {
 "id": "contractnli/validation/582/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0012396300444379449,
 "Contradiction": 0.0010215295478701591,
 "NotMentioned": 0.9977388381958008
 }
 },
 {
 "id": "contractnli/validation/582/nda-13",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9962196946144104,
 "Contradiction": 0.0005156259867362678,
 "NotMentioned": 0.0032646828331053257
 }
 },
 {
 "id": "contractnli/validation/205/nda-2",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0004354385891929269,
 "Contradiction": 0.9988823533058167,
 "NotMentioned": 0.0006822472787462175
 }
 },
 {
 "id": "contractnli/validation/205/nda-20",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9969283938407898,
 "Contradiction": 0.0011108701582998037,
 "NotMentioned": 0.0019608342554420233
 }
 },
 {
 "id": "contractnli/validation/205/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9993147850036621,
 "Contradiction": 0.0001565952115925029,
 "NotMentioned": 0.0005285341176204383
 }
 },
 {
 "id": "contractnli/validation/405/nda-19",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9781920313835144,
 "Contradiction": 0.003753121942281723,
 "NotMentioned": 0.01805492304265499
 }
 },
 {
 "id": "contractnli/validation/405/nda-18",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9979653358459473,
 "Contradiction": 0.00025154341710731387,
 "NotMentioned": 0.0017832270823419094
 }
 },
 {
 "id": "contractnli/validation/405/nda-20",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.0009807647438719869,
 "Contradiction": 0.9963070154190063,
 "NotMentioned": 0.002712240209802985
 }
 },
 {
 "id": "contractnli/validation/398/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.003243175568059087,
 "Contradiction": 0.003216218901798129,
 "NotMentioned": 0.9935405850410461
 }
 },
 {
 "id": "contractnli/validation/398/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 5.5301083193626255e-05,
 "Contradiction": 9.367662642034702e-06,
 "NotMentioned": 0.9999352693557739
 }
 },
 {
 "id": "contractnli/validation/398/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9972442388534546,
 "Contradiction": 0.0008102926658466458,
 "NotMentioned": 0.0019454638240858912
 }
 },
 {
 "id": "contractnli/validation/32/nda-16",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9988089799880981,
 "Contradiction": 0.0001849388499977067,
 "NotMentioned": 0.001006108010187745
 }
 },
 {
 "id": "contractnli/validation/32/nda-7",
 "source": "contractnli",
 "expected": "Contradiction",
 "choice": "Contradiction",
 "probabilities": {
 "Entailment": 0.002058574231341481,
 "Contradiction": 0.9958974123001099,
 "NotMentioned": 0.0020440821535885334
 }
 },
 {
 "id": "contractnli/validation/32/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990357160568237,
 "Contradiction": 0.0001755582488840446,
 "NotMentioned": 0.0007887361571192741
 }
 },
 {
 "id": "contractnli/validation/381/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 5.9524936659727246e-05,
 "Contradiction": 9.920291631715372e-06,
 "NotMentioned": 0.9999306201934814
 }
 },
 {
 "id": "contractnli/validation/381/nda-7",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.5116617679595947,
 "Contradiction": 0.47110867500305176,
 "NotMentioned": 0.017229538410902023
 }
 },
 {
 "id": "contractnli/validation/381/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991258978843689,
 "Contradiction": 0.00017495048814453185,
 "NotMentioned": 0.0006991469417698681
 }
 },
 {
 "id": "contractnli/validation/17/nda-3",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.04023417457938194,
 "Contradiction": 0.0038296582642942667,
 "NotMentioned": 0.9559361934661865
 }
 },
 {
 "id": "contractnli/validation/17/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.5422754883766174,
 "Contradiction": 0.006237328052520752,
 "NotMentioned": 0.45148715376853943
 }
 },
 {
 "id": "contractnli/validation/17/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00013636233052238822,
 "Contradiction": 1.7000444131554104e-05,
 "NotMentioned": 0.9998465776443481
 }
 },
 {
 "id": "contractnli/validation/17/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9991924166679382,
 "Contradiction": 0.00017688395746517926,
 "NotMentioned": 0.0006307169096544385
 }
 },
 {
 "id": "contractnli/validation/72/nda-1",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0002006098657147959,
 "Contradiction": 0.0002033748896792531,
 "NotMentioned": 0.9995959401130676
 }
 },
 {
 "id": "contractnli/validation/72/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0009284525876864791,
 "Contradiction": 7.187485607573763e-05,
 "NotMentioned": 0.9989996552467346
 }
 },
 {
 "id": "contractnli/validation/72/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9984205961227417,
 "Contradiction": 0.00028806121554225683,
 "NotMentioned": 0.0012914330000057817
 }
 },
 {
 "id": "contractnli/validation/69/nda-18",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.03447473794221878,
 "Contradiction": 0.001505821361206472,
 "NotMentioned": 0.9640194177627563
 }
 },
 {
 "id": "contractnli/validation/69/nda-4",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9990656971931458,
 "Contradiction": 0.00035423223744146526,
 "NotMentioned": 0.0005801388761028647
 }
 },
 {
 "id": "contractnli/validation/69/nda-7",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992258548736572,
 "Contradiction": 0.00030250041163526475,
 "NotMentioned": 0.00047154660569503903
 }
 },
 {
 "id": "contractnli/validation/69/nda-8",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9993544220924377,
 "Contradiction": 0.00013834888522978872,
 "NotMentioned": 0.0005072487983852625
 }
 },
 {
 "id": "contractnli/validation/610/nda-3",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.999406099319458,
 "Contradiction": 0.00019354932010173798,
 "NotMentioned": 0.00040028808871284127
 }
 },
 {
 "id": "contractnli/validation/610/nda-11",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 4.5787230192217976e-05,
 "Contradiction": 6.459075393649982e-06,
 "NotMentioned": 0.9999477863311768
 }
 },
 {
 "id": "contractnli/validation/610/nda-7",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9970079064369202,
 "Contradiction": 0.0014086059527471662,
 "NotMentioned": 0.0015835216036066413
 }
 },
 {
 "id": "contractnli/validation/9/nda-12",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.00014329668192658573,
 "Contradiction": 0.0004104017571080476,
 "NotMentioned": 0.9994462132453918
 }
 },
 {
 "id": "contractnli/validation/9/nda-15",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.03254914656281471,
 "Contradiction": 0.011157634668052197,
 "NotMentioned": 0.9562932252883911
 }
 },
 {
 "id": "contractnli/validation/9/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0007312813540920615,
 "Contradiction": 7.153044134611264e-05,
 "NotMentioned": 0.9991971850395203
 }
 },
 {
 "id": "contractnli/validation/9/nda-13",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "NotMentioned",
 "probabilities": {
 "Entailment": 0.0002441699034534395,
 "Contradiction": 0.0007873765425756574,
 "NotMentioned": 0.9989684820175171
 }
 },
 {
 "id": "contractnli/validation/37/nda-15",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9989936947822571,
 "Contradiction": 0.00021348152949940413,
 "NotMentioned": 0.0007928292034193873
 }
 },
 {
 "id": "contractnli/validation/37/nda-10",
 "source": "contractnli",
 "expected": "NotMentioned",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.7760099768638611,
 "Contradiction": 0.0016893204301595688,
 "NotMentioned": 0.222300723195076
 }
 },
 {
 "id": "contractnli/validation/37/nda-5",
 "source": "contractnli",
 "expected": "Entailment",
 "choice": "Entailment",
 "probabilities": {
 "Entailment": 0.9992838501930237,
 "Contradiction": 0.0001308802020503208,
 "NotMentioned": 0.0005852437461726367
 }
 }
 ]
}