{"panel_index": 0, "task_family": "browser", "candidate_logprobs": [-7.780969619750977, -7.689064979553223, -8.539347648620605, -8.411389350891113, -0.007969701662659645, -8.44794750213623, -8.399170875549316, -8.385729789733887, -7.850882530212402, -7.894782066345215, -8.608659744262695, -9.229808807373047, -8.68393325805664, -8.304038047790527, -9.15233325958252, -8.64741325378418], "probabilities": [0.00041938362997334894, 0.0004597536147551036, 0.0001964500217701977, 0.00022326655120656548, 0.9962823733647312, 0.00021525173492177767, 0.00022601126200104703, 0.0002290696064600724, 0.00039106476092346556, 0.00037426856917310614, 0.00018329483217400478, 9.848920005647895e-05, 0.00017000408180615215, 0.00024856830019522334, 0.00010642307735889144, 0.00017632739249339387], "choice": "candidate-c59cf6b3cdd1ee8dfe92", "correct": true, "reference_choice": "candidate-c59cf6b3cdd1ee8dfe92", "agrees_with_reference": true, "max_abs_probability_delta": 1.6239582377788387e-05, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9957638504227803}
{"panel_index": 1, "task_family": "browser", "candidate_logprobs": [-7.224492073059082, -7.184436798095703, -7.804478645324707, -7.79311466217041, -7.805455207824707, -7.8535566329956055, -8.090960502624512, -7.2595720291137695, -0.03727426752448082, -4.085118293762207, -7.543098449707031, -7.296975135803223, -7.468696594238281, -7.014921188354492, -7.79420280456543, -7.553362846374512], "probabilities": [0.0007374017339621936, 0.0007675380940835262, 0.00041287557029419375, 0.0004175942419919727, 0.00041247256830542083, 0.00039310166866860576, 0.00031002855547917514, 0.0007119821787096965, 0.9751539576398495, 0.0170261660060829, 0.0005362103137997507, 0.0006858437104981621, 0.0005776269910084553, 0.0009093260511125726, 0.0004171400871307363, 0.0005307345890230837], "choice": "candidate-d2449a13158601200ec0", "correct": true, "reference_choice": "candidate-d2449a13158601200ec0", "agrees_with_reference": true, "max_abs_probability_delta": 0.001846755708539205, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9879587310034368}
{"panel_index": 2, "task_family": "browser", "candidate_logprobs": [-7.353938102722168, -6.8105621337890625, -7.594781875610352, -7.302000045776367, -6.415900230407715, -0.019044900313019753, -7.433900833129883, -7.093676567077637, -7.129487037658691, -8.554797172546387, -7.993256568908691, -8.09126091003418, -7.358086585998535, -7.541778564453125, -8.868289947509766, -7.6872053146362305], "probabilities": [0.0006461950834412822, 0.0011126250518472154, 0.000507886334753214, 0.0006806440633505171, 0.0016510047438900277, 0.9905292223932408, 0.0005965354766990285, 0.0008382890593725833, 0.0008088006812543117, 0.00019446308974878292, 0.00034096607269084504, 0.0003091351740249382, 0.0006435199067438796, 0.0005355321809431172, 0.00014213106148857593, 0.0004630496265108049], "choice": "candidate-9409b72a36db8d4a0a79", "correct": true, "reference_choice": "candidate-9409b72a36db8d4a0a79", "agrees_with_reference": true, "max_abs_probability_delta": 0.0015038419721461471, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.990516267254389}
{"panel_index": 3, "task_family": "browser", "candidate_logprobs": [-2.9247214794158936, -6.102496147155762, -6.877346515655518, -6.70238733291626, -6.5498456954956055, -6.6782450675964355, -0.1457812637090683, -5.932514667510986, -6.274996757507324, -6.512213706970215, -3.135615110397339, -7.580994129180908, -6.987384796142578, -5.765324115753174, -7.762167453765869, -6.960508823394775], "probabilities": [0.05473907889864019, 0.002281431755659342, 0.0010512215200088753, 0.0012522125456029486, 0.0014585658530869982, 0.0012828116730341397, 0.8814057643108364, 0.002704142045804323, 0.0019199566324846276, 0.0015145004506985455, 0.04433100173145289, 0.0005201204949789031, 0.0009416840725673192, 0.003196240387829215, 0.00043393171424014324, 0.0009673359130751551], "choice": "candidate-3b25ae8d064a298ef021", "correct": true, "reference_choice": "candidate-3b25ae8d064a298ef021", "agrees_with_reference": true, "max_abs_probability_delta": 0.018527512756290485, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9806456702558632}
{"panel_index": 4, "task_family": "browser", "candidate_logprobs": [-0.010516254231333733, -6.615656852722168, -7.517752647399902, -6.879574775695801, -8.007676124572754, -8.184051513671875, -8.761321067810059, -8.505928039550781, -8.228714942932129, -9.44497013092041, -8.526959419250488, -8.845919609069824, -8.131993293762207, -8.071942329406738, -9.103547096252441, -8.253584861755371], "probabilities": [0.9944247872690448, 0.0013458474220934698, 0.000546035150871857, 0.001033660044844129, 0.00033454114463128557, 0.00028044692357363457, 0.00015745109972937333, 0.0002032644743503146, 0.0002681968047322562, 7.947685673583064e-05, 0.00019903418237620206, 0.00014467884022156162, 0.00029543318594768076, 0.0003137177419166814, 0.00011181982870394766, 0.00026160903022693716], "choice": "candidate-34e89f28b090ce5327b0", "correct": true, "reference_choice": "candidate-34e89f28b090ce5327b0", "agrees_with_reference": true, "max_abs_probability_delta": 5.856280603973177e-05, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9950866681042458}
{"panel_index": 5, "task_family": "browser", "candidate_logprobs": [-0.027373963966965675, -6.952297687530518, -7.499499797821045, -7.949809551239014, -5.2420525550842285, -7.936416149139404, -8.34442138671875, -4.424083232879639, -7.749864101409912, -9.029556274414062, -8.33309555053711, -8.710063934326172, -8.168766975402832, -7.6079020500183105, -8.420553207397461, -8.327203750610352], "probabilities": [0.9779579002389689, 0.0009613111921587023, 0.0005561822717668728, 0.00035452764075330224, 0.0053163556009914546, 0.0003593079125946294, 0.0002389309214462578, 0.012046297525222148, 0.0004329974175327247, 0.00012042649789600227, 0.0002416523963193072, 0.00016575850350387177, 0.00028481187934867633, 0.0004990437739489545, 0.0002214158611316446, 0.00024308036641650462], "choice": "candidate-fb3025ec54721504f9b5", "correct": true, "reference_choice": "candidate-fb3025ec54721504f9b5", "agrees_with_reference": true, "max_abs_probability_delta": 0.0004786862779210463, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9949276009823605}
{"panel_index": 6, "task_family": "browser", "candidate_logprobs": [-6.204646587371826, -6.429118633270264, -7.513152599334717, -7.54860258102417, -5.861850261688232, -6.526648998260498, -4.81212854385376, -5.3145432472229, -6.075936794281006, -3.6931374073028564, -0.10380303114652634, -7.252926349639893, -6.648753643035889, -6.471350193023682, -8.136758804321289, -5.514804363250732], "probabilities": [0.0021074154084523033, 0.0016836940618869393, 0.0005694730951104364, 0.0005496389219461516, 0.00296909917116134, 0.0015272364507772401, 0.008482288767953865, 0.005132360100160685, 0.002396890002728678, 0.025970764208215154, 0.9404004808355978, 0.0007387339100110287, 0.001351689398088641, 0.0016140695588471984, 0.0003052421309981859, 0.0042009239780642854], "choice": "candidate-888bb5997d1aba7b25f7", "correct": true, "reference_choice": "candidate-888bb5997d1aba7b25f7", "agrees_with_reference": true, "max_abs_probability_delta": 0.008967637627284808, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9585308031927607}
{"panel_index": 7, "task_family": "browser", "candidate_logprobs": [-0.009682350791990757, -7.198635578155518, -8.023320198059082, -8.269811630249023, -7.921165943145752, -8.561928749084473, -8.642773628234863, -8.454522132873535, -8.207071304321289, -9.54409122467041, -8.655503273010254, -8.95446491241455, -7.920139789581299, -8.186528205871582, -9.401754379272461, -8.512316703796387], "probabilities": [0.9961296162576068, 0.0007519572226372706, 0.00032963791264115685, 0.0002576245739496198, 0.00036509188898802637, 0.0001923633952229272, 0.00017742382954011042, 0.0002141748811270931, 0.00027430584633660304, 7.204016358273401e-05, 0.00017517960163260097, 0.00012991106530200225, 0.0003654667216162919, 0.0002799992178646775, 8.305978389655263e-05, 0.00020214763805546262], "choice": "candidate-41c5af16d7bf373cc74c", "correct": true, "reference_choice": "candidate-41c5af16d7bf373cc74c", "agrees_with_reference": true, "max_abs_probability_delta": 5.082702344538781e-05, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9942123555861029}
{"panel_index": 8, "task_family": "browser", "candidate_logprobs": [-7.93802547454834, -7.716438293457031, -0.010614381171762943, -7.524219512939453, -7.886161804199219, -8.397835731506348, -6.990211486816406, -7.805412292480469, -8.155538558959961, -9.341255187988281, -8.844905853271484, -9.007189750671387, -8.297470092773438, -8.690597534179688, -9.37160873413086, -8.607495307922363], "probabilities": [0.00035905036252982856, 0.0004481149773416149, 0.9953739330226875, 0.0005430865100065312, 0.00037816338383306856, 0.00022670563351423546, 0.0009263727134388143, 0.00040996666286840697, 0.0002888621485957379, 8.825522940817399e-05, 0.00014497804041306767, 0.00012326030107406518, 0.00025064010010025946, 0.00016916771252055432, 8.561661841483036e-05, 0.00018382658325317392], "choice": "candidate-a1a3ad1a8495aa678cbe", "correct": true, "reference_choice": "candidate-a1a3ad1a8495aa678cbe", "agrees_with_reference": true, "max_abs_probability_delta": 0.00026896744423143826, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9940402493598238}
{"panel_index": 9, "task_family": "browser", "candidate_logprobs": [-0.03160245716571808, -4.662675380706787, -6.726959705352783, -7.209039211273193, -7.088446140289307, -6.459153652191162, -7.77041482925415, -7.141021251678467, -7.233113765716553, -8.202003479003906, -7.582809925079346, -7.856788158416748, -7.3243327140808105, -7.197627544403076, -8.502228736877441, -7.57142972946167], "probabilities": [0.9807549621228464, 0.009556769208598773, 0.0012128408241443953, 0.0007489267408616549, 0.0008449134874957591, 0.0015852964896787773, 0.0004272056663513773, 0.0008016395990034208, 0.0007311119650812645, 0.00027746003415166873, 0.0005153624583148395, 0.0003918551367277927, 0.0006673720367562241, 0.0007575221942848307, 0.00020550115284375954, 0.0005212608828589976], "choice": "candidate-976c2731e7518a1f1b2b", "correct": true, "reference_choice": "candidate-976c2731e7518a1f1b2b", "agrees_with_reference": true, "max_abs_probability_delta": 0.0013190414075143764, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9879039300248557}
{"panel_index": 10, "task_family": "browser", "candidate_logprobs": [-4.986140251159668, -4.521906852722168, -6.569631576538086, -5.962360382080078, -3.6671409606933594, -5.224387168884277, -5.647228240966797, -4.305950164794922, -5.358579635620117, -4.945310592651367, -4.953245162963867, -0.21049705147743225, -5.177520751953125, -4.01359748840332, -6.846708297729492, -5.672080993652344], "probabilities": [0.007370748908611251, 0.011725344731397401, 0.0015128994745911913, 0.002776800001042077, 0.027564221445447908, 0.005808209798123358, 0.0038054388778006984, 0.014551722961807444, 0.005078824983629304, 0.007677922289749209, 0.007617242328550708, 0.8740717940845377, 0.006086899361757622, 0.01949312968450406, 0.0011467722652803959, 0.0037120288031697304], "choice": "candidate-016db9380de74cbcab5d", "correct": true, "reference_choice": "candidate-016db9380de74cbcab5d", "agrees_with_reference": true, "max_abs_probability_delta": 0.004078643850315311, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9269049172747235}
{"panel_index": 11, "task_family": "browser", "candidate_logprobs": [-5.058596134185791, -6.181138515472412, -6.128952503204346, -6.525069713592529, -7.010655879974365, -7.177795886993408, -6.9336771965026855, -6.602798938751221, -6.101531505584717, -3.0732998847961426, -6.192391872406006, -6.638314723968506, -4.936987400054932, -0.11233384162187576, -7.47758150100708, -6.640085697174072], "probabilities": [0.006546141513654055, 0.002130450400281017, 0.0022445822532902097, 0.0015104418361967503, 0.0009294298441898196, 0.0007863730674619892, 0.0010038019409794583, 0.0013974833189425895, 0.0023069825674391525, 0.047663792547853336, 0.0021066100749330937, 0.0013487216317866393, 0.007392637173336773, 0.9207035322313492, 0.0005826844026100626, 0.0013463351956958883], "choice": "candidate-2dc16f49fe3acb051b39", "correct": true, "reference_choice": "candidate-2dc16f49fe3acb051b39", "agrees_with_reference": true, "max_abs_probability_delta": 0.0036070327634628585, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9707205488443852}
{"panel_index": 12, "task_family": "browser", "candidate_logprobs": [-6.807129859924316, -7.080336570739746, -6.543583869934082, -7.029326438903809, -7.722956657409668, -7.490495681762695, -7.872960090637207, -7.6326446533203125, -6.254118919372559, -7.7951154708862305, -7.728812217712402, -7.795975685119629, -6.669875144958496, -2.4154157638549805, -8.34870719909668, -0.118132583796978], "probabilities": [0.0011184587484613397, 0.0008510749421349323, 0.0014557156132586654, 0.0008956147196837012, 0.00044759110412865, 0.000564726236914586, 0.00038524391086815316, 0.0004898955013102163, 0.0019444177223246861, 0.0004164312109417668, 0.00044497786588132697, 0.0004160731449157092, 0.0012830067350940153, 0.09034771409926234, 0.00023939850778339637, 0.8986996599370364], "choice": "candidate-bb1945d8d438379625ee", "correct": true, "reference_choice": "candidate-bb1945d8d438379625ee", "agrees_with_reference": true, "max_abs_probability_delta": 0.003970564034606827, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9887376988553295}
{"panel_index": 13, "task_family": "browser", "candidate_logprobs": [-5.035866737365723, -5.250459671020508, -6.25041389465332, -6.184078216552734, -6.331635475158691, -6.189233779907227, -5.649030685424805, -5.138763427734375, -5.661108016967773, -4.717384338378906, -0.549705445766449, -6.201421737670898, -4.589093208312988, -4.229206085205078, -1.9921932220458984, -5.245304107666016], "probabilities": [0.008260855422744596, 0.006665434951835827, 0.002452188734968697, 0.0026203729764355805, 0.0022608919565307024, 0.002606898242312232, 0.0044743638991740844, 0.007453110439209117, 0.004420650532879434, 0.011359001839974747, 0.733398665366033, 0.002575318314973549, 0.012913866716517808, 0.01850773580688948, 0.17333075704000192, 0.006699887759519182], "choice": "candidate-94dddd06eb43ed2921b0", "correct": false, "reference_choice": "candidate-94dddd06eb43ed2921b0", "agrees_with_reference": true, "max_abs_probability_delta": 0.011658181128362144, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.7869114107711379}
{"panel_index": 14, "task_family": "browser", "candidate_logprobs": [-6.9889702796936035, -0.03018982522189617, -7.35592794418335, -7.70407247543335, -7.861998081207275, -8.113790512084961, -7.923895359039307, -7.593282699584961, -7.485607624053955, -8.136871337890625, -7.873423099517822, -4.383961200714111, -7.020341396331787, -7.9429402351379395, -8.512199401855469, -7.819299221038818], "probabilities": [0.0009318872173220726, 0.9806709413062774, 0.0006456477855041671, 0.00045582529001468987, 0.00038923528027511495, 0.0003025938760917341, 0.0003658731602198809, 0.0005092297938232005, 0.0005671220019365645, 0.0002956897425964507, 0.00038481356724449364, 0.012609690376936168, 0.0009031066739437513, 0.00035897108442304687, 0.00020315773027080252, 0.0004062151131205583], "choice": "candidate-b56ad12ceea4c52feb3f", "correct": true, "reference_choice": "candidate-b56ad12ceea4c52feb3f", "agrees_with_reference": true, "max_abs_probability_delta": 0.003514813946560724, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9893852210004069}
{"panel_index": 15, "task_family": "browser", "candidate_logprobs": [-3.8454301357269287, -4.551257610321045, -5.471189022064209, -0.4078918397426605, -1.9670559167861938, -4.351660251617432, -5.706044673919678, -2.970144033432007, -5.57391881942749, -5.926000118255615, -4.571796894073486, -6.164609432220459, -3.74930739402771, -5.2576470375061035, -7.333190441131592, -6.183730602264404], "probabilities": [0.022292821357563317, 0.011005963253147523, 0.004386386766160539, 0.6935359038777189, 0.14585878583854975, 0.01343730237138308, 0.0034682501118610343, 0.053493000668621005, 0.003958147175200953, 0.002783459926946554, 0.01078221433802904, 0.0021925942195959182, 0.024542037317837294, 0.005430590157405402, 0.0006814750826791699, 0.0021510675373005443], "choice": "candidate-da095abe0a63c03f3114", "correct": false, "reference_choice": "candidate-da095abe0a63c03f3114", "agrees_with_reference": true, "max_abs_probability_delta": 0.01999519247482584, "all_candidates_in_top20": false, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9589277257030578}
{"panel_index": 16, "task_family": "hs3", "candidate_logprobs": [-3.4314515590667725, -0.17397478222846985], "probabilities": [0.03705914734574311, 0.9629408526542568], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.005028586399831725, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8726580516597102}
{"panel_index": 17, "task_family": "hs3", "candidate_logprobs": [-1.2162247896194458, -0.6244661211967468], "probabilities": [0.3562314359242254, 0.6437685640757747], "choice": "no", "correct": false, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.035410150676422614, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8318940989518127}
{"panel_index": 18, "task_family": "hs3", "candidate_logprobs": [-0.09869252890348434, -3.31358003616333], "probabilities": [0.9613906927742436, 0.03860930722575636], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.0012824237011000647, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9424069188051267}
{"panel_index": 19, "task_family": "hs3", "candidate_logprobs": [-0.03940027579665184, -6.09519100189209], "probabilities": [0.9976612349149839, 0.0023387650850159817], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 1.5674054895220557e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9636195002935233}
{"panel_index": 20, "task_family": "hs3", "candidate_logprobs": [-0.03710593655705452, -6.127596378326416], "probabilities": [0.9977408173270491, 0.0022591826729508294], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 6.397983617762197e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9657558711548354}
{"panel_index": 21, "task_family": "hs3", "candidate_logprobs": [-6.817233562469482, -0.03300144150853157], "probabilities": [0.0011301974155038096, 0.9988698025844961], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 3.14663484721877e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9686319101563823}
{"panel_index": 22, "task_family": "hs3", "candidate_logprobs": [-0.024646401405334473, -6.675843715667725], "probabilities": [0.9987091946504391, 0.0012908053495609025], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.00012130261442300228, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9769158494267454}
{"panel_index": 23, "task_family": "hs3", "candidate_logprobs": [-0.11625587195158005, -4.08884334564209], "probabilities": [0.9815231585425918, 0.018476841457408247], "choice": "yes", "correct": false, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.0074805641097609454, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9070060110793057}
{"panel_index": 24, "task_family": "hs3", "candidate_logprobs": [-0.0361950621008873, -6.190542697906494], "probabilities": [0.9978802764698935, 0.0021197235301065277], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 6.299856156832817e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.966500861689982}
{"panel_index": 25, "task_family": "hs3", "candidate_logprobs": [-5.907456874847412, -0.04164664074778557], "probabilities": [0.0028267123705053195, 0.9971732876294946], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 2.5785338720069745e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9619277590477674}
{"panel_index": 26, "task_family": "hs3", "candidate_logprobs": [-0.1615656167268753, -3.2343852519989014], "probabilities": [0.9557575541692597, 0.04424244583074041], "choice": "yes", "correct": false, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.00916088699873499, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8901951112066053}
{"panel_index": 27, "task_family": "hs3", "candidate_logprobs": [-0.025619886815547943, -6.3911309242248535], "probabilities": [0.9982830905271143, 0.001716909472885646], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 1.6210143509498565e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9763818769138365}
{"panel_index": 28, "task_family": "hs3", "candidate_logprobs": [-2.966789722442627, -0.2220349758863449], "probabilities": [0.06038356683239759, 0.9396164331676025], "choice": "no", "correct": false, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.0037311401127320606, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8523556249435313}
{"panel_index": 29, "task_family": "hs3", "candidate_logprobs": [-6.538778781890869, -0.04088180884718895], "probabilities": [0.0015043378512415444, 0.9984956621487585], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 9.350772639332662e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9613888335755396}
{"panel_index": 30, "task_family": "hs3", "candidate_logprobs": [-0.10783321410417557, -4.846033573150635], "probabilities": [0.9913215874098881, 0.008678412590111975], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.000447225231741186, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9056368104365716}
{"panel_index": 31, "task_family": "hs3", "candidate_logprobs": [-0.030743354931473732, -6.252763271331787], "probabilities": [0.9980187021705776, 0.001981297829422383], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 5.4563135142652805e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.971649543262744}
{"panel_index": 32, "task_family": "boolq", "candidate_logprobs": [-5.400937557220459, -0.07019393891096115], "probabilities": [0.004817151936433588, 0.9951828480635664], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.00020757923527744884, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9367253583715197}
{"panel_index": 33, "task_family": "boolq", "candidate_logprobs": [-6.578747749328613, -0.03087821789085865], "probabilities": [0.0014311146381512196, 0.9985688853618487], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 7.008794516172312e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9709832333641237}
{"panel_index": 34, "task_family": "boolq", "candidate_logprobs": [-5.902101516723633, -0.050222188234329224], "probabilities": [0.0028662529430406223, 0.9971337470569595], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.00011478000031243463, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.953751789867303}
{"panel_index": 35, "task_family": "boolq", "candidate_logprobs": [-0.03692025691270828, -5.856685638427734], "probabilities": [0.9970404832564468, 0.0029595167435531753], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.00048693726433568507, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9666136943362419}
{"panel_index": 36, "task_family": "boolq", "candidate_logprobs": [-0.009429908357560635, -7.789159774780273], "probabilities": [0.999582049646549, 0.0004179503534509989], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 1.2606737570911432e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9910286145606162}
{"panel_index": 37, "task_family": "boolq", "candidate_logprobs": [-0.009667725302278996, -7.670950412750244], "probabilities": [0.9995295181990685, 0.0004704818009314384], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 1.2241291898251738e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9908450314732672}
{"panel_index": 38, "task_family": "boolq", "candidate_logprobs": [-0.014053767547011375, -7.317083835601807], "probabilities": [0.9993269583453842, 0.0006730416546157473], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 4.6775651960762055e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9867086216454822}
{"panel_index": 39, "task_family": "boolq", "candidate_logprobs": [-5.932369232177734, -0.04167155921459198], "probabilities": [0.0027574225226686113, 0.9972425774773312], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 4.3518944616405264e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9618369551234223}
{"panel_index": 40, "task_family": "boolq", "candidate_logprobs": [-0.011410899460315704, -7.685575485229492], "probabilities": [0.9995355372313061, 0.0004644627686940163], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 3.393841948129417e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.989113364256541}
{"panel_index": 41, "task_family": "boolq", "candidate_logprobs": [-5.6178364753723145, -0.08767566084861755], "probabilities": [0.003949689445343818, 0.9960503105546562], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.00012044864774027851, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9196904335998702}
{"panel_index": 42, "task_family": "boolq", "candidate_logprobs": [-0.016608063131570816, -7.371506690979004], "probabilities": [0.999360956400907, 0.0006390435990929399], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 7.062682210879582e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9841580102912875}
{"panel_index": 43, "task_family": "boolq", "candidate_logprobs": [-3.7055561542510986, -0.14655901491641998], "probabilities": [0.027679399782514456, 0.9723206002174857], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 9.419136652921711e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8882613007928867}
{"panel_index": 44, "task_family": "boolq", "candidate_logprobs": [-0.03505300357937813, -6.446855068206787], "probabilities": [0.9983606292186054, 0.001639370781394563], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 6.135183104602308e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9671397376980978}
{"panel_index": 45, "task_family": "boolq", "candidate_logprobs": [-0.0345207154750824, -6.7564311027526855], "probabilities": [0.998797212979798, 0.0012027870202020686], "choice": "yes", "correct": true, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 3.3284285064105745e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9672317006499208}
{"panel_index": 46, "task_family": "boolq", "candidate_logprobs": [-4.7614617347717285, -0.08886975795030594], "probabilities": [0.009261432331413416, 0.9907385676685866], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.0011994403268996753, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9235178305915813}
{"panel_index": 47, "task_family": "boolq", "candidate_logprobs": [-5.95490837097168, -0.04559817537665367], "probabilities": [0.002706712236155755, 0.9972932877638442], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 7.504796003277026e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.958018880105012}
{"panel_index": 48, "task_family": "mnli", "candidate_logprobs": [-3.2034912109375, -0.12096980214118958, -5.233493804931641], "probabilities": [0.04358310927366833, 0.9506928952046328, 0.00572399552169885], "choice": "neutral", "correct": false, "reference_choice": "neutral", "agrees_with_reference": true, "max_abs_probability_delta": 0.0065010158709591925, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9320157127445068}
{"panel_index": 49, "task_family": "mnli", "candidate_logprobs": [-7.2607645988464355, -0.011258742772042751, -8.073200225830078], "probabilities": [0.000709797265893621, 0.9989752118304012, 0.0003149909037051669], "choice": "neutral", "correct": true, "reference_choice": "neutral", "agrees_with_reference": true, "max_abs_probability_delta": 0.00010282352421309149, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9898187542313649}
{"panel_index": 50, "task_family": "mnli", "candidate_logprobs": [-0.005124623887240887, -7.894443988800049, -8.858028411865234], "probabilities": [0.9994825773359101, 0.0003745306479075362, 0.00014289201618240919], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 4.530478723452358e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9954035289419666}
{"panel_index": 51, "task_family": "mnli", "candidate_logprobs": [-6.848641395568848, -0.01266569085419178, -7.842465400695801], "probabilities": [0.001072839136647355, 0.9985300403451104, 0.00039712051824230236], "choice": "neutral", "correct": true, "reference_choice": "neutral", "agrees_with_reference": true, "max_abs_probability_delta": 0.00012905415049979663, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9888677771764713}
{"panel_index": 52, "task_family": "mnli", "candidate_logprobs": [-8.607659339904785, -7.863791465759277, -0.006951416842639446], "probabilities": [0.00018387050400410447, 0.0003868742099858162, 0.99942925528601], "choice": "contradiction", "correct": true, "reference_choice": "contradiction", "agrees_with_reference": true, "max_abs_probability_delta": 3.698424413989265e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9936398030331484}
{"panel_index": 53, "task_family": "mnli", "candidate_logprobs": [-0.005445857997983694, -8.073294639587402, -8.910027503967285], "probabilities": [0.999550979369595, 0.0003133161218760804, 0.00013570450852878535], "choice": "entailment", "correct": false, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 1.7400335466329834e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9950157263937163}
{"panel_index": 54, "task_family": "mnli", "candidate_logprobs": [-2.7079408168792725, -0.11994151771068573, -6.175140380859375], "probabilities": [0.06976257305313882, 0.928060535103955, 0.0021768918429060802], "choice": "neutral", "correct": false, "reference_choice": "neutral", "agrees_with_reference": true, "max_abs_probability_delta": 0.009924229472003221, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9557267805511482}
{"panel_index": 55, "task_family": "mnli", "candidate_logprobs": [-0.006215416826307774, -7.670483112335205, -8.731745719909668], "probabilities": [0.99936871182064, 0.00046900403645810967, 0.0001622841429017645], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 1.4737733803338315e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9944316318549495}
{"panel_index": 56, "task_family": "mnli", "candidate_logprobs": [-0.007270055823028088, -7.740359306335449, -8.44603157043457], "probabilities": [0.9993460214179786, 0.00043780215829093455, 0.00021617642373041615], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 3.4574274423881945e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9934059733372765}
{"panel_index": 57, "task_family": "mnli", "candidate_logprobs": [-5.266855716705322, -2.013657331466675, -0.2122313380241394], "probabilities": [0.005446071317256347, 0.1409059821557189, 0.8536479465270248], "choice": "contradiction", "correct": false, "reference_choice": "contradiction", "agrees_with_reference": true, "max_abs_probability_delta": 0.06603396017310947, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9474369126140557}
{"panel_index": 58, "task_family": "mnli", "candidate_logprobs": [-5.874185562133789, -0.04763038456439972, -4.466728210449219], "probabilities": [0.0029046650438291437, 0.98522816889618, 0.011867166059990754], "choice": "neutral", "correct": true, "reference_choice": "neutral", "agrees_with_reference": true, "max_abs_probability_delta": 0.001045060225981831, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9677820582618318}
{"panel_index": 59, "task_family": "mnli", "candidate_logprobs": [-0.27892330288887024, -4.814178466796875, -1.6473602056503296], "probabilities": [0.7903709833503444, 0.00847607314902183, 0.20115294350063384], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 0.0964994008659642, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9572693656567529}
{"panel_index": 60, "task_family": "mnli", "candidate_logprobs": [-5.715946197509766, -4.8776092529296875, -0.03245251253247261], "probabilities": [0.003363750486563963, 0.0077787333176096716, 0.9888575161958264], "choice": "contradiction", "correct": false, "reference_choice": "contradiction", "agrees_with_reference": true, "max_abs_probability_delta": 0.0015083466505481091, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9789766513396659}
{"panel_index": 61, "task_family": "mnli", "candidate_logprobs": [-0.10189643502235413, -3.0190908908843994, -5.614595890045166], "probabilities": [0.9450720237369742, 0.051114421796173495, 0.0038135544668521835], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 0.00898626314355988, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9556129650056607}
{"panel_index": 62, "task_family": "mnli", "candidate_logprobs": [-0.009009532630443573, -7.270882606506348, -8.168879508972168], "probabilities": [0.9990132827116093, 0.0007010998243471923, 0.0002856174640436623], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 4.240090121421236e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9920097647813888}
{"panel_index": 63, "task_family": "mnli", "candidate_logprobs": [-0.006529562175273895, -7.8254241943359375, -8.67029094696045], "probabilities": [0.9994255309696202, 0.0004018348935545603, 0.0001726341368253625], "choice": "entailment", "correct": true, "reference_choice": "entailment", "agrees_with_reference": true, "max_abs_probability_delta": 9.298860550344124e-06, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9940627673673951}
{"panel_index": 64, "task_family": "score", "candidate_logprobs": [-0.79500812292099, -0.8070663809776306], "probabilities": [0.5030145279878191, 0.4969854720121809], "choice": "yes", "correct": false, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 0.0896520979887434, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.8977425932114107}
{"panel_index": 65, "task_family": "score", "candidate_logprobs": [-7.587325096130371, -0.013337108306586742], "probabilities": [0.0005133762781836013, 0.9994866237218164], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 2.5571865341800226e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9872582718170745}
{"panel_index": 66, "task_family": "score", "candidate_logprobs": [-7.610119342803955, -0.013575132004916668], "probabilities": [0.0005019318177928887, 0.999498068182207], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 4.369130875980787e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9870120072946812}
{"panel_index": 67, "task_family": "score", "candidate_logprobs": [-5.736418724060059, -0.03978227078914642], "probabilities": [0.003346005378277338, 0.9966539946217227], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.00017332231438282756, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9642249557417368}
{"panel_index": 68, "task_family": "score", "candidate_logprobs": [-6.998847484588623, -0.02004973404109478], "probabilities": [0.0009305555920631048, 0.9990694444079368], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 1.955820131616104e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9810628588073128}
{"panel_index": 69, "task_family": "score", "candidate_logprobs": [-0.013291684910655022, -7.228087902069092], "probabilities": [0.9992649202543895, 0.000735079745610479], "choice": "yes", "correct": false, "reference_choice": "yes", "agrees_with_reference": true, "max_abs_probability_delta": 2.542684491468883e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9875221670023797}
{"panel_index": 70, "task_family": "score", "candidate_logprobs": [-6.989453315734863, -0.018794091418385506], "probabilities": [0.0009381527109410643, 0.999061847289059], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 3.16148595546073e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9823029664861871}
{"panel_index": 71, "task_family": "score", "candidate_logprobs": [-7.053216457366943, -0.018542861565947533], "probabilities": [0.0008800304808731547, 0.9991199695191268], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 7.594455611737284e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9824926230323592}
{"panel_index": 72, "task_family": "score", "candidate_logprobs": [-1.0027729272842407, -0.6071944236755371], "probabilities": [0.4023751139148115, 0.5976248860851884], "choice": "no", "correct": true, "reference_choice": "yes", "agrees_with_reference": false, "max_abs_probability_delta": 0.12883429532209645, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9117381730984387}
{"panel_index": 73, "task_family": "score", "candidate_logprobs": [-7.561438083648682, -0.014469556510448456], "probabilities": [0.0005274291073024173, 0.9994725708926976], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 8.124279981096554e-06, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9861547511530943}
{"panel_index": 74, "task_family": "score", "candidate_logprobs": [-7.631584644317627, -0.01266508735716343], "probabilities": [0.0004908310865796135, 0.9995091689134203], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 2.8504183348321277e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9878996692097755}
{"panel_index": 75, "task_family": "score", "candidate_logprobs": [-6.678159713745117, -0.021890558302402496], "probabilities": [0.001284283550473958, 0.998715716449526], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 3.9440195093959395e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9796053922730869}
{"panel_index": 76, "task_family": "score", "candidate_logprobs": [-4.14695930480957, -0.08814021199941635], "probabilities": [0.01697623129839656, 0.9830237687016035], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 6.132718081230948e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9314449092827471}
{"panel_index": 77, "task_family": "score", "candidate_logprobs": [-4.080027103424072, -0.08877211064100266], "probabilities": [0.018141323302808852, 0.9818586766971912], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.0034740105523432784, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9319610883118976}
{"panel_index": 78, "task_family": "score", "candidate_logprobs": [-5.818037509918213, -0.035955917090177536], "probabilities": [0.003072821312960629, 0.9969271786870393], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 0.0004411570368376694, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9676562533303529}
{"panel_index": 79, "task_family": "score", "candidate_logprobs": [-6.195009231567383, -0.031961482018232346], "probabilities": [0.002101400270765558, 0.9978985997292346], "choice": "no", "correct": true, "reference_choice": "no", "agrees_with_reference": true, "max_abs_probability_delta": 8.132182090936357e-05, "all_candidates_in_top20": true, "reference_choice_in_top20": true, "candidate_mass_full_vocab": 0.9705834720875778}