evaluation/per-question/mlx-8bit-mac.jsonl

{"panel_index": 0, "task_family": "browser", "logits": [12.8125, 12.875, 12.0625, 12.125, 20.625, 12.125, 12.1875, 12.1875, 12.75, 12.75, 11.9375, 11.375, 11.875, 12.25, 11.4375, 11.9375], "probabilities": [0.00040321299456991255, 0.0004292181693017483, 0.0001904644159367308, 0.00020274825510568917, 0.9964608550071716, 0.00020274825510568917, 0.00021582446061074734, 0.00021582446061074734, 0.0003787837049458176, 0.0003787837049458176, 0.0001680842397036031, 9.577153105055913e-05, 0.00015790048928465694, 0.00022974387684371322, 0.00010194822971243411, 0.0001680842397036031], "choice": "candidate-c59cf6b3cdd1ee8dfe92", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00017052888870239258}
{"panel_index": 1, "task_family": "browser", "logits": [13.375, 13.4375, 12.8125, 12.8125, 12.875, 12.75, 12.5, 13.3125, 20.625, 16.375, 13.0625, 13.3125, 13.125, 13.5625, 12.875, 13.0625], "probabilities": [0.0006949820090085268, 0.0007398043526336551, 0.00039598860894329846, 0.00039598860894329846, 0.0004215278313495219, 0.00037199698272161186, 0.00028971151914447546, 0.0006528749945573509, 0.978607177734375, 0.013959080912172794, 0.0005084595177322626, 0.0006528749945573509, 0.000541252491530031, 0.0008383081876672804, 0.0004215278313495219, 0.0005084595177322626], "choice": "candidate-d2449a13158601200ec0", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0018326649442315102}
{"panel_index": 2, "task_family": "browser", "logits": [12.75, 13.375, 12.5, 12.8125, 13.6875, 20.25, 12.625, 12.9375, 12.9375, 11.5625, 12.0625, 11.9375, 12.6875, 12.5, 11.1875, 12.4375], "probabilities": [0.0005486929439939559, 0.0010250930208712816, 0.00042732246220111847, 0.0005840804078616202, 0.0014011366292834282, 0.9920598268508911, 0.00048421983956359327, 0.0006618499173782766, 0.0006618499173782766, 0.00016734187374822795, 0.00027590017998591065, 0.00024348103033844382, 0.0005154491518624127, 0.00042732246220111847, 0.00011501226254040375, 0.0004014321602880955], "choice": "candidate-9409b72a36db8d4a0a79", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 8.492660708725452e-05}
{"panel_index": 3, "task_family": "browser", "logits": [17.5, 14.5, 13.6875, 13.75, 14.0625, 14.0625, 20.5, 14.6875, 14.4375, 14.0625, 17.625, 13.0, 13.625, 14.9375, 12.9375, 13.75], "probabilities": [0.044130101799964905, 0.002197108929976821, 0.0009749610908329487, 0.0010378403821960092, 0.0014185601612553, 0.0014185601612553, 0.886376678943634, 0.00265021831728518, 0.002063991967588663, 0.0014185601612553, 0.0500059612095356, 0.0004902412183582783, 0.0009158909088000655, 0.0034029479138553143, 0.00046053886762820184, 0.0010378403821960092], "choice": "candidate-3b25ae8d064a298ef021", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.018641650676727295}
{"panel_index": 4, "task_family": "browser", "logits": [20.25, 13.75, 12.8125, 13.5, 12.3125, 12.125, 11.5, 11.8125, 12.125, 10.8125, 11.75, 11.4375, 12.1875, 12.25, 11.25, 12.0625], "probabilities": [0.993965208530426, 0.0014943663263693452, 0.0005852022441104054, 0.0011638135183602571, 0.0003549430111888796, 0.0002942581777460873, 0.00015750502643641084, 0.00021528392971958965, 0.0002942581777460873, 7.919850031612441e-05, 0.0002022404660237953, 0.00014796231698710471, 0.00031323631992563605, 0.00033343824907206, 0.00012266510748304427, 0.0002764300734270364], "choice": "candidate-34e89f28b090ce5327b0", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00041282176971435547}
{"panel_index": 5, "task_family": "browser", "logits": [20.25, 13.3125, 12.75, 12.3125, 15.125, 12.375, 11.9375, 15.5625, 12.5, 11.25, 11.875, 11.5625, 12.0625, 12.6875, 11.875, 11.9375], "probabilities": [0.9804983735084534, 0.0009517634171061218, 0.000542298483196646, 0.0003501340397633612, 0.0058302562683820724, 0.0003727159055415541, 0.00024064349418040365, 0.00903007946908474, 0.00042234244756400585, 0.00012100316962460056, 0.0002260635665152222, 0.0001653916697250679, 0.00027268484700471163, 0.0005094421212561429, 0.0002260635665152222, 0.00024064349418040365], "choice": "candidate-fb3025ec54721504f9b5", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.002537531778216362}
{"panel_index": 6, "task_family": "browser", "logits": [13.8125, 13.5625, 12.5625, 12.4375, 14.375, 13.6875, 15.375, 14.875, 13.875, 16.25, 19.75, 12.75, 13.3125, 13.5625, 11.875, 14.5625], "probabilities": [0.002450336702167988, 0.0019083237275481224, 0.0007020332850515842, 0.0006195421447046101, 0.004300475586205721, 0.0021624143701046705, 0.011689901351928711, 0.007090284489095211, 0.002608370501548052, 0.028042616322636604, 0.9286439418792725, 0.0008468135492876172, 0.001486204913817346, 0.0019083237275481224, 0.0003530045214574784, 0.005187362432479858], "choice": "candidate-888bb5997d1aba7b25f7", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0027889013290405273}
{"panel_index": 7, "task_family": "browser", "logits": [20.125, 12.9375, 12.0625, 11.875, 12.25, 11.5625, 11.5, 11.625, 11.9375, 10.5625, 11.4375, 11.125, 12.1875, 11.9375, 10.6875, 11.625], "probabilities": [0.9961690306663513, 0.0007530807051807642, 0.0003139308246318251, 0.0002602576860226691, 0.0003786727611441165, 0.00019040863844566047, 0.00017887230205815285, 0.0002026888687396422, 0.00027704297099262476, 7.004739745752886e-05, 0.00016803502512630075, 0.00012293708277866244, 0.00035573000786826015, 0.00027704297099262476, 7.937409827718511e-05, 0.0002026888687396422], "choice": "candidate-41c5af16d7bf373cc74c", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 9.02414321899414e-05}
{"panel_index": 8, "task_family": "browser", "logits": [12.5, 12.625, 20.375, 12.875, 12.5625, 12.0, 13.375, 12.625, 12.1875, 11.0625, 11.4375, 11.375, 12.0625, 11.6875, 11.0, 11.8125], "probabilities": [0.0003783677238970995, 0.000428746803663671, 0.9953665733337402, 0.0005505218869075179, 0.0004027702088933438, 0.0002294915757374838, 0.0009076567366719246, 0.000428746803663671, 0.0002768198028206825, 8.987019100459293e-05, 0.0001307603670284152, 0.00012283804244361818, 0.00024429260520264506, 0.00016789966321084648, 8.442525722784922e-05, 0.00019025524670723826], "choice": "candidate-a1a3ad1a8495aa678cbe", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00027632713317871094}
{"panel_index": 9, "task_family": "browser", "logits": [20.0, 15.3125, 13.1875, 12.75, 12.9375, 13.5, 12.1875, 12.8125, 12.8125, 11.75, 12.3125, 12.0625, 12.625, 12.8125, 11.5, 12.4375], "probabilities": [0.9818562865257263, 0.009042585268616676, 0.0010799829615280032, 0.0006972894771024585, 0.0008410915033891797, 0.0014761612983420491, 0.00039730334538035095, 0.0007422605995088816, 0.0007422605995088816, 0.000256518367677927, 0.00045020371908321977, 0.00035061893868260086, 0.0006153557915240526, 0.0007422605995088816, 0.00019977668125648052, 0.000510147656314075], "choice": "candidate-976c2731e7518a1f1b2b", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.002420365810394287}
{"panel_index": 10, "task_family": "browser", "logits": [14.4375, 14.8125, 12.8125, 13.375, 15.8125, 14.4375, 13.75, 15.1875, 14.0, 14.5, 14.25, 19.125, 14.25, 15.1875, 12.5, 13.75], "probabilities": [0.007981746457517147, 0.011613370850682259, 0.0015716985799372196, 0.0027584186755120754, 0.031568415462970734, 0.007981746457517147, 0.004013474564999342, 0.016897358000278473, 0.005153402220457792, 0.008496524766087532, 0.006617099978029728, 0.8666689991950989, 0.006617099978029728, 0.016897358000278473, 0.0011498794192448258, 0.004013474564999342], "choice": "candidate-016db9380de74cbcab5d", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.003324151039123535}
{"panel_index": 11, "task_family": "browser", "logits": [14.625, 13.5, 13.5625, 13.1875, 12.75, 12.5625, 12.8125, 13.125, 13.6875, 16.625, 13.5, 13.125, 14.6875, 19.625, 12.25, 13.125], "probabilities": [0.006222897209227085, 0.002020279411226511, 0.002150575630366802, 0.0014780682977288961, 0.0009543122141622007, 0.0007911527645774186, 0.001015860354527831, 0.001388516160659492, 0.002436921698972583, 0.04598134383559227, 0.002020279411226511, 0.001388516160659492, 0.0066242399625480175, 0.923559844493866, 0.0005788198322989047, 0.001388516160659492], "choice": "candidate-2dc16f49fe3acb051b39", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0007507205009460449}
{"panel_index": 12, "task_family": "browser", "logits": [13.6875, 13.3125, 13.8125, 13.4375, 12.75, 13.0, 12.5, 12.8125, 14.3125, 12.875, 12.6875, 12.6875, 13.75, 18.125, 12.0625, 20.25], "probabilities": [0.001246684929355979, 0.000856833066791296, 0.001412679092027247, 0.0009709190926514566, 0.00048820878146216273, 0.0006268725846894085, 0.0003802173596341163, 0.0005196954007260501, 0.002329113194718957, 0.0005532131181098521, 0.00045862956903874874, 0.00045862956903874874, 0.001327088801190257, 0.1054237112402916, 0.0002454868226777762, 0.8827019333839417], "choice": "candidate-bb1945d8d438379625ee", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.01216500997543335}
{"panel_index": 13, "task_family": "browser", "logits": [13.5, 13.3125, 12.3125, 12.3125, 12.25, 12.375, 13.0, 13.4375, 12.875, 14.0625, 18.0, 12.375, 13.875, 14.25, 16.625, 13.375], "probabilities": [0.008035835810005665, 0.006661941763013601, 0.0024507911875844, 0.0024507911875844, 0.002302306005731225, 0.0026088545564562082, 0.004873979836702347, 0.007548969238996506, 0.004301271867007017, 0.014103330671787262, 0.7233627438545227, 0.0026088545564562082, 0.011692069470882416, 0.017011864110827446, 0.182894766330719, 0.007091599982231855], "choice": "candidate-94dddd06eb43ed2921b0", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.0016222596168518066}
{"panel_index": 14, "task_family": "browser", "logits": [13.0, 20.0, 12.625, 12.25, 12.1875, 11.875, 12.0625, 12.3125, 12.5, 11.75, 12.0625, 15.5, 12.875, 12.0, 11.5, 12.1875], "probabilities": [0.0008961857529357076, 0.9827871322631836, 0.0006159392069093883, 0.00042332831071689725, 0.000397680007154122, 0.00029094895580783486, 0.00035095136263407767, 0.00045063052675686777, 0.0005435643834061921, 0.0002567615592852235, 0.00035095136263407767, 0.010917781852185726, 0.0007908811094239354, 0.0003296883951406926, 0.00019996608898509294, 0.000397680007154122], "choice": "candidate-b56ad12ceea4c52feb3f", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.001398622989654541}
{"panel_index": 15, "task_family": "browser", "logits": [16.625, 15.875, 14.75, 19.375, 18.25, 15.4375, 14.3125, 17.25, 14.5, 14.25, 15.4375, 14.0, 16.5, 14.875, 12.75, 13.9375], "probabilities": [0.037944961339235306, 0.017923928797245026, 0.00581904873251915, 0.5935590267181396, 0.19270041584968567, 0.011572560295462608, 0.0037570595741271973, 0.07089051604270935, 0.00453187944367528, 0.0035294308327138424, 0.011572560295462608, 0.0027487240731716156, 0.03348631411790848, 0.006593846715986729, 0.0007875224691815674, 0.002582187531515956], "choice": "candidate-da095abe0a63c03f3114", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.07998168468475342}
{"panel_index": 16, "task_family": "hs3", "logits": [15.5, 18.625], "probabilities": [0.04208773374557495, 0.957912266254425], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 7.450580596923828e-09}
{"panel_index": 17, "task_family": "hs3", "logits": [17.25, 18.5], "probabilities": [0.22270014882087708, 0.7772998809814453], "choice": "no", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.09812116622924805}
{"panel_index": 18, "task_family": "hs3", "logits": [19.625, 16.375], "probabilities": [0.9626730680465698, 0.037326887249946594], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 3.725290298461914e-09}
{"panel_index": 19, "task_family": "hs3", "logits": [18.875, 12.8125], "probabilities": [0.9976769089698792, 0.0023231625091284513], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 20, "task_family": "hs3", "logits": [19.125, 13.0], "probabilities": [0.9978172779083252, 0.0021827169694006443], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00014044553972780704}
{"panel_index": 21, "task_family": "hs3", "logits": [11.6875, 18.5], "probabilities": [0.0010987314162775874, 0.9989012479782104], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 3.4924596548080444e-10}
{"panel_index": 22, "task_family": "hs3", "logits": [19.375, 12.6875], "probabilities": [0.9987550973892212, 0.0012448437046259642], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 7.539987564086914e-05}
{"panel_index": 23, "task_family": "hs3", "logits": [19.0, 15.5625], "probabilities": [0.9688562154769897, 0.031143829226493835], "choice": "yes", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.005186472088098526}
{"panel_index": 24, "task_family": "hs3", "logits": [19.0, 12.875], "probabilities": [0.9978172779083252, 0.0021827169694006443], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 4.656612873077393e-10}
{"panel_index": 25, "task_family": "hs3", "logits": [13.125, 19.0], "probabilities": [0.0028009277302771807, 0.9971990585327148], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 6.984919309616089e-10}
{"panel_index": 26, "task_family": "hs3", "logits": [18.875, 15.875], "probabilities": [0.9525741338729858, 0.047425881028175354], "choice": "yes", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.005977451801300049}
{"panel_index": 27, "task_family": "hs3", "logits": [19.25, 13.0], "probabilities": [0.9980732202529907, 0.001926734927110374], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00022608041763305664}
{"panel_index": 28, "task_family": "hs3", "logits": [15.6875, 18.5], "probabilities": [0.05665243789553642, 0.9433475732803345], "choice": "no", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 1.1175870895385742e-08}
{"panel_index": 29, "task_family": "hs3", "logits": [12.0625, 18.5], "probabilities": [0.0015978461597114801, 0.9984021782875061], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 5.820766091346741e-10}
{"panel_index": 30, "task_family": "hs3", "logits": [18.75, 14.125], "probabilities": [0.9902915358543396, 0.00970847811549902], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0005828738212585449}
{"panel_index": 31, "task_family": "hs3", "logits": [19.25, 13.0625], "probabilities": [0.9979493021965027, 0.0020507434383034706], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0001240087440237403}
{"panel_index": 32, "task_family": "boolq", "logits": [13.125, 18.5], "probabilities": [0.004609572701156139, 0.9953904151916504], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 33, "task_family": "boolq", "logits": [12.375, 19.0], "probabilities": [0.0013250223128125072, 0.9986749291419983], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00017615989781916142}
{"panel_index": 34, "task_family": "boolq", "logits": [12.8125, 18.625], "probabilities": [0.0029810331761837006, 0.9970190525054932], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.3283064365386963e-10}
{"panel_index": 35, "task_family": "boolq", "logits": [19.5, 13.625], "probabilities": [0.9971990585327148, 0.0028009277302771807], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00032836198806762695}
{"panel_index": 36, "task_family": "boolq", "logits": [19.875, 12.1875], "probabilities": [0.9995417594909668, 0.0004583129193633795], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.775582834146917e-05}
{"panel_index": 37, "task_family": "boolq", "logits": [20.0, 12.3125], "probabilities": [0.9995417594909668, 0.0004583129193633795], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 38, "task_family": "boolq", "logits": [19.625, 12.25], "probabilities": [0.999373733997345, 0.0006263344548642635], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.9103830456733704e-10}
{"panel_index": 39, "task_family": "boolq", "logits": [13.125, 19.125], "probabilities": [0.002472624182701111, 0.9975274205207825], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00032836198806762695}
{"panel_index": 40, "task_family": "boolq", "logits": [19.75, 12.0625], "probabilities": [0.9995417594909668, 0.0004583129193633795], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.775582834146917e-05}
{"panel_index": 41, "task_family": "boolq", "logits": [13.25, 18.75], "probabilities": [0.004070138558745384, 0.9959298968315125], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 4.656612873077393e-10}
{"panel_index": 42, "task_family": "boolq", "logits": [19.5, 12.25], "probabilities": [0.9992903470993042, 0.0007096707122400403], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.9103830456733704e-10}
{"panel_index": 43, "task_family": "boolq", "logits": [14.75, 18.375], "probabilities": [0.025957360863685608, 0.9740425944328308], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.001627923920750618}
{"panel_index": 44, "task_family": "boolq", "logits": [19.125, 12.8125], "probabilities": [0.9981897473335266, 0.0018102105241268873], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00010955333709716797}
{"panel_index": 45, "task_family": "boolq", "logits": [20.5, 13.75], "probabilities": [0.9988304972648621, 0.0011695101857185364], "choice": "yes", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 46, "task_family": "boolq", "logits": [13.8125, 18.625], "probabilities": [0.00806199200451374, 0.9919379353523254], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 47, "task_family": "boolq", "logits": [12.75, 18.625], "probabilities": [0.0028009277302771807, 0.9971990585327148], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00016927719116210938}
{"panel_index": 48, "task_family": "mnli", "logits": [16.375, 19.375, 14.3125], "probabilities": [0.04714163765311241, 0.9468649625778198, 0.005993385799229145], "choice": "neutral", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.010328948497772217}
{"panel_index": 49, "task_family": "mnli", "logits": [13.125, 20.5, 12.3125], "probabilities": [0.0006261604139581323, 0.9990960359573364, 0.0002778570051304996], "choice": "neutral", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 1.800060272216797e-05}
{"panel_index": 50, "task_family": "mnli", "logits": [20.25, 12.4375, 11.4375], "probabilities": [0.9994468092918396, 0.0004044212691951543, 0.00014877831563353539], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 9.593830327503383e-06}
{"panel_index": 51, "task_family": "mnli", "logits": [13.5, 20.25, 12.5], "probabilities": [0.001169007271528244, 0.9984009861946106, 0.00043005385668948293], "choice": "neutral", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 8.731149137020111e-11}
{"panel_index": 52, "task_family": "mnli", "logits": [11.25, 12.0, 19.875], "probabilities": [0.0001794597483240068, 0.0003799163969233632, 0.999440610408783], "choice": "contradiction", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 4.83393669128418e-05}
{"panel_index": 53, "task_family": "mnli", "logits": [20.125, 12.0625, 11.25], "probabilities": [0.999545156955719, 0.0003149947733618319, 0.00013977808703202754], "choice": "entailment", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 1.919269561767578e-05}
{"panel_index": 54, "task_family": "mnli", "logits": [16.75, 19.75, 13.5], "probabilities": [0.0473388247191906, 0.9508256316184998, 0.0018355256179347634], "choice": "neutral", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.012840867042541504}
{"panel_index": 55, "task_family": "mnli", "logits": [20.125, 12.375, 11.375], "probabilities": [0.9994112253189087, 0.000430489017162472, 0.0001583680132171139], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.777576446533203e-05}
{"panel_index": 56, "task_family": "mnli", "logits": [19.875, 12.125, 11.375], "probabilities": [0.9993662238121033, 0.0004304696340113878, 0.00020333939755801111], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 5.4776668548583984e-05}
{"panel_index": 57, "task_family": "mnli", "logits": [14.5, 17.625, 19.625], "probabilities": [0.005210123024880886, 0.1185818687081337, 0.8762080073356628], "choice": "contradiction", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.043473899364471436}
{"panel_index": 58, "task_family": "mnli", "logits": [13.875, 19.75, 15.25], "probabilities": [0.0027702394872903824, 0.9862732291221619, 0.010956508107483387], "choice": "neutral", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 9.313225746154785e-10}
{"panel_index": 59, "task_family": "mnli", "logits": [19.375, 14.375, 16.125], "probabilities": [0.9564689993858337, 0.006444637198001146, 0.037086330354213715], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.06959861516952515}
{"panel_index": 60, "task_family": "mnli", "logits": [14.1875, 14.9375, 19.875], "probabilities": [0.0033526471816003323, 0.007097553927451372, 0.9895498752593994], "choice": "contradiction", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.0008159875869750977}
{"panel_index": 61, "task_family": "mnli", "logits": [19.5, 16.75, 14.0625], "probabilities": [0.9360857605934143, 0.05984196439385414, 0.004072297364473343], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 7.450580596923828e-09}
{"panel_index": 62, "task_family": "mnli", "logits": [19.875, 12.5625, 11.625], "probabilities": [0.9990724325180054, 0.0006665283581241965, 0.00026101621915586293], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 1.6829435480758548e-05}
{"panel_index": 63, "task_family": "mnli", "logits": [20.0, 12.1875, 11.375], "probabilities": [0.9994162321090698, 0.0004044089000672102, 0.00017945536819752306], "choice": "entailment", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 2.9103830456733704e-11}
{"panel_index": 64, "task_family": "score", "logits": [18.75, 18.375], "probabilities": [0.5926666259765625, 0.4073334336280823], "choice": "yes", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 65, "task_family": "score", "logits": [12.125, 19.75], "probabilities": [0.00048785729450173676, 0.9995121955871582], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 1.1641532182693481e-10}
{"panel_index": 66, "task_family": "score", "logits": [12.0625, 19.75], "probabilities": [0.0004583129193633795, 0.9995417594909668], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 67, "task_family": "score", "logits": [13.5625, 19.375], "probabilities": [0.0029810331761837006, 0.9970190525054932], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00019174814224243164}
{"panel_index": 68, "task_family": "score", "logits": [12.5625, 19.5], "probabilities": [0.0009697522036731243, 0.9990302324295044], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 5.8770179748535156e-05}
{"panel_index": 69, "task_family": "score", "logits": [19.75, 12.5625], "probabilities": [0.9992445707321167, 0.0007554057519882917], "choice": "yes", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 4.57763671875e-05}
{"panel_index": 70, "task_family": "score", "logits": [12.75, 19.75], "probabilities": [0.0009110511164180934, 0.9990890026092529], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 5.8770179748535156e-05}
{"panel_index": 71, "task_family": "score", "logits": [12.625, 19.75], "probabilities": [0.0008040857501327991, 0.9991958737373352], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 1.7462298274040222e-10}
{"panel_index": 72, "task_family": "score", "logits": [19.125, 18.375], "probabilities": [0.6791787147521973, 0.3208213150501251], "choice": "yes", "correct": false, "agrees_with_reference": true, "max_abs_probability_delta": 0.1479693353176117}
{"panel_index": 73, "task_family": "score", "logits": [11.9375, 19.5], "probabilities": [0.0005193048273213208, 0.999480664730072], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 74, "task_family": "score", "logits": [12.1875, 19.75], "probabilities": [0.0005193048273213208, 0.999480664730072], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0}
{"panel_index": 75, "task_family": "score", "logits": [12.6875, 19.375], "probabilities": [0.0012448437046259642, 0.9987550973892212], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 3.4924596548080444e-10}
{"panel_index": 76, "task_family": "score", "logits": [15.25, 19.375], "probabilities": [0.01590639166533947, 0.9840936064720154], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0010085217654705048}
{"panel_index": 77, "task_family": "score", "logits": [15.375, 19.125], "probabilities": [0.022977374494075775, 0.977022647857666], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.0013620853424072266}
{"panel_index": 78, "task_family": "score", "logits": [13.75, 19.625], "probabilities": [0.0028009277302771807, 0.9971990585327148], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00016927719116210938}
{"panel_index": 79, "task_family": "score", "logits": [13.375, 19.375], "probabilities": [0.002472624182701111, 0.9975274205207825], "choice": "no", "correct": true, "agrees_with_reference": true, "max_abs_probability_delta": 0.00028990767896175385}