| { |
| "cases": 157, |
| "argmax_flips": 0, |
| "max_probability_difference": 0.04756533417616565, |
| "selection": "Previous fixed 144 merge cases plus 12 hash-selected focused holdout cases and longest API-Bank request. Not an independent benchmark.", |
| "calibration": "Inherited v6.1 temperatures, not refitted.", |
| "base_download_required": false, |
| "wrapper_smoke": { |
| "model": "michaljach/jet", |
| "answers": { |
| "category": { |
| "type": "choice", |
| "choice": "damage", |
| "probabilities": { |
| "damage": 0.9903, |
| "billing": 0.0097 |
| }, |
| "confidence": 0.9209 |
| }, |
| "severity": { |
| "type": "score", |
| "score": 1.555, |
| "level": "severe", |
| "probabilities": [ |
| 0.0161, |
| 0.4128, |
| 0.5711 |
| ], |
| "confidence": 0.3159 |
| }, |
| "damaged": { |
| "type": "noul", |
| "probability": 0.9487, |
| "confidence": 0.7082 |
| } |
| } |
| }, |
| "context_probe": { |
| "passed": true, |
| "tokens": 11495, |
| "max_tokens": 16384 |
| }, |
| "records": [ |
| { |
| "case": 0, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.002987165380278878 |
| }, |
| { |
| "case": 1, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 2, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 3, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0008522927862807308 |
| }, |
| { |
| "case": 4, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 5, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0005733880531314777 |
| }, |
| { |
| "case": 6, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0006614689289068752 |
| }, |
| { |
| "case": 7, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 8, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.000899492764544152 |
| }, |
| { |
| "case": 9, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0016077412194476226 |
| }, |
| { |
| "case": 10, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0036649808951511043 |
| }, |
| { |
| "case": 11, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0011758151845370698 |
| }, |
| { |
| "case": 12, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0007087150140926424 |
| }, |
| { |
| "case": 13, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0006711365186986666 |
| }, |
| { |
| "case": 14, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0020361407568059223 |
| }, |
| { |
| "case": 15, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 6.886235910508187e-05 |
| }, |
| { |
| "case": 16, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0012989605282783812 |
| }, |
| { |
| "case": 17, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0005189743557448834 |
| }, |
| { |
| "case": 18, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0027470613975926206 |
| }, |
| { |
| "case": 19, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.010187698043257254 |
| }, |
| { |
| "case": 20, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 21, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.001454303669506496 |
| }, |
| { |
| "case": 22, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.013085229248764652 |
| }, |
| { |
| "case": 23, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0010584632570356245 |
| }, |
| { |
| "case": 24, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0020582766106718824 |
| }, |
| { |
| "case": 25, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0005125698442036922 |
| }, |
| { |
| "case": 26, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0005830234269115341 |
| }, |
| { |
| "case": 27, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0007246417457215798 |
| }, |
| { |
| "case": 28, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.000490928734522722 |
| }, |
| { |
| "case": 29, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0013795481306675406 |
| }, |
| { |
| "case": 30, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0006006729030451161 |
| }, |
| { |
| "case": 31, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0011758151845370698 |
| }, |
| { |
| "case": 32, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.012648434256519445 |
| }, |
| { |
| "case": 33, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.01321697504337338 |
| }, |
| { |
| "case": 34, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0024019477258491392 |
| }, |
| { |
| "case": 35, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 36, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 37, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 38, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.003854598977123258 |
| }, |
| { |
| "case": 39, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.013136055229324622 |
| }, |
| { |
| "case": 40, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0018665708326972563 |
| }, |
| { |
| "case": 41, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.000692937761531856 |
| }, |
| { |
| "case": 42, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 43, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.000723617570070112 |
| }, |
| { |
| "case": 44, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.007833504601458263 |
| }, |
| { |
| "case": 45, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 46, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.017761061358939845 |
| }, |
| { |
| "case": 47, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0043677575202951124 |
| }, |
| { |
| "case": 48, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 49, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0022369316702290296 |
| }, |
| { |
| "case": 50, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.002032526605908448 |
| }, |
| { |
| "case": 51, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.012227965640579014 |
| }, |
| { |
| "case": 52, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.00040181997200938735 |
| }, |
| { |
| "case": 53, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 54, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 55, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 56, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.005661205903540578 |
| }, |
| { |
| "case": 57, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 58, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 59, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 60, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0019360099221095162 |
| }, |
| { |
| "case": 61, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 62, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 63, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.02634154028477992 |
| }, |
| { |
| "case": 64, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.008880156115223817 |
| }, |
| { |
| "case": 65, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 66, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.003854598977123258 |
| }, |
| { |
| "case": 67, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0007447583966543204 |
| }, |
| { |
| "case": 68, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0001146676099607895 |
| }, |
| { |
| "case": 69, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0029591419592648327 |
| }, |
| { |
| "case": 70, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.015230252265583333 |
| }, |
| { |
| "case": 71, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 72, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.020053874683156447 |
| }, |
| { |
| "case": 73, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0199179731534751 |
| }, |
| { |
| "case": 74, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 75, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0050653758778449465 |
| }, |
| { |
| "case": 76, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0024803049079245507 |
| }, |
| { |
| "case": 77, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 78, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.012005088055840801 |
| }, |
| { |
| "case": 79, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0055398063440276965 |
| }, |
| { |
| "case": 80, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.018784706325125744 |
| }, |
| { |
| "case": 81, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 82, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.003274203457591829 |
| }, |
| { |
| "case": 83, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 84, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.01720608744436994 |
| }, |
| { |
| "case": 85, |
| "type": "noul", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 86, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.000461312361028976 |
| }, |
| { |
| "case": 87, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.026329934699847568 |
| }, |
| { |
| "case": 88, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 89, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0008237772015591352 |
| }, |
| { |
| "case": 90, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 91, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.0004971529853300544 |
| }, |
| { |
| "case": 92, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.012287200915141808 |
| }, |
| { |
| "case": 93, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.006122007063984927 |
| }, |
| { |
| "case": 94, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.04756533417616565 |
| }, |
| { |
| "case": 95, |
| "type": "score", |
| "same_argmax": true, |
| "max_probability_difference": 0.011753993010866715 |
| }, |
| { |
| "case": 96, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 97, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 98, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 99, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 100, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.04477957280437994 |
| }, |
| { |
| "case": 101, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0018032455547183357 |
| }, |
| { |
| "case": 102, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.01684820403685175 |
| }, |
| { |
| "case": 103, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 104, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.01599744069486378 |
| }, |
| { |
| "case": 105, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 106, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.01890952286570502 |
| }, |
| { |
| "case": 107, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0034353363281827143 |
| }, |
| { |
| "case": 108, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 109, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 110, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 111, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 112, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.01023223405796686 |
| }, |
| { |
| "case": 113, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 114, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 115, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 116, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.001163385786891992 |
| }, |
| { |
| "case": 117, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.010916428349646567 |
| }, |
| { |
| "case": 118, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.01999364952511129 |
| }, |
| { |
| "case": 119, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.004639380151772588 |
| }, |
| { |
| "case": 120, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0004504036257898525 |
| }, |
| { |
| "case": 121, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.001900401342050162 |
| }, |
| { |
| "case": 122, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0016769366925224183 |
| }, |
| { |
| "case": 123, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0011628789777733337 |
| }, |
| { |
| "case": 124, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 125, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.012155667530471606 |
| }, |
| { |
| "case": 126, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0015901874331142977 |
| }, |
| { |
| "case": 127, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.002292551478361493 |
| }, |
| { |
| "case": 128, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 129, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 130, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 131, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 132, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 133, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.002977611927637354 |
| }, |
| { |
| "case": 134, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.029698411658723306 |
| }, |
| { |
| "case": 135, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 136, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 137, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 138, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0015901737796318767 |
| }, |
| { |
| "case": 139, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 140, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 141, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0023174474419890914 |
| }, |
| { |
| "case": 142, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 143, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 144, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.00018020386372263886 |
| }, |
| { |
| "case": 145, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0006541617315441572 |
| }, |
| { |
| "case": 146, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.006506129065911526 |
| }, |
| { |
| "case": 147, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.008089011161791393 |
| }, |
| { |
| "case": 148, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 149, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.004038183129593295 |
| }, |
| { |
| "case": 150, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 151, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.005692637355534935 |
| }, |
| { |
| "case": 152, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 153, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.00397777773328345 |
| }, |
| { |
| "case": 154, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 155, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0 |
| }, |
| { |
| "case": 156, |
| "type": "choice", |
| "same_argmax": true, |
| "max_probability_difference": 0.0138867288785047 |
| } |
| ], |
| "merged_holdout": { |
| "objective": 0.7372740069451986, |
| "families": { |
| "retention": { |
| "n": 500, |
| "accuracy": 0.934, |
| "nll": 0.21870660245549345, |
| "metric": 0.934, |
| "metric_name": "accuracy" |
| }, |
| "finance": { |
| "n": 180, |
| "accuracy": 0.7111111111111111, |
| "nll": 0.7353250763802403, |
| "metric": 0.7121353544404619, |
| "metric_name": "macro-F1" |
| }, |
| "banking": { |
| "n": 154, |
| "accuracy": 0.7467532467532467, |
| "nll": 1.2266273153352905, |
| "metric": 0.7467532467532467, |
| "metric_name": "accuracy" |
| }, |
| "sarcasm": { |
| "n": 80, |
| "accuracy": 0.725, |
| "nll": 0.6158682200969627, |
| "metric": 0.5, |
| "metric_name": "positive-class F1" |
| } |
| } |
| }, |
| "released_holdout": { |
| "objective": 0.7297131700522546, |
| "families": { |
| "retention": { |
| "n": 500, |
| "accuracy": 0.934, |
| "nll": 0.2194680695671052, |
| "metric": 0.934, |
| "metric_name": "accuracy" |
| }, |
| "finance": { |
| "n": 180, |
| "accuracy": 0.7111111111111111, |
| "nll": 0.7504884215008701, |
| "metric": 0.7116466613734374, |
| "metric_name": "macro-F1" |
| }, |
| "banking": { |
| "n": 154, |
| "accuracy": 0.7467532467532467, |
| "nll": 1.2402327169109493, |
| "metric": 0.7467532467532467, |
| "metric_name": "accuracy" |
| }, |
| "sarcasm": { |
| "n": 80, |
| "accuracy": 0.6875, |
| "nll": 0.6216465693736991, |
| "metric": 0.46808510638297873, |
| "metric_name": "positive-class F1" |
| } |
| } |
| }, |
| "merge_passed": true, |
| "holdout_passed": true, |
| "passed": true |
| } |
|
|
