| { |
| "val": { |
| "states": 180, |
| "questions": 1325, |
| "ece": 0.08648470789194107, |
| "groups": { |
| "maze/action": { |
| "n": 97, |
| "boards": 97, |
| "accuracy": 1.0, |
| "uniform_control": 0.49054983073903113, |
| "tvd": 0.018844484970534103, |
| "brier": 0.002279498944011423 |
| }, |
| "maze/boolean": { |
| "n": 530, |
| "boards": 106, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0006993013012381095, |
| "brier": 1.9829470994958475e-06 |
| }, |
| "maze/choice": { |
| "n": 97, |
| "boards": 97, |
| "accuracy": 1.0, |
| "uniform_control": 0.49054983073903113, |
| "tvd": 0.018844484970534103, |
| "brier": 0.002279498944011423 |
| }, |
| "maze/clear": { |
| "n": 424, |
| "boards": 106, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0004252428954954804, |
| "brier": 8.113499066392354e-07 |
| }, |
| "maze/distance": { |
| "n": 106, |
| "boards": 106, |
| "accuracy": 0.39622641509433965, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.7763906526115706, |
| "brier": 0.7639503164111443 |
| }, |
| "maze/score": { |
| "n": 106, |
| "boards": 106, |
| "accuracy": 0.39622641509433965, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.7763906526115706, |
| "brier": 0.7639503164111443 |
| }, |
| "maze/solvable": { |
| "n": 106, |
| "boards": 106, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.001795534924208626, |
| "brier": 6.6693358709222975e-06 |
| }, |
| "snake/action": { |
| "n": 74, |
| "boards": 74, |
| "accuracy": 0.9864864864864865, |
| "uniform_control": 0.4954955098596779, |
| "tvd": 0.35000925611176, |
| "brier": 0.3147798128456769 |
| }, |
| "snake/boolean": { |
| "n": 444, |
| "boards": 74, |
| "accuracy": 0.8828828828828829, |
| "uniform_control": 0.5, |
| "tvd": 0.23378823729450102, |
| "brier": 0.20216913486784743 |
| }, |
| "snake/choice": { |
| "n": 74, |
| "boards": 74, |
| "accuracy": 0.9864864864864865, |
| "uniform_control": 0.4954955098596779, |
| "tvd": 0.35000925611176, |
| "brier": 0.3147798128456769 |
| }, |
| "snake/escape": { |
| "n": 222, |
| "boards": 74, |
| "accuracy": 0.8108108108108109, |
| "uniform_control": 0.5, |
| "tvd": 0.2772626264507438, |
| "brier": 0.28911788216246675 |
| }, |
| "snake/room": { |
| "n": 74, |
| "boards": 74, |
| "accuracy": 0.28378378378378377, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.7990304521612219, |
| "brier": 0.7980756687151419 |
| }, |
| "snake/safe": { |
| "n": 222, |
| "boards": 74, |
| "accuracy": 0.954954954954955, |
| "uniform_control": 0.5, |
| "tvd": 0.19031384813825827, |
| "brier": 0.1152203875732281 |
| }, |
| "snake/score": { |
| "n": 74, |
| "boards": 74, |
| "accuracy": 0.28378378378378377, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.7990304521612219, |
| "brier": 0.7980756687151419 |
| } |
| } |
| }, |
| "test": { |
| "states": 240, |
| "questions": 1765, |
| "ece": 0.059004783630371094, |
| "groups": { |
| "maze/action": { |
| "n": 141, |
| "boards": 140, |
| "accuracy": 1.0, |
| "uniform_control": 0.48817967269437534, |
| "tvd": 0.03961129047573943, |
| "brier": 0.007873877364980677 |
| }, |
| "maze/boolean": { |
| "n": 740, |
| "boards": 147, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0006845150103578596, |
| "brier": 2.021968730086637e-06 |
| }, |
| "maze/choice": { |
| "n": 141, |
| "boards": 140, |
| "accuracy": 1.0, |
| "uniform_control": 0.48817967269437534, |
| "tvd": 0.03961129047573943, |
| "brier": 0.007873877364980677 |
| }, |
| "maze/clear": { |
| "n": 592, |
| "boards": 147, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.00038996367161539763, |
| "brier": 7.440200639129634e-07 |
| }, |
| "maze/distance": { |
| "n": 148, |
| "boards": 147, |
| "accuracy": 0.2972972972972973, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.7846677939634066, |
| "brier": 0.7795776144878285 |
| }, |
| "maze/score": { |
| "n": 148, |
| "boards": 147, |
| "accuracy": 0.2972972972972973, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.7846677939634066, |
| "brier": 0.7795776144878285 |
| }, |
| "maze/solvable": { |
| "n": 148, |
| "boards": 147, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0018627203653277073, |
| "brier": 7.133763394781331e-06 |
| }, |
| "snake/action": { |
| "n": 92, |
| "boards": 92, |
| "accuracy": 0.967391304347826, |
| "uniform_control": 0.47826088349456375, |
| "tvd": 0.3106858467774304, |
| "brier": 0.2770434002752654 |
| }, |
| "snake/boolean": { |
| "n": 552, |
| "boards": 92, |
| "accuracy": 0.8894927536231884, |
| "uniform_control": 0.5, |
| "tvd": 0.22231864719314204, |
| "brier": 0.19377556414448022 |
| }, |
| "snake/choice": { |
| "n": 92, |
| "boards": 92, |
| "accuracy": 0.967391304347826, |
| "uniform_control": 0.47826088349456375, |
| "tvd": 0.3106858467774304, |
| "brier": 0.2770434002752654 |
| }, |
| "snake/escape": { |
| "n": 276, |
| "boards": 92, |
| "accuracy": 0.8188405797101449, |
| "uniform_control": 0.5, |
| "tvd": 0.26280311700226605, |
| "brier": 0.2747444752047 |
| }, |
| "snake/room": { |
| "n": 92, |
| "boards": 92, |
| "accuracy": 0.25, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.7993024775515432, |
| "brier": 0.7986197102329006 |
| }, |
| "snake/safe": { |
| "n": 276, |
| "boards": 92, |
| "accuracy": 0.9601449275362319, |
| "uniform_control": 0.5, |
| "tvd": 0.18183417738401803, |
| "brier": 0.1128066530842604 |
| }, |
| "snake/score": { |
| "n": 92, |
| "boards": 92, |
| "accuracy": 0.25, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.7993024775515432, |
| "brier": 0.7986197102329006 |
| } |
| } |
| }, |
| "ood": { |
| "states": 160, |
| "questions": 1167, |
| "ece": 0.05399147421121597, |
| "groups": { |
| "maze/action": { |
| "n": 107, |
| "boards": 107, |
| "accuracy": 1.0, |
| "uniform_control": 0.5132398789174089, |
| "tvd": 0.26991402065364, |
| "brier": 0.1504033250384964 |
| }, |
| "maze/boolean": { |
| "n": 550, |
| "boards": 110, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0007242930423341353, |
| "brier": 2.258754991337562e-06 |
| }, |
| "maze/choice": { |
| "n": 107, |
| "boards": 107, |
| "accuracy": 1.0, |
| "uniform_control": 0.5132398789174089, |
| "tvd": 0.26991402065364, |
| "brier": 0.1504033250384964 |
| }, |
| "maze/clear": { |
| "n": 440, |
| "boards": 110, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0004054153733622198, |
| "brier": 7.734664848157991e-07 |
| }, |
| "maze/distance": { |
| "n": 110, |
| "boards": 110, |
| "accuracy": 0.05454545454545454, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.8541775340383703, |
| "brier": 0.9160033637827093 |
| }, |
| "maze/score": { |
| "n": 110, |
| "boards": 110, |
| "accuracy": 0.05454545454545454, |
| "uniform_control": 0.1428571492433548, |
| "tvd": 0.8541775340383703, |
| "brier": 0.9160033637827093 |
| }, |
| "maze/solvable": { |
| "n": 110, |
| "boards": 110, |
| "accuracy": 1.0, |
| "uniform_control": 0.5, |
| "tvd": 0.0019998037182217972, |
| "brier": 8.199909017424612e-06 |
| }, |
| "snake/action": { |
| "n": 50, |
| "boards": 50, |
| "accuracy": 0.98, |
| "uniform_control": 0.4866666805744171, |
| "tvd": 0.35739153509028254, |
| "brier": 0.3199479156875387 |
| }, |
| "snake/boolean": { |
| "n": 300, |
| "boards": 50, |
| "accuracy": 0.8333333333333334, |
| "uniform_control": 0.5, |
| "tvd": 0.26355026371777057, |
| "brier": 0.262066046569186 |
| }, |
| "snake/choice": { |
| "n": 50, |
| "boards": 50, |
| "accuracy": 0.98, |
| "uniform_control": 0.4866666805744171, |
| "tvd": 0.35739153509028254, |
| "brier": 0.3199479156875387 |
| }, |
| "snake/escape": { |
| "n": 150, |
| "boards": 50, |
| "accuracy": 0.68, |
| "uniform_control": 0.5, |
| "tvd": 0.35605070618291695, |
| "brier": 0.4470668915938586 |
| }, |
| "snake/room": { |
| "n": 50, |
| "boards": 50, |
| "accuracy": 0.2, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.798902131319046, |
| "brier": 0.7978189885616302 |
| }, |
| "snake/safe": { |
| "n": 150, |
| "boards": 50, |
| "accuracy": 0.9866666666666667, |
| "uniform_control": 0.5, |
| "tvd": 0.1710498212526242, |
| "brier": 0.07706520154451331 |
| }, |
| "snake/score": { |
| "n": 50, |
| "boards": 50, |
| "accuracy": 0.2, |
| "uniform_control": 0.20000000298023224, |
| "tvd": 0.798902131319046, |
| "brier": 0.7978189885616302 |
| } |
| } |
| } |
| } |
