{
 "model": "decider-4b-v2.1",
 "temperature": 1.099,
 "temperature_by_type": {
  "choice": 1.11,
  "noul": 1.56,
  "score": 1.287
 },
 "temperature_fit": {
  "global": "one scalar, pooled row NLL on the in-task half of the public regression set without banking77, clinc_oos, mmlu, arc, winogrande and hellaswag (61 tasks, 102,804 rows)",
  "by_type": "decider.calibrate.fit_by_type (decider-ai 1.4.0), min_rows 50, on the same regression rows plus our own validation rows (val_jb, val_teacher, val_teacher2, cal_human without mmlu, arc and banking77, val_replay without six excluded families)",
  "rows": {
   "choice": 108927,
   "noul": 1677,
   "score": 535
  },
  "note": "no JevBench or Decision Index item was used for training, selection or temperature"
 },
 "protocol": "Regression set: eval half of the mixture, 95 tasks (67 in-task / 28 held-out), plain state-first layout, large label sets sub-sampled to 10 options, ECE 15 bins, per-task mean. Own sets (heldout_jb, test_teacher2, cal_human, guard): stored T=1 logits read at the served temperatures, ECE 10 bins top label. Card lanes (fixtures, games, browser, probes, Bespoke, issue #9) through decider-ai 1.4.0 with the map; parents through 1.3.0 at their released temperatures, same rows and seeds; intervals are 95% paired bootstrap. JevBench public items read in process through 1.4.0 with the map; parents' files read earlier through 1.2.1.",
 "regression": {
  "map": {
   "reg_in": 0.8308,
   "reg_out": 0.7838,
   "reg_in_nll": 0.4145,
   "reg_out_nll": 0.5689,
   "reg_in_ece": 0.0309,
   "reg_out_ece": 0.0775
  },
  "global_T": {
   "reg_in": 0.8308,
   "reg_out": 0.7838,
   "reg_in_nll": 0.4145,
   "reg_out_nll": 0.5703,
   "reg_in_ece": 0.0308,
   "reg_out_ece": 0.0781
  }
 },
 "own_sets": {
  "map": {
   "heldout_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.556,
     "nll": 1.2571,
     "ece": 0.1466,
     "mean_conf": 0.7026
    },
    "by_type": {
     "choice": {
      "n": 3293,
      "accuracy": 0.5056,
      "nll": 1.4995,
      "ece": 0.1701,
      "mean_conf": 0.6757
     },
     "noul": {
      "n": 1289,
      "accuracy": 0.7114,
      "nll": 0.6074,
      "ece": 0.1013,
      "mean_conf": 0.8127
     },
     "score": {
      "n": 418,
      "accuracy": 0.4737,
      "nll": 1.3504,
      "ece": 0.1081,
      "mean_conf": 0.5747
     }
    }
   },
   "test_teacher2": {
    "all": {
     "n": 449,
     "accuracy": 0.8196,
     "nll": 0.4304,
     "ece": 0.043,
     "mean_conf": 0.8559
    },
    "by_type": {
     "choice": {
      "n": 257,
      "accuracy": 0.8016,
      "nll": 0.5041,
      "ece": 0.0569,
      "mean_conf": 0.8585
     },
     "noul": {
      "n": 130,
      "accuracy": 0.8692,
      "nll": 0.2489,
      "ece": 0.0479,
      "mean_conf": 0.905
     },
     "score": {
      "n": 62,
      "accuracy": 0.7903,
      "nll": 0.5051,
      "ece": 0.0863,
      "mean_conf": 0.7423
     }
    }
   },
   "guard": {
    "all": {
     "n": 2994,
     "accuracy": 0.8186,
     "nll": 0.4866,
     "ece": 0.0168,
     "mean_conf": 0.8344
    },
    "by_type": {
     "choice": {
      "n": 2994,
      "accuracy": 0.8186,
      "nll": 0.4866,
      "ece": 0.0168,
      "mean_conf": 0.8344
     },
     "noul": null,
     "score": null
    }
   },
   "val_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.796,
     "nll": 0.5264,
     "ece": 0.0157,
     "mean_conf": 0.8117
    },
    "by_type": {
     "choice": {
      "n": 3374,
      "accuracy": 0.7647,
      "nll": 0.6258,
      "ece": 0.0289,
      "mean_conf": 0.7935
     },
     "noul": {
      "n": 1273,
      "accuracy": 0.901,
      "nll": 0.2233,
      "ece": 0.0161,
      "mean_conf": 0.8849
     },
     "score": {
      "n": 353,
      "accuracy": 0.7167,
      "nll": 0.6681,
      "ece": 0.0317,
      "mean_conf": 0.722
     }
    }
   },
   "val_teacher": {
    "all": {
     "n": 944,
     "accuracy": 0.8379,
     "nll": 0.4169,
     "ece": 0.0369,
     "mean_conf": 0.8619
    },
    "by_type": {
     "choice": {
      "n": 540,
      "accuracy": 0.8315,
      "nll": 0.4643,
      "ece": 0.0464,
      "mean_conf": 0.8547
     },
     "noul": {
      "n": 284,
      "accuracy": 0.8873,
      "nll": 0.2931,
      "ece": 0.0453,
      "mean_conf": 0.9088
     },
     "score": {
      "n": 120,
      "accuracy": 0.75,
      "nll": 0.4969,
      "ece": 0.0576,
      "mean_conf": 0.7831
     }
    }
   },
   "val_teacher2": {
    "all": {
     "n": 447,
     "accuracy": 0.8322,
     "nll": 0.4435,
     "ece": 0.0627,
     "mean_conf": 0.8747
    },
    "by_type": {
     "choice": {
      "n": 265,
      "accuracy": 0.8377,
      "nll": 0.4219,
      "ece": 0.0503,
      "mean_conf": 0.8858
     },
     "noul": {
      "n": 120,
      "accuracy": 0.85,
      "nll": 0.3702,
      "ece": 0.0827,
      "mean_conf": 0.9034
     },
     "score": {
      "n": 62,
      "accuracy": 0.7742,
      "nll": 0.6773,
      "ece": 0.1441,
      "mean_conf": 0.772
     }
    }
   },
   "cal_human": {
    "all": {
     "n": 1595,
     "accuracy": 0.8564,
     "nll": 0.3892,
     "ece": 0.0205,
     "mean_conf": 0.8717
    },
    "by_type": {
     "choice": {
      "n": 1595,
      "accuracy": 0.8564,
      "nll": 0.3892,
      "ece": 0.0205,
      "mean_conf": 0.8717
     },
     "noul": null,
     "score": null
    }
   },
   "val_replay4": {
    "all": {
     "n": 1000,
     "accuracy": 0.889,
     "nll": 0.2848,
     "ece": 0.0297,
     "mean_conf": 0.8702
    },
    "by_type": {
     "choice": {
      "n": 1000,
      "accuracy": 0.889,
      "nll": 0.2848,
      "ece": 0.0297,
      "mean_conf": 0.8702
     },
     "noul": null,
     "score": null
    }
   }
  },
  "global_T": {
   "heldout_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.5562,
     "nll": 1.2937,
     "ece": 0.163,
     "mean_conf": 0.7192
    },
    "by_type": {
     "choice": {
      "n": 3293,
      "accuracy": 0.5056,
      "nll": 1.5059,
      "ece": 0.1724,
      "mean_conf": 0.678
     },
     "noul": {
      "n": 1289,
      "accuracy": 0.7114,
      "nll": 0.7112,
      "ece": 0.1463,
      "mean_conf": 0.8577
     },
     "score": {
      "n": 418,
      "accuracy": 0.4761,
      "nll": 1.4183,
      "ece": 0.1402,
      "mean_conf": 0.6163
     }
    }
   },
   "test_teacher2": {
    "all": {
     "n": 449,
     "accuracy": 0.8196,
     "nll": 0.4381,
     "ece": 0.0497,
     "mean_conf": 0.8664
    },
    "by_type": {
     "choice": {
      "n": 257,
      "accuracy": 0.8016,
      "nll": 0.5058,
      "ece": 0.058,
      "mean_conf": 0.8596
     },
     "noul": {
      "n": 130,
      "accuracy": 0.8692,
      "nll": 0.2755,
      "ece": 0.0602,
      "mean_conf": 0.9294
     },
     "score": {
      "n": 62,
      "accuracy": 0.7903,
      "nll": 0.4984,
      "ece": 0.0798,
      "mean_conf": 0.7623
     }
    }
   },
   "guard": {
    "all": {
     "n": 2994,
     "accuracy": 0.8186,
     "nll": 0.4871,
     "ece": 0.0194,
     "mean_conf": 0.836
    },
    "by_type": {
     "choice": {
      "n": 2994,
      "accuracy": 0.8186,
      "nll": 0.4871,
      "ece": 0.0194,
      "mean_conf": 0.836
     },
     "noul": null,
     "score": null
    }
   },
   "val_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.796,
     "nll": 0.5269,
     "ece": 0.0264,
     "mean_conf": 0.8224
    },
    "by_type": {
     "choice": {
      "n": 3374,
      "accuracy": 0.7647,
      "nll": 0.6267,
      "ece": 0.0303,
      "mean_conf": 0.795
     },
     "noul": {
      "n": 1273,
      "accuracy": 0.901,
      "nll": 0.223,
      "ece": 0.0172,
      "mean_conf": 0.9146
     },
     "score": {
      "n": 353,
      "accuracy": 0.7167,
      "nll": 0.6687,
      "ece": 0.0363,
      "mean_conf": 0.7514
     }
    }
   },
   "val_teacher": {
    "all": {
     "n": 944,
     "accuracy": 0.8379,
     "nll": 0.433,
     "ece": 0.046,
     "mean_conf": 0.8722
    },
    "by_type": {
     "choice": {
      "n": 540,
      "accuracy": 0.8315,
      "nll": 0.4654,
      "ece": 0.0435,
      "mean_conf": 0.8558
     },
     "noul": {
      "n": 284,
      "accuracy": 0.8873,
      "nll": 0.3405,
      "ece": 0.0634,
      "mean_conf": 0.9331
     },
     "score": {
      "n": 120,
      "accuracy": 0.75,
      "nll": 0.5064,
      "ece": 0.0603,
      "mean_conf": 0.8016
     }
    }
   },
   "val_teacher2": {
    "all": {
     "n": 447,
     "accuracy": 0.8277,
     "nll": 0.4695,
     "ece": 0.0806,
     "mean_conf": 0.8852
    },
    "by_type": {
     "choice": {
      "n": 265,
      "accuracy": 0.8377,
      "nll": 0.4233,
      "ece": 0.0546,
      "mean_conf": 0.8868
     },
     "noul": {
      "n": 120,
      "accuracy": 0.85,
      "nll": 0.4441,
      "ece": 0.1059,
      "mean_conf": 0.9311
     },
     "score": {
      "n": 62,
      "accuracy": 0.7419,
      "nll": 0.7165,
      "ece": 0.1556,
      "mean_conf": 0.7893
     }
    }
   },
   "cal_human": {
    "all": {
     "n": 1595,
     "accuracy": 0.8564,
     "nll": 0.3893,
     "ece": 0.0195,
     "mean_conf": 0.8733
    },
    "by_type": {
     "choice": {
      "n": 1595,
      "accuracy": 0.8564,
      "nll": 0.3893,
      "ece": 0.0195,
      "mean_conf": 0.8733
     },
     "noul": null,
     "score": null
    }
   },
   "val_replay4": {
    "all": {
     "n": 1000,
     "accuracy": 0.889,
     "nll": 0.2843,
     "ece": 0.0287,
     "mean_conf": 0.8713
    },
    "by_type": {
     "choice": {
      "n": 1000,
      "accuracy": 0.889,
      "nll": 0.2843,
      "ece": 0.0287,
      "mean_conf": 0.8713
     },
     "noul": null,
     "score": null
    }
   }
  }
 },
 "fixtures": {
  "general_validation": {
   "this": {
    "rows": 847,
    "accuracy": 0.8500590318772137,
    "macro_accuracy": 0.8552272255409907,
    "groups": 65,
    "nll": 0.43237405419192004,
    "ece": 0.03505942904541031,
    "brier": 0.22353081852050935,
    "mean_confidence": 0.8244136475475226
   },
   "paired": {
    "cand4b_B4_e2_tt-cand4b_B4_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": 0.00012970495542040443,
      "ci95": [
       -0.00039863639304211823,
       0.0005942988193083918
      ]
     }
    },
    "cand4b_B4_e2_tt-v1_4b_p130": {
     "accuracy": {
      "delta": -0.010625737898465172,
      "ci95": [
       -0.021251475796930343,
       0.0
      ]
     },
     "nll": {
      "delta": 0.015307099810435664,
      "ci95": [
       0.0030912070436396634,
       0.028028207832146323
      ]
     }
    },
    "cand4b_B4_e2_tt-v2_4b_p130": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       -0.015348288075560802,
       0.015348288075560802
      ]
     },
     "nll": {
      "delta": 0.013765296214476044,
      "ci95": [
       -0.006225581100979938,
       0.0345539752628619
      ]
     }
    }
   }
  },
  "typesafe": {
   "this": {
    "rows": 102,
    "accuracy": 0.8431372549019608,
    "macro_accuracy": 0.8464696223316912,
    "groups": 4,
    "nll": 0.44128428424617644,
    "ece": 0.06267209380280735,
    "brier": 0.24132394423929263,
    "mean_confidence": 0.8996653276331285,
    "mean_total_variation_to_reference": 0.19246147475275616
   },
   "paired": {
    "cand4b_B4_e2_tt-cand4b_B4_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": -0.0017070072924830098,
      "ci95": [
       -0.003544773928864442,
       -8.629601238631066e-05
      ]
     }
    },
    "cand4b_B4_e2_tt-v1_4b_p130": {
     "accuracy": {
      "delta": 0.029411764705882353,
      "ci95": [
       -0.029411764705882353,
       0.08823529411764706
      ]
     },
     "nll": {
      "delta": -0.16956755666314244,
      "ci95": [
       -0.3236270765220259,
       -0.035752501984007506
      ]
     }
    },
    "cand4b_B4_e2_tt-v2_4b_p130": {
     "accuracy": {
      "delta": -0.0196078431372549,
      "ci95": [
       -0.0784313725490196,
       0.0392156862745098
      ]
     },
     "nll": {
      "delta": 0.03116252024698791,
      "ci95": [
       -0.06909971743191749,
       0.13434316002254298
      ]
     }
    }
   }
  },
  "openjev_external": {
   "this": {
    "rows": 5252,
    "accuracy": 0.6597486671744097,
    "macro_accuracy": 0.8063444444444444,
    "groups": 3,
    "nll": 0.8457289344223444,
    "ece": 0.1539857582574297,
    "brier": 0.5022800176356607,
    "mean_confidence": 0.8111537299704116
   },
   "paired": {
    "cand4b_B4_e2_tt-cand4b_B4_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": -0.003617502816960803,
      "ci95": [
       -0.003919285763621306,
       -0.00332506503930084
      ]
     }
    },
    "cand4b_B4_e2_tt-v1_4b_p130": {
     "accuracy": {
      "delta": 0.020944402132520943,
      "ci95": [
       0.012376237623762377,
       0.029322162985529324
      ]
     },
     "nll": {
      "delta": -0.047333755296254304,
      "ci95": [
       -0.059559262901540976,
       -0.035193633735333484
      ]
     }
    },
    "cand4b_B4_e2_tt-v2_4b_p130": {
     "accuracy": {
      "delta": -0.007425742574257425,
      "ci95": [
       -0.015993907083015995,
       0.0017136329017517135
      ]
     },
     "nll": {
      "delta": 0.05671352997929895,
      "ci95": [
       0.04359541016972144,
       0.07033520922345497
      ]
     }
    }
   }
  },
  "mind2web": {
   "this": {
    "rows": 1770,
    "accuracy": 0.8751412429378531,
    "macro_accuracy": 0.8773196126201471,
    "groups": 3,
    "nll": 0.37523612083453944,
    "ece": 0.016833950116135952,
    "brier": 0.18402882486455008,
    "mean_confidence": 0.8624463401776923
   },
   "paired": {
    "cand4b_B4_e2_tt-cand4b_B4_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": 0.00026909733615400604,
      "ci95": [
       -4.997823592689937e-05,
       0.0006061414472582659
      ]
     }
    },
    "cand4b_B4_e2_tt-v1_4b_p130": {
     "accuracy": {
      "delta": -0.00847457627118644,
      "ci95": [
       -0.01694915254237288,
       0.0
      ]
     },
     "nll": {
      "delta": 0.00924747244030173,
      "ci95": [
       -0.004092843962589085,
       0.022512164121517977
      ]
     }
    },
    "cand4b_B4_e2_tt-v2_4b_p130": {
     "accuracy": {
      "delta": 0.0011299435028248588,
      "ci95": [
       -0.00847457627118644,
       0.011299435028248588
      ]
     },
     "nll": {
      "delta": -0.018244809747118285,
      "ci95": [
       -0.032040489157299956,
       -0.005027723688309381
      ]
     }
    }
   }
  }
 },
 "games": {
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|all": {
   "n": 234,
   "a": 0.2692307692307692,
   "b": 0.2692307692307692,
   "delta": 0.0,
   "ci95": [
    -0.004273504273504274,
    0.004273504273504274
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|bag_game": {
   "n": 64,
   "a": 0.51953125,
   "b": 0.51953125,
   "delta": 0.0,
   "ci95": [
    -0.01171875,
    0.01171875
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|stochastic_grid": {
   "n": 64,
   "a": 0.16796875,
   "b": 0.16796875,
   "delta": 0.0,
   "ci95": [
    -0.01171875,
    0.01171875
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|tic_tac_toe": {
   "n": 74,
   "a": 0.2533783783783784,
   "b": 0.2533783783783784,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|exact_minesweeper": {
   "n": 32,
   "a": 0.0078125,
   "b": 0.0078125,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|all": {
   "n": 234,
   "a": 0.25213675213675213,
   "b": 0.25213675213675213,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|bag_game": {
   "n": 64,
   "a": 0.53125,
   "b": 0.53125,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|stochastic_grid": {
   "n": 64,
   "a": 0.109375,
   "b": 0.109375,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|tic_tac_toe": {
   "n": 74,
   "a": 0.24324324324324326,
   "b": 0.24324324324324326,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|exact_minesweeper": {
   "n": 32,
   "a": 0.0,
   "b": 0.0,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|all": {
   "n": 234,
   "a": 0.2692307692307692,
   "b": 0.2777777777777778,
   "delta": -0.008547008547008548,
   "ci95": [
    -0.02564102564102564,
    0.009615384615384616
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|bag_game": {
   "n": 64,
   "a": 0.51953125,
   "b": 0.56640625,
   "delta": -0.046875,
   "ci95": [
    -0.08984375,
    -0.00390625
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|stochastic_grid": {
   "n": 64,
   "a": 0.16796875,
   "b": 0.1640625,
   "delta": 0.00390625,
   "ci95": [
    -0.02734375,
    0.03515625
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|tic_tac_toe": {
   "n": 74,
   "a": 0.2533783783783784,
   "b": 0.24324324324324326,
   "delta": 0.010135135135135136,
   "ci95": [
    -0.013513513513513514,
    0.037162162162162164
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|exact_minesweeper": {
   "n": 32,
   "a": 0.0078125,
   "b": 0.0078125,
   "delta": 0.0,
   "ci95": [
    -0.0234375,
    0.0234375
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|all": {
   "n": 234,
   "a": 0.25213675213675213,
   "b": 0.2905982905982906,
   "delta": -0.038461538461538464,
   "ci95": [
    -0.07264957264957266,
    -0.004273504273504274
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|bag_game": {
   "n": 64,
   "a": 0.53125,
   "b": 0.625,
   "delta": -0.09375,
   "ci95": [
    -0.171875,
    -0.03125
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|stochastic_grid": {
   "n": 64,
   "a": 0.109375,
   "b": 0.125,
   "delta": -0.015625,
   "ci95": [
    -0.07851562499999987,
    0.046875
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|tic_tac_toe": {
   "n": 74,
   "a": 0.24324324324324326,
   "b": 0.2702702702702703,
   "delta": -0.02702702702702703,
   "ci95": [
    -0.0945945945945946,
    0.04054054054054054
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|exact_minesweeper": {
   "n": 32,
   "a": 0.0,
   "b": 0.0,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|all": {
   "n": 234,
   "a": 0.2692307692307692,
   "b": 0.22435897435897437,
   "delta": 0.04487179487179487,
   "ci95": [
    0.024572649572649572,
    0.06623931623931624
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|bag_game": {
   "n": 64,
   "a": 0.51953125,
   "b": 0.37890625,
   "delta": 0.140625,
   "ci95": [
    0.08203125,
    0.19921875
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|stochastic_grid": {
   "n": 64,
   "a": 0.16796875,
   "b": 0.16015625,
   "delta": 0.0078125,
   "ci95": [
    -0.0234375,
    0.0390625
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|tic_tac_toe": {
   "n": 74,
   "a": 0.2533783783783784,
   "b": 0.24324324324324326,
   "delta": 0.010135135135135136,
   "ci95": [
    -0.010135135135135136,
    0.033783783783783786
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|exact_minesweeper": {
   "n": 32,
   "a": 0.0078125,
   "b": 0.0,
   "delta": 0.0078125,
   "ci95": [
    0.0,
    0.0234375
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|all": {
   "n": 234,
   "a": 0.25213675213675213,
   "b": 0.2777777777777778,
   "delta": -0.02564102564102564,
   "ci95": [
    -0.0641025641025641,
    0.01282051282051282
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|bag_game": {
   "n": 64,
   "a": 0.53125,
   "b": 0.484375,
   "delta": 0.046875,
   "ci95": [
    -0.03125,
    0.125
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|stochastic_grid": {
   "n": 64,
   "a": 0.109375,
   "b": 0.1875,
   "delta": -0.078125,
   "ci95": [
    -0.15625,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|tic_tac_toe": {
   "n": 74,
   "a": 0.24324324324324326,
   "b": 0.2972972972972973,
   "delta": -0.05405405405405406,
   "ci95": [
    -0.12162162162162163,
    0.02702702702702703
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|exact_minesweeper": {
   "n": 32,
   "a": 0.0,
   "b": 0.0,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  }
 },
 "browser": {
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|all": {
   "n": 176,
   "a": 0.9318181818181818,
   "b": 0.9318181818181818,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|rewarded": {
   "n": 128,
   "a": 0.953125,
   "b": 0.953125,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.875,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|all": {
   "n": 176,
   "a": 0.9375,
   "b": 0.9375,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|rewarded": {
   "n": 128,
   "a": 0.9609375,
   "b": 0.9609375,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.875,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|all": {
   "n": 176,
   "a": 0.9318181818181818,
   "b": 0.9090909090909091,
   "delta": 0.022727272727272728,
   "ci95": [
    -0.017045454545454544,
    0.0625
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|rewarded": {
   "n": 128,
   "a": 0.953125,
   "b": 0.9609375,
   "delta": -0.0078125,
   "ci95": [
    -0.0390625,
    0.0234375
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|sampled|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.7708333333333334,
   "delta": 0.10416666666666667,
   "ci95": [
    0.0,
    0.20833333333333334
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|all": {
   "n": 176,
   "a": 0.9375,
   "b": 0.9147727272727273,
   "delta": 0.022727272727272728,
   "ci95": [
    -0.005681818181818182,
    0.056818181818181816
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|rewarded": {
   "n": 128,
   "a": 0.9609375,
   "b": 0.9765625,
   "delta": -0.015625,
   "ci95": [
    -0.0390625,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v1_4b_p130|greedy|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.75,
   "delta": 0.125,
   "ci95": [
    0.041666666666666664,
    0.22916666666666666
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|all": {
   "n": 176,
   "a": 0.9318181818181818,
   "b": 0.8806818181818182,
   "delta": 0.05113636363636364,
   "ci95": [
    0.005681818181818182,
    0.09659090909090909
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|rewarded": {
   "n": 128,
   "a": 0.953125,
   "b": 0.9296875,
   "delta": 0.0234375,
   "ci95": [
    -0.0234375,
    0.0703125
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|sampled|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.75,
   "delta": 0.125,
   "ci95": [
    0.0,
    0.25
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|all": {
   "n": 176,
   "a": 0.9375,
   "b": 0.9261363636363636,
   "delta": 0.011363636363636364,
   "ci95": [
    -0.017045454545454544,
    0.045454545454545456
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|rewarded": {
   "n": 128,
   "a": 0.9609375,
   "b": 0.96875,
   "delta": -0.0078125,
   "ci95": [
    -0.0234375,
    0.0
   ]
  },
  "cand4b_B4_e2_tt-v2_4b_p130|greedy|held_out": {
   "n": 48,
   "a": 0.875,
   "b": 0.8125,
   "delta": 0.0625,
   "ci95": [
    -0.041666666666666664,
    0.16666666666666666
   ]
  }
 },
 "text_games_greedy": {
  "pong": -5.0,
  "freeway": 0.0,
  "breakout": 35.0,
  "frozenlake": 0.0,
  "cliffwalking": -13.0,
  "blackjack": -0.6,
  "minigrid_empty": 0.0,
  "minigrid_lavagap": 0.0,
  "minigrid_doorkey": 0.0,
  "babyai_goto": 0.1915625
 },
 "probes": {
  "router_n": 31,
  "router_tier": 0.968,
  "needs_live_data": 0.839,
  "commands_n": 45,
  "command_risk": 0.889,
  "destructive_called_safe": 0,
  "outside_project": 0.933,
  "browser_element": 0.938,
  "browser_action": 0.75,
  "batteries": {
   "generic": 1.0,
   "specific": 1.0,
   "catchall": 0.95,
   "noul_acc": 1.0,
   "noul_brier": 0.01,
   "n_choice": 60,
   "n_noul": 49,
   "abstain_battery": "8/8"
  }
 },
 "issue9": {
  "c_1": {
   "pred": 26,
   "gold": 6,
   "ok": false,
   "p_gold": 0.198,
   "p_pred": 0.764,
   "T": 1.099
  },
  "c_2": {
   "pred": 20,
   "gold": 20,
   "ok": true,
   "p_gold": 0.409,
   "p_pred": 0.409,
   "T": 1.099
  },
  "b_10": {
   "pred": 0,
   "gold": 0,
   "ok": true,
   "p_gold": 0.732,
   "p_pred": 0.732,
   "T": 1.099
  },
  "a2_4": {
   "pred": 4,
   "gold": 4,
   "ok": true,
   "p_gold": 0.577,
   "p_pred": 0.577,
   "T": 1.099
  }
 },
 "bespoke": {
  "vitaminc-dev": {
   "n": 599,
   "type": "choice",
   "acc": 0.7429,
   "ece": 0.1456,
   "brier": 0.3945,
   "nll": 0.7789,
   "trained": false,
   "sec": 11.8
  },
  "massive-en-US": {
   "n": 350,
   "type": "choice",
   "acc": 0.8629,
   "ece": 0.0411,
   "brier": 0.1936,
   "nll": 0.4173,
   "trained": true,
   "sec": 6.2
  },
  "massive-de-DE": {
   "n": 350,
   "type": "choice",
   "acc": 0.84,
   "ece": 0.0297,
   "brier": 0.2269,
   "nll": 0.4947,
   "trained": false,
   "sec": 4.5
  },
  "boolq": {
   "n": 300,
   "type": "noul",
   "acc": 0.8667,
   "ece": 0.0319,
   "brier": 0.1862,
   "nll": 0.3007,
   "trained": true,
   "sec": 6.2
  },
  "squad2": {
   "n": 299,
   "type": "noul",
   "acc": 0.7893,
   "ece": 0.0441,
   "brier": 0.2955,
   "nll": 0.452,
   "trained": false,
   "sec": 7.2
  },
  "paws": {
   "n": 250,
   "type": "noul",
   "acc": 0.764,
   "ece": 0.1238,
   "brier": 0.3398,
   "nll": 0.5538,
   "trained": true,
   "sec": 3.0
  },
  "multinli": {
   "n": 299,
   "type": "choice",
   "acc": 0.9331,
   "ece": 0.0103,
   "brier": 0.1089,
   "nll": 0.2099,
   "trained": true,
   "sec": 3.1
  },
  "civil_comments": {
   "n": 300,
   "type": "noul",
   "acc": 0.8733,
   "ece": 0.0566,
   "brier": 0.1917,
   "nll": 0.3199,
   "trained": true,
   "sec": 3.2
  },
  "aegis2": {
   "n": 250,
   "type": "noul",
   "acc": 0.804,
   "ece": 0.0467,
   "brier": 0.2797,
   "nll": 0.4527,
   "trained": false,
   "sec": 6.1
  },
  "helpsteer2": {
   "n": 249,
   "type": "score",
   "acc": 0.4498,
   "ece": 0.142,
   "brier": 0.6713,
   "nll": 1.2502,
   "trained": true,
   "sec": 25.8,
   "score_mae": 0.797
  },
  "summeval-relevance": {
   "n": 240,
   "type": "score",
   "acc": 0.3958,
   "ece": 0.0696,
   "brier": 0.6844,
   "nll": 1.2541,
   "trained": false,
   "sec": 20.1,
   "score_mae": 0.6172
  },
  "summeval-consistency": {
   "n": 144,
   "type": "score",
   "acc": 0.7986,
   "ece": 0.3095,
   "brier": 0.4741,
   "nll": 0.9007,
   "trained": false,
   "sec": 12.2,
   "score_mae": 0.7974
  },
  "pubmedqa": {
   "n": 250,
   "type": "choice",
   "acc": 0.708,
   "ece": 0.0863,
   "brier": 0.4012,
   "nll": 0.7205,
   "trained": true,
   "sec": 3.8
  },
  "_macro": 0.756,
  "_micro": 0.7652,
  "_macro_untrained": 0.7284
 },
 "jevbench_public": {
  "easy": {
   "n": 48,
   "accuracy": 1.0,
   "invalid": 0,
   "ece": 0.0027979166666670663,
   "by_family": {
    "extraction": {
     "n": 12,
     "accuracy": 1.0
    },
    "fact": {
     "n": 12,
     "accuracy": 1.0
    },
    "intent": {
     "n": 12,
     "accuracy": 1.0
    },
    "tool_selection": {
     "n": 12,
     "accuracy": 1.0
    }
   }
  },
  "standard": {
   "n": 72,
   "accuracy": 0.9861111111111112,
   "invalid": 0,
   "ece": 0.0753152777777778,
   "by_family": {
    "adequacy": {
     "n": 12,
     "accuracy": 0.9166666666666666
    },
    "extraction": {
     "n": 12,
     "accuracy": 1.0
    },
    "intent": {
     "n": 12,
     "accuracy": 1.0
    },
    "ordinal": {
     "n": 12,
     "accuracy": 1.0
    },
    "policy": {
     "n": 12,
     "accuracy": 1.0
    },
    "routing": {
     "n": 12,
     "accuracy": 1.0
    }
   }
  },
  "hard": {
   "n": 111,
   "accuracy": 0.6486486486486487,
   "invalid": 0,
   "ece": 0.18370720720720732,
   "by_family": {
    "adversarial": {
     "n": 6,
     "accuracy": 1.0
    },
    "ambiguous": {
     "n": 7,
     "accuracy": 0.7142857142857143
    },
    "judge_hard": {
     "n": 17,
     "accuracy": 0.5294117647058824
    },
    "long_policy": {
     "n": 19,
     "accuracy": 0.47368421052631576
    },
    "multi_hop": {
     "n": 18,
     "accuracy": 0.8333333333333334
    },
    "probability": {
     "n": 10,
     "accuracy": 0.8
    },
    "routing_hard": {
     "n": 5,
     "accuracy": 1.0
    },
    "temporal_numeric": {
     "n": 15,
     "accuracy": 0.3333333333333333
    },
    "tradeoff": {
     "n": 6,
     "accuracy": 0.3333333333333333
    },
    "trap": {
     "n": 8,
     "accuracy": 1.0
    }
   }
  }
 },
 "references": {
  "v1_4b": {
   "reg_in": 0.8338,
   "reg_out": 0.7876,
   "reg_in_nll": 0.4044,
   "reg_out_nll": 0.5583,
   "reg_in_ece": 0.0269,
   "reg_out_ece": 0.0708
  },
  "v2_4b": {
   "reg_in": 0.8241,
   "reg_out": 0.7788,
   "reg_in_nll": 0.4411,
   "reg_out_nll": 0.5663,
   "reg_in_ece": 0.0413,
   "reg_out_ece": 0.0803
  }
 },
 "speed": "not measured again: the architecture and size equal the parent's, and the temperature map does not change the computation"
}