{
  "baseline": {
    "typed-decisions": 0.767,
    "ag-news": 0.9466666666666667,
    "boolq": 0.82,
    "emotion": 0.5816666666666667,
    "prompt-injections": 0.6724137931034483,
    "sst5": 0.48,
    "massive-intent.en": 0.81,
    "xnli.en": 0.8633333333333333
  },
  "evaluation_reference": {
    "name": "Official Laya-TD",
    "model_id": "convaiinnovations/laya-typed-decisions",
    "revision": "1a793eb568e6718f15941d08f85432581df534e3",
    "accuracy": {
      "typed-decisions": 0.767,
      "ag-news": 0.9466666666666667,
      "boolq": 0.82,
      "emotion": 0.5816666666666667,
      "prompt-injections": 0.6724137931034483,
      "sst5": 0.48,
      "massive-intent.en": 0.81,
      "xnli.en": 0.8633333333333333
    },
    "benchmark_cases_sha256": "10fb671245cdac1ee09e9c4d2b9b09829fbf1abd06e0954fddee0b9ec8519854",
    "predictions_sha256": "15952331cc2eb4732075b326f6ff764129f84cbbac09529b30dcf84842df67d3",
    "environment": {
      "python": "3.12.3",
      "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39",
      "processor": "x86_64",
      "packages": {
        "laya": "0.3.20",
        "torch": "2.13.0+cu130",
        "transformers": "5.17.0",
        "datasets": "5.0.1",
        "huggingface-hub": "1.33.0"
      },
      "harness": "0.1.0"
    },
    "model_metadata": {
      "max_len": 1024,
      "head_max_len": 256,
      "native_prediction": true,
      "device": "cuda:1",
      "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition",
      "dtype": "torch.bfloat16",
      "torch_version": "2.13.0+cu130",
      "transformers_version": "5.17.0",
      "laya_version": "0.3.20",
      "parameters": 421293827,
      "temperature": [
        1.0148024559020996,
        1.0374259948730469,
        1.0575125217437744
      ],
      "temperature_by_options": {
        "choice:3-5": 1.7601518630981445,
        "choice:6-10": 1.0000158548355103,
        "score:3-5": 1.2514300346374512,
        "noul:2": 1.983399510383606,
        "choice:11+": 0.5,
        "choice:2": 1.9063563346862793
      },
      "reproducibility": {
        "deterministic_algorithms": true,
        "warn_only": false,
        "cublas_workspace_config": ":4096:8",
        "python_hash_seed": "0",
        "cudnn_benchmark": false,
        "cudnn_deterministic": true,
        "cudnn_allow_tf32": false,
        "float32_matmul_precision": "highest",
        "cudnn_sdp_enabled": false,
        "flash_sdp_enabled": true,
        "mem_efficient_sdp_enabled": true,
        "math_sdp_enabled": true,
        "cuda_version": "13.0",
        "cudnn_version": 92000,
        "scope": "fixed hardware/runtime; cross-platform bitwise agreement is not promised"
      },
      "interface": null
    },
    "model_weights_sha256": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e",
    "matched_runtime_and_token_budgets": true,
    "independent_scoring_matches": true
  },
  "training_base_control": {
    "model_id": "convaiinnovations/laya",
    "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
    "accuracy": {
      "typed-decisions": 0.3625,
      "ag-news": 0.9483333333333334,
      "boolq": 0.83,
      "emotion": 0.5733333333333334,
      "prompt-injections": 0.6896551724137931,
      "sst5": 0.37,
      "massive-intent.en": 0.7866666666666666,
      "xnli.en": 0.86
    }
  },
  "runs": {
    "0": {
      "seed": 0,
      "selected_epoch": 11,
      "elapsed_s": 612.5042744020466,
      "initial_validation": {
        "soft_cross_entropy": 1.544869564374288,
        "accuracy": 0.36833333333333335,
        "brier_soft": 0.43051501592000324,
        "decisions": 600
      },
      "selected_validation": {
        "soft_cross_entropy": 0.8771124110619227,
        "accuracy": 0.7116666666666667,
        "brier_soft": 0.0828241604194045,
        "decisions": 600
      },
      "stopping": {
        "reason": "early_stopping",
        "epochs_completed": 14,
        "patience": 3,
        "min_epochs": 10,
        "epochs_without_improvement": 3,
        "metric": "validation.soft_cross_entropy",
        "min_delta": 0.0
      },
      "frozen_parameters_unchanged": true,
      "suites": {
        "typed-decisions": {
          "attempted": 2000,
          "valid": 2000,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7425,
          "accuracy_valid": 0.7425,
          "ece_top_label": 0.19222331993705236,
          "mean_confidence": 0.5502766800629476,
          "brier_hard": 0.41714854154295294,
          "brier_hard_n": 2000,
          "nll_hard": 0.7346455747590726,
          "nll_hard_n": 2000,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 2000,
          "soft_accuracy": 0.4642362342348646,
          "soft_accuracy_n": 2000,
          "brier_soft": 0.07282817778125876,
          "brier_soft_n": 2000,
          "kl_gold_to_prediction": 0.13635658195747785,
          "kl_gold_to_prediction_n": 2000,
          "total_variation": 0.18626503447270873,
          "total_variation_n": 2000,
          "score_mae": 0.2670681801272688,
          "score_mae_n": 800,
          "within_one_level": 0.98125,
          "within_one_level_n": 800,
          "macro_f1": 0.6211545025911077
        },
        "ag-news": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.9416666666666667,
          "accuracy_valid": 0.9416666666666667,
          "ece_top_label": 0.032780915974200804,
          "mean_confidence": 0.9120336837609119,
          "brier_hard": 0.0911117485859852,
          "brier_hard_n": 600,
          "nll_hard": 0.18384446182116462,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.9391643390047506
        },
        "emotion": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.585,
          "accuracy_valid": 0.585,
          "ece_top_label": 0.3290210797041948,
          "mean_confidence": 0.9044507912979094,
          "brier_hard": 0.7352614358764159,
          "brier_hard_n": 600,
          "nll_hard": 2.325969372091789,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.01,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.4846329953255675
        },
        "boolq": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8083333333333333,
          "accuracy_valid": 0.8083333333333333,
          "ece_top_label": 0.09874549999999996,
          "mean_confidence": 0.9070788333333333,
          "brier_hard": 0.2864656637,
          "brier_hard_n": 600,
          "nll_hard": 0.45967158478397707,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.7946275764565816
        },
        "sst5": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.38166666666666665,
          "accuracy_valid": 0.38166666666666665,
          "ece_top_label": 0.23314279850468647,
          "mean_confidence": 0.614809465171353,
          "brier_hard": 0.7997842513426261,
          "brier_hard_n": 600,
          "nll_hard": 1.7175170777887119,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "score_mae": 0.9144952235712186,
          "score_mae_n": 600,
          "within_one_level": 0.6466666666666666,
          "within_one_level_n": 600,
          "macro_f1": 0.3319436236284081
        },
        "prompt-injections": {
          "attempted": 116,
          "valid": 116,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.6982758620689655,
          "accuracy_valid": 0.6982758620689655,
          "ece_top_label": 0.25387413793103447,
          "mean_confidence": 0.95215,
          "brier_hard": 0.5115509806896552,
          "brier_hard_n": 116,
          "nll_hard": 2.710472238032788,
          "nll_hard_n": 116,
          "zero_probability_gold": 0.06896551724137931,
          "zero_probability_gold_n": 116,
          "macro_f1": 0.6809931641392315
        },
        "massive-intent.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.79,
          "accuracy_valid": 0.79,
          "ece_top_label": 0.16259349215790048,
          "mean_confidence": 0.9477817275562629,
          "brier_hard": 0.3655905743529746,
          "brier_hard_n": 300,
          "nll_hard": 2.4460407722040176,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.06,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.5695724937908466
        },
        "xnli.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.89,
          "accuracy_valid": 0.89,
          "ece_top_label": 0.05910627327283609,
          "mean_confidence": 0.9123050519493905,
          "brier_hard": 0.18251872229879984,
          "brier_hard_n": 300,
          "nll_hard": 0.33404125735595414,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.8905240397272149
        }
      }
    },
    "1": {
      "seed": 1,
      "selected_epoch": 9,
      "elapsed_s": 541.4585086829611,
      "initial_validation": {
        "soft_cross_entropy": 1.544869564374288,
        "accuracy": 0.36833333333333335,
        "brier_soft": 0.43051501592000324,
        "decisions": 600
      },
      "selected_validation": {
        "soft_cross_entropy": 0.8583925066391627,
        "accuracy": 0.7466666666666667,
        "brier_soft": 0.07437195796364297,
        "decisions": 600
      },
      "stopping": {
        "reason": "early_stopping",
        "epochs_completed": 12,
        "patience": 3,
        "min_epochs": 10,
        "epochs_without_improvement": 3,
        "metric": "validation.soft_cross_entropy",
        "min_delta": 0.0
      },
      "frozen_parameters_unchanged": true,
      "suites": {
        "typed-decisions": {
          "attempted": 2000,
          "valid": 2000,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7565,
          "accuracy_valid": 0.7565,
          "ece_top_label": 0.19864268503011012,
          "mean_confidence": 0.5578573149698899,
          "brier_hard": 0.40482086772099385,
          "brier_hard_n": 2000,
          "nll_hard": 0.7162594531200565,
          "nll_hard_n": 2000,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 2000,
          "soft_accuracy": 0.469737413364262,
          "soft_accuracy_n": 2000,
          "brier_soft": 0.0659469654317038,
          "brier_soft_n": 2000,
          "kl_gold_to_prediction": 0.12376910578634269,
          "kl_gold_to_prediction_n": 2000,
          "total_variation": 0.17750424596946363,
          "total_variation_n": 2000,
          "score_mae": 0.24687533223920302,
          "score_mae_n": 800,
          "within_one_level": 0.98625,
          "within_one_level_n": 800,
          "macro_f1": 0.6300073343403317
        },
        "ag-news": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.9483333333333334,
          "accuracy_valid": 0.9483333333333334,
          "ece_top_label": 0.03462398162033311,
          "mean_confidence": 0.9188607247165506,
          "brier_hard": 0.08433672632610516,
          "brier_hard_n": 600,
          "nll_hard": 0.17417200848974151,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.9456145576151846
        },
        "emotion": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.5716666666666667,
          "accuracy_valid": 0.5716666666666667,
          "ece_top_label": 0.33449032496578573,
          "mean_confidence": 0.90098409027152,
          "brier_hard": 0.7328256742449804,
          "brier_hard_n": 600,
          "nll_hard": 2.388523752936007,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.013333333333333334,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.4746076611364094
        },
        "boolq": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8266666666666667,
          "accuracy_valid": 0.8266666666666667,
          "ece_top_label": 0.08269850000000001,
          "mean_confidence": 0.9093651666666667,
          "brier_hard": 0.26852242536666665,
          "brier_hard_n": 600,
          "nll_hard": 0.44571515698724307,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.8140998140998141
        },
        "sst5": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.4483333333333333,
          "accuracy_valid": 0.4483333333333333,
          "ece_top_label": 0.1552169576555402,
          "mean_confidence": 0.5963372951766569,
          "brier_hard": 0.715830700927659,
          "brier_hard_n": 600,
          "nll_hard": 1.4312799777899081,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "score_mae": 0.763809024216731,
          "score_mae_n": 600,
          "within_one_level": 0.7216666666666667,
          "within_one_level_n": 600,
          "macro_f1": 0.4241661418376961
        },
        "prompt-injections": {
          "attempted": 116,
          "valid": 116,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7844827586206896,
          "accuracy_valid": 0.7844827586206896,
          "ece_top_label": 0.17151379310344822,
          "mean_confidence": 0.9531103448275862,
          "brier_hard": 0.36615182827586207,
          "brier_hard_n": 116,
          "nll_hard": 1.9254142815070245,
          "nll_hard_n": 116,
          "zero_probability_gold": 0.05172413793103448,
          "zero_probability_gold_n": 116,
          "macro_f1": 0.7785414280259642
        },
        "massive-intent.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7566666666666667,
          "accuracy_valid": 0.7566666666666667,
          "ece_top_label": 0.19351909840865464,
          "mean_confidence": 0.9501857650753213,
          "brier_hard": 0.4290677996735415,
          "brier_hard_n": 300,
          "nll_hard": 3.2153508464750185,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.08666666666666667,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.4943833232448793
        },
        "xnli.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.87,
          "accuracy_valid": 0.87,
          "ece_top_label": 0.07153730747712309,
          "mean_confidence": 0.9147377091848204,
          "brier_hard": 0.22017864577608134,
          "brier_hard_n": 300,
          "nll_hard": 0.4059236764304257,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.8710695955086495
        }
      }
    },
    "2": {
      "seed": 2,
      "selected_epoch": 13,
      "elapsed_s": 699.7859417049913,
      "initial_validation": {
        "soft_cross_entropy": 1.544869564374288,
        "accuracy": 0.36833333333333335,
        "brier_soft": 0.43051501592000324,
        "decisions": 600
      },
      "selected_validation": {
        "soft_cross_entropy": 0.8543428432941437,
        "accuracy": 0.7583333333333333,
        "brier_soft": 0.07187043125430743,
        "decisions": 600
      },
      "stopping": {
        "reason": "early_stopping",
        "epochs_completed": 16,
        "patience": 3,
        "min_epochs": 10,
        "epochs_without_improvement": 3,
        "metric": "validation.soft_cross_entropy",
        "min_delta": 0.0
      },
      "frozen_parameters_unchanged": true,
      "suites": {
        "typed-decisions": {
          "attempted": 2000,
          "valid": 2000,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.745,
          "accuracy_valid": 0.745,
          "ece_top_label": 0.18594273050266366,
          "mean_confidence": 0.5591527012331047,
          "brier_hard": 0.40870440390133495,
          "brier_hard_n": 2000,
          "nll_hard": 0.7205417712302097,
          "nll_hard_n": 2000,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 2000,
          "soft_accuracy": 0.46875735649306255,
          "soft_accuracy_n": 2000,
          "brier_soft": 0.06782387317668824,
          "brier_soft_n": 2000,
          "kl_gold_to_prediction": 0.12687181845569698,
          "kl_gold_to_prediction_n": 2000,
          "total_variation": 0.17935579359201664,
          "total_variation_n": 2000,
          "score_mae": 0.25587944352517583,
          "score_mae_n": 800,
          "within_one_level": 0.99,
          "within_one_level_n": 800,
          "macro_f1": 0.608344198168488
        },
        "ag-news": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.945,
          "accuracy_valid": 0.945,
          "ece_top_label": 0.03201179295141958,
          "mean_confidence": 0.9180146184255912,
          "brier_hard": 0.08699114920264774,
          "brier_hard_n": 600,
          "nll_hard": 0.17458965637122142,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.9421288699146882
        },
        "emotion": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.56,
          "accuracy_valid": 0.56,
          "ece_top_label": 0.3501647005433899,
          "mean_confidence": 0.9077857005433899,
          "brier_hard": 0.7581475668473461,
          "brier_hard_n": 600,
          "nll_hard": 2.427279492608694,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.013333333333333334,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.4667304743466239
        },
        "boolq": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8183333333333334,
          "accuracy_valid": 0.8183333333333334,
          "ece_top_label": 0.08817149999999999,
          "mean_confidence": 0.9065048333333333,
          "brier_hard": 0.2762812369666667,
          "brier_hard_n": 600,
          "nll_hard": 0.453869636876017,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.8046122269724754
        },
        "sst5": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.3983333333333333,
          "accuracy_valid": 0.3983333333333333,
          "ece_top_label": 0.21926419101950614,
          "mean_confidence": 0.6138282869290971,
          "brier_hard": 0.7663539453668811,
          "brier_hard_n": 600,
          "nll_hard": 1.5341982352412973,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "score_mae": 0.7954480085445932,
          "score_mae_n": 600,
          "within_one_level": 0.695,
          "within_one_level_n": 600,
          "macro_f1": 0.3618869575527743
        },
        "prompt-injections": {
          "attempted": 116,
          "valid": 116,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7155172413793104,
          "accuracy_valid": 0.7155172413793104,
          "ece_top_label": 0.2621448275862069,
          "mean_confidence": 0.9488327586206896,
          "brier_hard": 0.4988274203448276,
          "brier_hard_n": 116,
          "nll_hard": 2.61023697496577,
          "nll_hard_n": 116,
          "zero_probability_gold": 0.06896551724137931,
          "zero_probability_gold_n": 116,
          "macro_f1": 0.6992221261884184
        },
        "massive-intent.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.75,
          "accuracy_valid": 0.75,
          "ece_top_label": 0.20336797564979028,
          "mean_confidence": 0.9533679756497903,
          "brier_hard": 0.4358421604675745,
          "brier_hard_n": 300,
          "nll_hard": 3.1322756175288426,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.08,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.5034639202079357
        },
        "xnli.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8733333333333333,
          "accuracy_valid": 0.8733333333333333,
          "ece_top_label": 0.05108680216338136,
          "mean_confidence": 0.914837261771426,
          "brier_hard": 0.20561652153986584,
          "brier_hard_n": 300,
          "nll_hard": 0.3742216981370551,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.8742255731346232
        }
      }
    },
    "3": {
      "seed": 3,
      "selected_epoch": 8,
      "elapsed_s": 495.011001034989,
      "initial_validation": {
        "soft_cross_entropy": 1.544869564374288,
        "accuracy": 0.36833333333333335,
        "brier_soft": 0.43051501592000324,
        "decisions": 600
      },
      "selected_validation": {
        "soft_cross_entropy": 0.8545633804798126,
        "accuracy": 0.77,
        "brier_soft": 0.06962224062221746,
        "decisions": 600
      },
      "stopping": {
        "reason": "early_stopping",
        "epochs_completed": 11,
        "patience": 3,
        "min_epochs": 10,
        "epochs_without_improvement": 3,
        "metric": "validation.soft_cross_entropy",
        "min_delta": 0.0
      },
      "frozen_parameters_unchanged": true,
      "suites": {
        "typed-decisions": {
          "attempted": 2000,
          "valid": 2000,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7655,
          "accuracy_valid": 0.7655,
          "ece_top_label": 0.2096452515650509,
          "mean_confidence": 0.5558547484349491,
          "brier_hard": 0.4011842616732685,
          "brier_hard_n": 2000,
          "nll_hard": 0.7113399953872087,
          "nll_hard_n": 2000,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 2000,
          "soft_accuracy": 0.4683026193430771,
          "soft_accuracy_n": 2000,
          "brier_soft": 0.06585425455571205,
          "brier_soft_n": 2000,
          "kl_gold_to_prediction": 0.12507067261764912,
          "kl_gold_to_prediction_n": 2000,
          "total_variation": 0.17848467876854548,
          "total_variation_n": 2000,
          "score_mae": 0.24395106365305969,
          "score_mae_n": 800,
          "within_one_level": 0.98375,
          "within_one_level_n": 800,
          "macro_f1": 0.6399493527359326
        },
        "ag-news": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.9366666666666666,
          "accuracy_valid": 0.9366666666666666,
          "ece_top_label": 0.0312034834128748,
          "mean_confidence": 0.914535932360556,
          "brier_hard": 0.09381807669545053,
          "brier_hard_n": 600,
          "nll_hard": 0.1866731647377733,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.9336084571946038
        },
        "emotion": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.565,
          "accuracy_valid": 0.565,
          "ece_top_label": 0.3370553794856075,
          "mean_confidence": 0.8987647229086076,
          "brier_hard": 0.7489678137408704,
          "brier_hard_n": 600,
          "nll_hard": 2.3318777367931305,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.011666666666666667,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.4760859469034663
        },
        "boolq": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.82,
          "accuracy_valid": 0.82,
          "ece_top_label": 0.09564866666666666,
          "mean_confidence": 0.9069513333333333,
          "brier_hard": 0.2824305872,
          "brier_hard_n": 600,
          "nll_hard": 0.4619444556248629,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.8069498069498069
        },
        "sst5": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.4116666666666667,
          "accuracy_valid": 0.4116666666666667,
          "ece_top_label": 0.19022023187751014,
          "mean_confidence": 0.6018868985441768,
          "brier_hard": 0.7571439809814903,
          "brier_hard_n": 600,
          "nll_hard": 1.5683213482322946,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "score_mae": 0.8321864596740475,
          "score_mae_n": 600,
          "within_one_level": 0.6783333333333333,
          "within_one_level_n": 600,
          "macro_f1": 0.3816445513227996
        },
        "prompt-injections": {
          "attempted": 116,
          "valid": 116,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7844827586206896,
          "accuracy_valid": 0.7844827586206896,
          "ece_top_label": 0.17475344827586206,
          "mean_confidence": 0.9523568965517242,
          "brier_hard": 0.3675763103448276,
          "brier_hard_n": 116,
          "nll_hard": 1.8418113090371786,
          "nll_hard_n": 116,
          "zero_probability_gold": 0.04310344827586207,
          "zero_probability_gold_n": 116,
          "macro_f1": 0.7797524113313588
        },
        "massive-intent.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7733333333333333,
          "accuracy_valid": 0.7733333333333333,
          "ece_top_label": 0.17472328670404882,
          "mean_confidence": 0.9480566200373822,
          "brier_hard": 0.38383190196466177,
          "brier_hard_n": 300,
          "nll_hard": 2.4090375945333213,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.05,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.502438924453748
        },
        "xnli.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8666666666666667,
          "accuracy_valid": 0.8666666666666667,
          "ece_top_label": 0.054425126999604626,
          "mean_confidence": 0.9150511806609052,
          "brier_hard": 0.2024701804692983,
          "brier_hard_n": 300,
          "nll_hard": 0.3723722500840691,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.8674417053037536
        }
      }
    },
    "4": {
      "seed": 4,
      "selected_epoch": 8,
      "elapsed_s": 496.1691781419795,
      "initial_validation": {
        "soft_cross_entropy": 1.544869564374288,
        "accuracy": 0.36833333333333335,
        "brier_soft": 0.43051501592000324,
        "decisions": 600
      },
      "selected_validation": {
        "soft_cross_entropy": 0.8540202794472377,
        "accuracy": 0.76,
        "brier_soft": 0.07019127607345581,
        "decisions": 600
      },
      "stopping": {
        "reason": "early_stopping",
        "epochs_completed": 11,
        "patience": 3,
        "min_epochs": 10,
        "epochs_without_improvement": 3,
        "metric": "validation.soft_cross_entropy",
        "min_delta": 0.0
      },
      "frozen_parameters_unchanged": true,
      "suites": {
        "typed-decisions": {
          "attempted": 2000,
          "valid": 2000,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7545,
          "accuracy_valid": 0.7545,
          "ece_top_label": 0.2042592508573846,
          "mean_confidence": 0.5504920240151281,
          "brier_hard": 0.4113014025244486,
          "brier_hard_n": 2000,
          "nll_hard": 0.7247566141198507,
          "nll_hard_n": 2000,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 2000,
          "soft_accuracy": 0.46590212671846004,
          "soft_accuracy_n": 2000,
          "brier_soft": 0.06812301484891332,
          "brier_soft_n": 2000,
          "kl_gold_to_prediction": 0.12790407031921358,
          "kl_gold_to_prediction_n": 2000,
          "total_variation": 0.18054231568172208,
          "total_variation_n": 2000,
          "score_mae": 0.25377940453843617,
          "score_mae_n": 800,
          "within_one_level": 0.98375,
          "within_one_level_n": 800,
          "macro_f1": 0.6337422972118567
        },
        "ag-news": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.9516666666666667,
          "accuracy_valid": 0.9516666666666667,
          "ece_top_label": 0.03233758800344756,
          "mean_confidence": 0.9199325508493339,
          "brier_hard": 0.08178925980745573,
          "brier_hard_n": 600,
          "nll_hard": 0.16974235809511878,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.9489828512364638
        },
        "emotion": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.5783333333333334,
          "accuracy_valid": 0.5783333333333334,
          "ece_top_label": 0.3295382068772924,
          "mean_confidence": 0.9035819218093762,
          "brier_hard": 0.7425345488860924,
          "brier_hard_n": 600,
          "nll_hard": 2.3910245621280453,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.013333333333333334,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.48527808236443093
        },
        "boolq": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8316666666666667,
          "accuracy_valid": 0.8316666666666667,
          "ece_top_label": 0.09521133333333336,
          "mean_confidence": 0.9097233333333333,
          "brier_hard": 0.27490432886666666,
          "brier_hard_n": 600,
          "nll_hard": 0.49707214589381826,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0016666666666666668,
          "zero_probability_gold_n": 600,
          "macro_f1": 0.8196294367140413
        },
        "sst5": {
          "attempted": 600,
          "valid": 600,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.41,
          "accuracy_valid": 0.41,
          "ece_top_label": 0.20145470242332547,
          "mean_confidence": 0.6098530357566588,
          "brier_hard": 0.7512849781669847,
          "brier_hard_n": 600,
          "nll_hard": 1.5195838995022448,
          "nll_hard_n": 600,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 600,
          "score_mae": 0.8012926973404637,
          "score_mae_n": 600,
          "within_one_level": 0.7016666666666667,
          "within_one_level_n": 600,
          "macro_f1": 0.37608916605464326
        },
        "prompt-injections": {
          "attempted": 116,
          "valid": 116,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.7413793103448276,
          "accuracy_valid": 0.7413793103448276,
          "ece_top_label": 0.21426120689655168,
          "mean_confidence": 0.9556405172413793,
          "brier_hard": 0.43549763362068966,
          "brier_hard_n": 116,
          "nll_hard": 2.2934410633534186,
          "nll_hard_n": 116,
          "zero_probability_gold": 0.0603448275862069,
          "zero_probability_gold_n": 116,
          "macro_f1": 0.7276995305164319
        },
        "massive-intent.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.77,
          "accuracy_valid": 0.77,
          "ece_top_label": 0.17971198952007902,
          "mean_confidence": 0.9447345124535086,
          "brier_hard": 0.4058348152163361,
          "brier_hard_n": 300,
          "nll_hard": 2.9106532352305194,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.07333333333333333,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.5030980723143226
        },
        "xnli.en": {
          "attempted": 300,
          "valid": 300,
          "failed": 0,
          "coverage": 1.0,
          "accuracy_all": 0.8633333333333333,
          "accuracy_valid": 0.8633333333333333,
          "ece_top_label": 0.0751956003124493,
          "mean_confidence": 0.9162548569668819,
          "brier_hard": 0.22384958324541668,
          "brier_hard_n": 300,
          "nll_hard": 0.4127381776210696,
          "nll_hard_n": 300,
          "zero_probability_gold": 0.0,
          "zero_probability_gold_n": 300,
          "macro_f1": 0.864378848443158
        }
      }
    }
  },
  "summary": {
    "typed-decisions": {
      "n": 5,
      "mean_percent": 75.28,
      "sample_variance_pp_squared": 0.8619999999999957,
      "sample_std_pp": 0.9284395510748105
    },
    "ag-news": {
      "n": 5,
      "mean_percent": 94.46666666666667,
      "sample_variance_pp_squared": 0.3388888888888897,
      "sample_std_pp": 0.5821416398857667
    },
    "boolq": {
      "n": 5,
      "mean_percent": 82.10000000000001,
      "sample_variance_pp_squared": 0.7861111111111168,
      "sample_std_pp": 0.8866290718846956
    },
    "emotion": {
      "n": 5,
      "mean_percent": 57.2,
      "sample_variance_pp_squared": 1.0055555555555546,
      "sample_std_pp": 1.0027739304327543
    },
    "prompt-injections": {
      "n": 5,
      "mean_percent": 74.48275862068965,
      "sample_variance_pp_squared": 15.457788347205712,
      "sample_std_pp": 3.93163939689358
    },
    "sst5": {
      "n": 5,
      "mean_percent": 41.0,
      "sample_variance_pp_squared": 6.027777777777775,
      "sample_std_pp": 2.4551533104427055
    },
    "massive-intent.en": {
      "n": 5,
      "mean_percent": 76.8,
      "sample_variance_pp_squared": 2.422222222222218,
      "sample_std_pp": 1.556349003990499
    },
    "xnli.en": {
      "n": 5,
      "mean_percent": 87.26666666666667,
      "sample_variance_pp_squared": 1.0777777777777784,
      "sample_std_pp": 1.038160766826496
    }
  },
  "candidate_seed": 4,
  "variance_definition": "sample variance, n-1, percentage points squared",
  "benchmark_manifest": {
    "format_version": 1,
    "sha256": "10fb671245cdac1ee09e9c4d2b9b09829fbf1abd06e0954fddee0b9ec8519854",
    "cases": 3516,
    "decisions": 5116,
    "profile": "heldout-eight",
    "role": "test",
    "parent_bundles": {
      "core": "477c9cbd4f9eaf8844fa4d6870560e751960735e7441b803306d6e011e689435",
      "language_en": "a3d41f5060826770f0f28b16a0aff26b4294691423624349db6b008789ea9096"
    },
    "purpose": "Base vs official typed-decisions specialist vs trained Laya-only input interface; no new training."
  }
}