| { |
| "baseline": { |
| "typed-decisions": 0.767, |
| "ag-news": 0.9466666666666667, |
| "boolq": 0.82, |
| "emotion": 0.5816666666666667, |
| "prompt-injections": 0.6724137931034483, |
| "sst5": 0.48, |
| "massive-intent.en": 0.81, |
| "xnli.en": 0.8633333333333333 |
| }, |
| "evaluation_reference": { |
| "name": "Official Laya-TD", |
| "model_id": "convaiinnovations/laya-typed-decisions", |
| "revision": "1a793eb568e6718f15941d08f85432581df534e3", |
| "accuracy": { |
| "typed-decisions": 0.767, |
| "ag-news": 0.9466666666666667, |
| "boolq": 0.82, |
| "emotion": 0.5816666666666667, |
| "prompt-injections": 0.6724137931034483, |
| "sst5": 0.48, |
| "massive-intent.en": 0.81, |
| "xnli.en": 0.8633333333333333 |
| }, |
| "benchmark_cases_sha256": "10fb671245cdac1ee09e9c4d2b9b09829fbf1abd06e0954fddee0b9ec8519854", |
| "predictions_sha256": "15952331cc2eb4732075b326f6ff764129f84cbbac09529b30dcf84842df67d3", |
| "environment": { |
| "python": "3.12.3", |
| "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", |
| "processor": "x86_64", |
| "packages": { |
| "laya": "0.3.20", |
| "torch": "2.13.0+cu130", |
| "transformers": "5.17.0", |
| "datasets": "5.0.1", |
| "huggingface-hub": "1.33.0" |
| }, |
| "harness": "0.1.0" |
| }, |
| "model_metadata": { |
| "max_len": 1024, |
| "head_max_len": 256, |
| "native_prediction": true, |
| "device": "cuda:1", |
| "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", |
| "dtype": "torch.bfloat16", |
| "torch_version": "2.13.0+cu130", |
| "transformers_version": "5.17.0", |
| "laya_version": "0.3.20", |
| "parameters": 421293827, |
| "temperature": [ |
| 1.0148024559020996, |
| 1.0374259948730469, |
| 1.0575125217437744 |
| ], |
| "temperature_by_options": { |
| "choice:3-5": 1.7601518630981445, |
| "choice:6-10": 1.0000158548355103, |
| "score:3-5": 1.2514300346374512, |
| "noul:2": 1.983399510383606, |
| "choice:11+": 0.5, |
| "choice:2": 1.9063563346862793 |
| }, |
| "reproducibility": { |
| "deterministic_algorithms": true, |
| "warn_only": false, |
| "cublas_workspace_config": ":4096:8", |
| "python_hash_seed": "0", |
| "cudnn_benchmark": false, |
| "cudnn_deterministic": true, |
| "cudnn_allow_tf32": false, |
| "float32_matmul_precision": "highest", |
| "cudnn_sdp_enabled": false, |
| "flash_sdp_enabled": true, |
| "mem_efficient_sdp_enabled": true, |
| "math_sdp_enabled": true, |
| "cuda_version": "13.0", |
| "cudnn_version": 92000, |
| "scope": "fixed hardware/runtime; cross-platform bitwise agreement is not promised" |
| }, |
| "interface": null |
| }, |
| "model_weights_sha256": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e", |
| "matched_runtime_and_token_budgets": true, |
| "independent_scoring_matches": true |
| }, |
| "training_base_control": { |
| "model_id": "convaiinnovations/laya", |
| "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", |
| "accuracy": { |
| "typed-decisions": 0.3625, |
| "ag-news": 0.9483333333333334, |
| "boolq": 0.83, |
| "emotion": 0.5733333333333334, |
| "prompt-injections": 0.6896551724137931, |
| "sst5": 0.37, |
| "massive-intent.en": 0.7866666666666666, |
| "xnli.en": 0.86 |
| } |
| }, |
| "runs": { |
| "0": { |
| "seed": 0, |
| "selected_epoch": 11, |
| "elapsed_s": 612.5042744020466, |
| "initial_validation": { |
| "soft_cross_entropy": 1.544869564374288, |
| "accuracy": 0.36833333333333335, |
| "brier_soft": 0.43051501592000324, |
| "decisions": 600 |
| }, |
| "selected_validation": { |
| "soft_cross_entropy": 0.8771124110619227, |
| "accuracy": 0.7116666666666667, |
| "brier_soft": 0.0828241604194045, |
| "decisions": 600 |
| }, |
| "stopping": { |
| "reason": "early_stopping", |
| "epochs_completed": 14, |
| "patience": 3, |
| "min_epochs": 10, |
| "epochs_without_improvement": 3, |
| "metric": "validation.soft_cross_entropy", |
| "min_delta": 0.0 |
| }, |
| "frozen_parameters_unchanged": true, |
| "suites": { |
| "typed-decisions": { |
| "attempted": 2000, |
| "valid": 2000, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7425, |
| "accuracy_valid": 0.7425, |
| "ece_top_label": 0.19222331993705236, |
| "mean_confidence": 0.5502766800629476, |
| "brier_hard": 0.41714854154295294, |
| "brier_hard_n": 2000, |
| "nll_hard": 0.7346455747590726, |
| "nll_hard_n": 2000, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 2000, |
| "soft_accuracy": 0.4642362342348646, |
| "soft_accuracy_n": 2000, |
| "brier_soft": 0.07282817778125876, |
| "brier_soft_n": 2000, |
| "kl_gold_to_prediction": 0.13635658195747785, |
| "kl_gold_to_prediction_n": 2000, |
| "total_variation": 0.18626503447270873, |
| "total_variation_n": 2000, |
| "score_mae": 0.2670681801272688, |
| "score_mae_n": 800, |
| "within_one_level": 0.98125, |
| "within_one_level_n": 800, |
| "macro_f1": 0.6211545025911077 |
| }, |
| "ag-news": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.9416666666666667, |
| "accuracy_valid": 0.9416666666666667, |
| "ece_top_label": 0.032780915974200804, |
| "mean_confidence": 0.9120336837609119, |
| "brier_hard": 0.0911117485859852, |
| "brier_hard_n": 600, |
| "nll_hard": 0.18384446182116462, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.9391643390047506 |
| }, |
| "emotion": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.585, |
| "accuracy_valid": 0.585, |
| "ece_top_label": 0.3290210797041948, |
| "mean_confidence": 0.9044507912979094, |
| "brier_hard": 0.7352614358764159, |
| "brier_hard_n": 600, |
| "nll_hard": 2.325969372091789, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.01, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.4846329953255675 |
| }, |
| "boolq": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8083333333333333, |
| "accuracy_valid": 0.8083333333333333, |
| "ece_top_label": 0.09874549999999996, |
| "mean_confidence": 0.9070788333333333, |
| "brier_hard": 0.2864656637, |
| "brier_hard_n": 600, |
| "nll_hard": 0.45967158478397707, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.7946275764565816 |
| }, |
| "sst5": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.38166666666666665, |
| "accuracy_valid": 0.38166666666666665, |
| "ece_top_label": 0.23314279850468647, |
| "mean_confidence": 0.614809465171353, |
| "brier_hard": 0.7997842513426261, |
| "brier_hard_n": 600, |
| "nll_hard": 1.7175170777887119, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "score_mae": 0.9144952235712186, |
| "score_mae_n": 600, |
| "within_one_level": 0.6466666666666666, |
| "within_one_level_n": 600, |
| "macro_f1": 0.3319436236284081 |
| }, |
| "prompt-injections": { |
| "attempted": 116, |
| "valid": 116, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.6982758620689655, |
| "accuracy_valid": 0.6982758620689655, |
| "ece_top_label": 0.25387413793103447, |
| "mean_confidence": 0.95215, |
| "brier_hard": 0.5115509806896552, |
| "brier_hard_n": 116, |
| "nll_hard": 2.710472238032788, |
| "nll_hard_n": 116, |
| "zero_probability_gold": 0.06896551724137931, |
| "zero_probability_gold_n": 116, |
| "macro_f1": 0.6809931641392315 |
| }, |
| "massive-intent.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.79, |
| "accuracy_valid": 0.79, |
| "ece_top_label": 0.16259349215790048, |
| "mean_confidence": 0.9477817275562629, |
| "brier_hard": 0.3655905743529746, |
| "brier_hard_n": 300, |
| "nll_hard": 2.4460407722040176, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.06, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.5695724937908466 |
| }, |
| "xnli.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.89, |
| "accuracy_valid": 0.89, |
| "ece_top_label": 0.05910627327283609, |
| "mean_confidence": 0.9123050519493905, |
| "brier_hard": 0.18251872229879984, |
| "brier_hard_n": 300, |
| "nll_hard": 0.33404125735595414, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.8905240397272149 |
| } |
| } |
| }, |
| "1": { |
| "seed": 1, |
| "selected_epoch": 9, |
| "elapsed_s": 541.4585086829611, |
| "initial_validation": { |
| "soft_cross_entropy": 1.544869564374288, |
| "accuracy": 0.36833333333333335, |
| "brier_soft": 0.43051501592000324, |
| "decisions": 600 |
| }, |
| "selected_validation": { |
| "soft_cross_entropy": 0.8583925066391627, |
| "accuracy": 0.7466666666666667, |
| "brier_soft": 0.07437195796364297, |
| "decisions": 600 |
| }, |
| "stopping": { |
| "reason": "early_stopping", |
| "epochs_completed": 12, |
| "patience": 3, |
| "min_epochs": 10, |
| "epochs_without_improvement": 3, |
| "metric": "validation.soft_cross_entropy", |
| "min_delta": 0.0 |
| }, |
| "frozen_parameters_unchanged": true, |
| "suites": { |
| "typed-decisions": { |
| "attempted": 2000, |
| "valid": 2000, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7565, |
| "accuracy_valid": 0.7565, |
| "ece_top_label": 0.19864268503011012, |
| "mean_confidence": 0.5578573149698899, |
| "brier_hard": 0.40482086772099385, |
| "brier_hard_n": 2000, |
| "nll_hard": 0.7162594531200565, |
| "nll_hard_n": 2000, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 2000, |
| "soft_accuracy": 0.469737413364262, |
| "soft_accuracy_n": 2000, |
| "brier_soft": 0.0659469654317038, |
| "brier_soft_n": 2000, |
| "kl_gold_to_prediction": 0.12376910578634269, |
| "kl_gold_to_prediction_n": 2000, |
| "total_variation": 0.17750424596946363, |
| "total_variation_n": 2000, |
| "score_mae": 0.24687533223920302, |
| "score_mae_n": 800, |
| "within_one_level": 0.98625, |
| "within_one_level_n": 800, |
| "macro_f1": 0.6300073343403317 |
| }, |
| "ag-news": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.9483333333333334, |
| "accuracy_valid": 0.9483333333333334, |
| "ece_top_label": 0.03462398162033311, |
| "mean_confidence": 0.9188607247165506, |
| "brier_hard": 0.08433672632610516, |
| "brier_hard_n": 600, |
| "nll_hard": 0.17417200848974151, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.9456145576151846 |
| }, |
| "emotion": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.5716666666666667, |
| "accuracy_valid": 0.5716666666666667, |
| "ece_top_label": 0.33449032496578573, |
| "mean_confidence": 0.90098409027152, |
| "brier_hard": 0.7328256742449804, |
| "brier_hard_n": 600, |
| "nll_hard": 2.388523752936007, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.013333333333333334, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.4746076611364094 |
| }, |
| "boolq": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8266666666666667, |
| "accuracy_valid": 0.8266666666666667, |
| "ece_top_label": 0.08269850000000001, |
| "mean_confidence": 0.9093651666666667, |
| "brier_hard": 0.26852242536666665, |
| "brier_hard_n": 600, |
| "nll_hard": 0.44571515698724307, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.8140998140998141 |
| }, |
| "sst5": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.4483333333333333, |
| "accuracy_valid": 0.4483333333333333, |
| "ece_top_label": 0.1552169576555402, |
| "mean_confidence": 0.5963372951766569, |
| "brier_hard": 0.715830700927659, |
| "brier_hard_n": 600, |
| "nll_hard": 1.4312799777899081, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "score_mae": 0.763809024216731, |
| "score_mae_n": 600, |
| "within_one_level": 0.7216666666666667, |
| "within_one_level_n": 600, |
| "macro_f1": 0.4241661418376961 |
| }, |
| "prompt-injections": { |
| "attempted": 116, |
| "valid": 116, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7844827586206896, |
| "accuracy_valid": 0.7844827586206896, |
| "ece_top_label": 0.17151379310344822, |
| "mean_confidence": 0.9531103448275862, |
| "brier_hard": 0.36615182827586207, |
| "brier_hard_n": 116, |
| "nll_hard": 1.9254142815070245, |
| "nll_hard_n": 116, |
| "zero_probability_gold": 0.05172413793103448, |
| "zero_probability_gold_n": 116, |
| "macro_f1": 0.7785414280259642 |
| }, |
| "massive-intent.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7566666666666667, |
| "accuracy_valid": 0.7566666666666667, |
| "ece_top_label": 0.19351909840865464, |
| "mean_confidence": 0.9501857650753213, |
| "brier_hard": 0.4290677996735415, |
| "brier_hard_n": 300, |
| "nll_hard": 3.2153508464750185, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.08666666666666667, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.4943833232448793 |
| }, |
| "xnli.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.87, |
| "accuracy_valid": 0.87, |
| "ece_top_label": 0.07153730747712309, |
| "mean_confidence": 0.9147377091848204, |
| "brier_hard": 0.22017864577608134, |
| "brier_hard_n": 300, |
| "nll_hard": 0.4059236764304257, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.8710695955086495 |
| } |
| } |
| }, |
| "2": { |
| "seed": 2, |
| "selected_epoch": 13, |
| "elapsed_s": 699.7859417049913, |
| "initial_validation": { |
| "soft_cross_entropy": 1.544869564374288, |
| "accuracy": 0.36833333333333335, |
| "brier_soft": 0.43051501592000324, |
| "decisions": 600 |
| }, |
| "selected_validation": { |
| "soft_cross_entropy": 0.8543428432941437, |
| "accuracy": 0.7583333333333333, |
| "brier_soft": 0.07187043125430743, |
| "decisions": 600 |
| }, |
| "stopping": { |
| "reason": "early_stopping", |
| "epochs_completed": 16, |
| "patience": 3, |
| "min_epochs": 10, |
| "epochs_without_improvement": 3, |
| "metric": "validation.soft_cross_entropy", |
| "min_delta": 0.0 |
| }, |
| "frozen_parameters_unchanged": true, |
| "suites": { |
| "typed-decisions": { |
| "attempted": 2000, |
| "valid": 2000, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.745, |
| "accuracy_valid": 0.745, |
| "ece_top_label": 0.18594273050266366, |
| "mean_confidence": 0.5591527012331047, |
| "brier_hard": 0.40870440390133495, |
| "brier_hard_n": 2000, |
| "nll_hard": 0.7205417712302097, |
| "nll_hard_n": 2000, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 2000, |
| "soft_accuracy": 0.46875735649306255, |
| "soft_accuracy_n": 2000, |
| "brier_soft": 0.06782387317668824, |
| "brier_soft_n": 2000, |
| "kl_gold_to_prediction": 0.12687181845569698, |
| "kl_gold_to_prediction_n": 2000, |
| "total_variation": 0.17935579359201664, |
| "total_variation_n": 2000, |
| "score_mae": 0.25587944352517583, |
| "score_mae_n": 800, |
| "within_one_level": 0.99, |
| "within_one_level_n": 800, |
| "macro_f1": 0.608344198168488 |
| }, |
| "ag-news": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.945, |
| "accuracy_valid": 0.945, |
| "ece_top_label": 0.03201179295141958, |
| "mean_confidence": 0.9180146184255912, |
| "brier_hard": 0.08699114920264774, |
| "brier_hard_n": 600, |
| "nll_hard": 0.17458965637122142, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.9421288699146882 |
| }, |
| "emotion": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.56, |
| "accuracy_valid": 0.56, |
| "ece_top_label": 0.3501647005433899, |
| "mean_confidence": 0.9077857005433899, |
| "brier_hard": 0.7581475668473461, |
| "brier_hard_n": 600, |
| "nll_hard": 2.427279492608694, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.013333333333333334, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.4667304743466239 |
| }, |
| "boolq": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8183333333333334, |
| "accuracy_valid": 0.8183333333333334, |
| "ece_top_label": 0.08817149999999999, |
| "mean_confidence": 0.9065048333333333, |
| "brier_hard": 0.2762812369666667, |
| "brier_hard_n": 600, |
| "nll_hard": 0.453869636876017, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.8046122269724754 |
| }, |
| "sst5": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.3983333333333333, |
| "accuracy_valid": 0.3983333333333333, |
| "ece_top_label": 0.21926419101950614, |
| "mean_confidence": 0.6138282869290971, |
| "brier_hard": 0.7663539453668811, |
| "brier_hard_n": 600, |
| "nll_hard": 1.5341982352412973, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "score_mae": 0.7954480085445932, |
| "score_mae_n": 600, |
| "within_one_level": 0.695, |
| "within_one_level_n": 600, |
| "macro_f1": 0.3618869575527743 |
| }, |
| "prompt-injections": { |
| "attempted": 116, |
| "valid": 116, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7155172413793104, |
| "accuracy_valid": 0.7155172413793104, |
| "ece_top_label": 0.2621448275862069, |
| "mean_confidence": 0.9488327586206896, |
| "brier_hard": 0.4988274203448276, |
| "brier_hard_n": 116, |
| "nll_hard": 2.61023697496577, |
| "nll_hard_n": 116, |
| "zero_probability_gold": 0.06896551724137931, |
| "zero_probability_gold_n": 116, |
| "macro_f1": 0.6992221261884184 |
| }, |
| "massive-intent.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.75, |
| "accuracy_valid": 0.75, |
| "ece_top_label": 0.20336797564979028, |
| "mean_confidence": 0.9533679756497903, |
| "brier_hard": 0.4358421604675745, |
| "brier_hard_n": 300, |
| "nll_hard": 3.1322756175288426, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.08, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.5034639202079357 |
| }, |
| "xnli.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8733333333333333, |
| "accuracy_valid": 0.8733333333333333, |
| "ece_top_label": 0.05108680216338136, |
| "mean_confidence": 0.914837261771426, |
| "brier_hard": 0.20561652153986584, |
| "brier_hard_n": 300, |
| "nll_hard": 0.3742216981370551, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.8742255731346232 |
| } |
| } |
| }, |
| "3": { |
| "seed": 3, |
| "selected_epoch": 8, |
| "elapsed_s": 495.011001034989, |
| "initial_validation": { |
| "soft_cross_entropy": 1.544869564374288, |
| "accuracy": 0.36833333333333335, |
| "brier_soft": 0.43051501592000324, |
| "decisions": 600 |
| }, |
| "selected_validation": { |
| "soft_cross_entropy": 0.8545633804798126, |
| "accuracy": 0.77, |
| "brier_soft": 0.06962224062221746, |
| "decisions": 600 |
| }, |
| "stopping": { |
| "reason": "early_stopping", |
| "epochs_completed": 11, |
| "patience": 3, |
| "min_epochs": 10, |
| "epochs_without_improvement": 3, |
| "metric": "validation.soft_cross_entropy", |
| "min_delta": 0.0 |
| }, |
| "frozen_parameters_unchanged": true, |
| "suites": { |
| "typed-decisions": { |
| "attempted": 2000, |
| "valid": 2000, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7655, |
| "accuracy_valid": 0.7655, |
| "ece_top_label": 0.2096452515650509, |
| "mean_confidence": 0.5558547484349491, |
| "brier_hard": 0.4011842616732685, |
| "brier_hard_n": 2000, |
| "nll_hard": 0.7113399953872087, |
| "nll_hard_n": 2000, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 2000, |
| "soft_accuracy": 0.4683026193430771, |
| "soft_accuracy_n": 2000, |
| "brier_soft": 0.06585425455571205, |
| "brier_soft_n": 2000, |
| "kl_gold_to_prediction": 0.12507067261764912, |
| "kl_gold_to_prediction_n": 2000, |
| "total_variation": 0.17848467876854548, |
| "total_variation_n": 2000, |
| "score_mae": 0.24395106365305969, |
| "score_mae_n": 800, |
| "within_one_level": 0.98375, |
| "within_one_level_n": 800, |
| "macro_f1": 0.6399493527359326 |
| }, |
| "ag-news": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.9366666666666666, |
| "accuracy_valid": 0.9366666666666666, |
| "ece_top_label": 0.0312034834128748, |
| "mean_confidence": 0.914535932360556, |
| "brier_hard": 0.09381807669545053, |
| "brier_hard_n": 600, |
| "nll_hard": 0.1866731647377733, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.9336084571946038 |
| }, |
| "emotion": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.565, |
| "accuracy_valid": 0.565, |
| "ece_top_label": 0.3370553794856075, |
| "mean_confidence": 0.8987647229086076, |
| "brier_hard": 0.7489678137408704, |
| "brier_hard_n": 600, |
| "nll_hard": 2.3318777367931305, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.011666666666666667, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.4760859469034663 |
| }, |
| "boolq": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.82, |
| "accuracy_valid": 0.82, |
| "ece_top_label": 0.09564866666666666, |
| "mean_confidence": 0.9069513333333333, |
| "brier_hard": 0.2824305872, |
| "brier_hard_n": 600, |
| "nll_hard": 0.4619444556248629, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.8069498069498069 |
| }, |
| "sst5": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.4116666666666667, |
| "accuracy_valid": 0.4116666666666667, |
| "ece_top_label": 0.19022023187751014, |
| "mean_confidence": 0.6018868985441768, |
| "brier_hard": 0.7571439809814903, |
| "brier_hard_n": 600, |
| "nll_hard": 1.5683213482322946, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "score_mae": 0.8321864596740475, |
| "score_mae_n": 600, |
| "within_one_level": 0.6783333333333333, |
| "within_one_level_n": 600, |
| "macro_f1": 0.3816445513227996 |
| }, |
| "prompt-injections": { |
| "attempted": 116, |
| "valid": 116, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7844827586206896, |
| "accuracy_valid": 0.7844827586206896, |
| "ece_top_label": 0.17475344827586206, |
| "mean_confidence": 0.9523568965517242, |
| "brier_hard": 0.3675763103448276, |
| "brier_hard_n": 116, |
| "nll_hard": 1.8418113090371786, |
| "nll_hard_n": 116, |
| "zero_probability_gold": 0.04310344827586207, |
| "zero_probability_gold_n": 116, |
| "macro_f1": 0.7797524113313588 |
| }, |
| "massive-intent.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7733333333333333, |
| "accuracy_valid": 0.7733333333333333, |
| "ece_top_label": 0.17472328670404882, |
| "mean_confidence": 0.9480566200373822, |
| "brier_hard": 0.38383190196466177, |
| "brier_hard_n": 300, |
| "nll_hard": 2.4090375945333213, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.05, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.502438924453748 |
| }, |
| "xnli.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8666666666666667, |
| "accuracy_valid": 0.8666666666666667, |
| "ece_top_label": 0.054425126999604626, |
| "mean_confidence": 0.9150511806609052, |
| "brier_hard": 0.2024701804692983, |
| "brier_hard_n": 300, |
| "nll_hard": 0.3723722500840691, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.8674417053037536 |
| } |
| } |
| }, |
| "4": { |
| "seed": 4, |
| "selected_epoch": 8, |
| "elapsed_s": 496.1691781419795, |
| "initial_validation": { |
| "soft_cross_entropy": 1.544869564374288, |
| "accuracy": 0.36833333333333335, |
| "brier_soft": 0.43051501592000324, |
| "decisions": 600 |
| }, |
| "selected_validation": { |
| "soft_cross_entropy": 0.8540202794472377, |
| "accuracy": 0.76, |
| "brier_soft": 0.07019127607345581, |
| "decisions": 600 |
| }, |
| "stopping": { |
| "reason": "early_stopping", |
| "epochs_completed": 11, |
| "patience": 3, |
| "min_epochs": 10, |
| "epochs_without_improvement": 3, |
| "metric": "validation.soft_cross_entropy", |
| "min_delta": 0.0 |
| }, |
| "frozen_parameters_unchanged": true, |
| "suites": { |
| "typed-decisions": { |
| "attempted": 2000, |
| "valid": 2000, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7545, |
| "accuracy_valid": 0.7545, |
| "ece_top_label": 0.2042592508573846, |
| "mean_confidence": 0.5504920240151281, |
| "brier_hard": 0.4113014025244486, |
| "brier_hard_n": 2000, |
| "nll_hard": 0.7247566141198507, |
| "nll_hard_n": 2000, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 2000, |
| "soft_accuracy": 0.46590212671846004, |
| "soft_accuracy_n": 2000, |
| "brier_soft": 0.06812301484891332, |
| "brier_soft_n": 2000, |
| "kl_gold_to_prediction": 0.12790407031921358, |
| "kl_gold_to_prediction_n": 2000, |
| "total_variation": 0.18054231568172208, |
| "total_variation_n": 2000, |
| "score_mae": 0.25377940453843617, |
| "score_mae_n": 800, |
| "within_one_level": 0.98375, |
| "within_one_level_n": 800, |
| "macro_f1": 0.6337422972118567 |
| }, |
| "ag-news": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.9516666666666667, |
| "accuracy_valid": 0.9516666666666667, |
| "ece_top_label": 0.03233758800344756, |
| "mean_confidence": 0.9199325508493339, |
| "brier_hard": 0.08178925980745573, |
| "brier_hard_n": 600, |
| "nll_hard": 0.16974235809511878, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.9489828512364638 |
| }, |
| "emotion": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.5783333333333334, |
| "accuracy_valid": 0.5783333333333334, |
| "ece_top_label": 0.3295382068772924, |
| "mean_confidence": 0.9035819218093762, |
| "brier_hard": 0.7425345488860924, |
| "brier_hard_n": 600, |
| "nll_hard": 2.3910245621280453, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.013333333333333334, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.48527808236443093 |
| }, |
| "boolq": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8316666666666667, |
| "accuracy_valid": 0.8316666666666667, |
| "ece_top_label": 0.09521133333333336, |
| "mean_confidence": 0.9097233333333333, |
| "brier_hard": 0.27490432886666666, |
| "brier_hard_n": 600, |
| "nll_hard": 0.49707214589381826, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0016666666666666668, |
| "zero_probability_gold_n": 600, |
| "macro_f1": 0.8196294367140413 |
| }, |
| "sst5": { |
| "attempted": 600, |
| "valid": 600, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.41, |
| "accuracy_valid": 0.41, |
| "ece_top_label": 0.20145470242332547, |
| "mean_confidence": 0.6098530357566588, |
| "brier_hard": 0.7512849781669847, |
| "brier_hard_n": 600, |
| "nll_hard": 1.5195838995022448, |
| "nll_hard_n": 600, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 600, |
| "score_mae": 0.8012926973404637, |
| "score_mae_n": 600, |
| "within_one_level": 0.7016666666666667, |
| "within_one_level_n": 600, |
| "macro_f1": 0.37608916605464326 |
| }, |
| "prompt-injections": { |
| "attempted": 116, |
| "valid": 116, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.7413793103448276, |
| "accuracy_valid": 0.7413793103448276, |
| "ece_top_label": 0.21426120689655168, |
| "mean_confidence": 0.9556405172413793, |
| "brier_hard": 0.43549763362068966, |
| "brier_hard_n": 116, |
| "nll_hard": 2.2934410633534186, |
| "nll_hard_n": 116, |
| "zero_probability_gold": 0.0603448275862069, |
| "zero_probability_gold_n": 116, |
| "macro_f1": 0.7276995305164319 |
| }, |
| "massive-intent.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.77, |
| "accuracy_valid": 0.77, |
| "ece_top_label": 0.17971198952007902, |
| "mean_confidence": 0.9447345124535086, |
| "brier_hard": 0.4058348152163361, |
| "brier_hard_n": 300, |
| "nll_hard": 2.9106532352305194, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.07333333333333333, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.5030980723143226 |
| }, |
| "xnli.en": { |
| "attempted": 300, |
| "valid": 300, |
| "failed": 0, |
| "coverage": 1.0, |
| "accuracy_all": 0.8633333333333333, |
| "accuracy_valid": 0.8633333333333333, |
| "ece_top_label": 0.0751956003124493, |
| "mean_confidence": 0.9162548569668819, |
| "brier_hard": 0.22384958324541668, |
| "brier_hard_n": 300, |
| "nll_hard": 0.4127381776210696, |
| "nll_hard_n": 300, |
| "zero_probability_gold": 0.0, |
| "zero_probability_gold_n": 300, |
| "macro_f1": 0.864378848443158 |
| } |
| } |
| } |
| }, |
| "summary": { |
| "typed-decisions": { |
| "n": 5, |
| "mean_percent": 75.28, |
| "sample_variance_pp_squared": 0.8619999999999957, |
| "sample_std_pp": 0.9284395510748105 |
| }, |
| "ag-news": { |
| "n": 5, |
| "mean_percent": 94.46666666666667, |
| "sample_variance_pp_squared": 0.3388888888888897, |
| "sample_std_pp": 0.5821416398857667 |
| }, |
| "boolq": { |
| "n": 5, |
| "mean_percent": 82.10000000000001, |
| "sample_variance_pp_squared": 0.7861111111111168, |
| "sample_std_pp": 0.8866290718846956 |
| }, |
| "emotion": { |
| "n": 5, |
| "mean_percent": 57.2, |
| "sample_variance_pp_squared": 1.0055555555555546, |
| "sample_std_pp": 1.0027739304327543 |
| }, |
| "prompt-injections": { |
| "n": 5, |
| "mean_percent": 74.48275862068965, |
| "sample_variance_pp_squared": 15.457788347205712, |
| "sample_std_pp": 3.93163939689358 |
| }, |
| "sst5": { |
| "n": 5, |
| "mean_percent": 41.0, |
| "sample_variance_pp_squared": 6.027777777777775, |
| "sample_std_pp": 2.4551533104427055 |
| }, |
| "massive-intent.en": { |
| "n": 5, |
| "mean_percent": 76.8, |
| "sample_variance_pp_squared": 2.422222222222218, |
| "sample_std_pp": 1.556349003990499 |
| }, |
| "xnli.en": { |
| "n": 5, |
| "mean_percent": 87.26666666666667, |
| "sample_variance_pp_squared": 1.0777777777777784, |
| "sample_std_pp": 1.038160766826496 |
| } |
| }, |
| "candidate_seed": 4, |
| "variance_definition": "sample variance, n-1, percentage points squared", |
| "benchmark_manifest": { |
| "format_version": 1, |
| "sha256": "10fb671245cdac1ee09e9c4d2b9b09829fbf1abd06e0954fddee0b9ec8519854", |
| "cases": 3516, |
| "decisions": 5116, |
| "profile": "heldout-eight", |
| "role": "test", |
| "parent_bundles": { |
| "core": "477c9cbd4f9eaf8844fa4d6870560e751960735e7441b803306d6e011e689435", |
| "language_en": "a3d41f5060826770f0f28b16a0aff26b4294691423624349db6b008789ea9096" |
| }, |
| "purpose": "Base vs official typed-decisions specialist vs trained Laya-only input interface; no new training." |
| } |
| } |
|
|
