{ "domain": "finance-topics", "seed": 0, "selected_epoch": 2, "test": { "cases": 4117, "accuracy": 0.8430896283701724, "macro_f1": 0.8256463980064556, "per_class": { "0": { "precision": 0.7466666666666667, "recall": 0.7671232876712328, "f1": 0.7567567567567568, "support": 73 }, "1": { "precision": 0.8585365853658536, "recall": 0.822429906542056, "f1": 0.8400954653937948, "support": 214 }, "2": { "precision": 0.8650887573964497, "recall": 0.857981220657277, "f1": 0.8615203299941072, "support": 852 }, "3": { "precision": 0.8378378378378378, "recall": 0.8051948051948052, "f1": 0.8211920529801324, "support": 77 }, "4": { "precision": 0.9494949494949495, "recall": 0.9690721649484536, "f1": 0.9591836734693877, "support": 97 }, "5": { "precision": 0.9744680851063829, "recall": 0.9462809917355371, "f1": 0.960167714884696, "support": 242 }, "6": { "precision": 0.7730061349693251, "recall": 0.863013698630137, "f1": 0.8155339805825242, "support": 146 }, "7": { "precision": 0.8305084745762712, "recall": 0.91875, "f1": 0.8724035608308606, "support": 160 }, "8": { "precision": 0.7073170731707317, "recall": 0.90625, "f1": 0.7945205479452054, "support": 32 }, "9": { "precision": 0.6902173913043478, "recall": 0.7559523809523809, "f1": 0.7215909090909091, "support": 336 }, "10": { "precision": 0.47368421052631576, "recall": 0.6923076923076923, "f1": 0.5625, "support": 13 }, "11": { "precision": 0.7647058823529411, "recall": 0.9285714285714286, "f1": 0.8387096774193549, "support": 14 }, "12": { "precision": 0.8378378378378378, "recall": 0.7815126050420168, "f1": 0.808695652173913, "support": 119 }, "13": { "precision": 0.8235294117647058, "recall": 0.7241379310344828, "f1": 0.7706422018348624, "support": 116 }, "14": { "precision": 0.8372093023255814, "recall": 0.7807228915662651, "f1": 0.8079800498753117, "support": 415 }, "15": { "precision": 0.8521739130434782, "recall": 0.784, "f1": 0.8166666666666667, "support": 125 }, "16": { "precision": 0.9012345679012346, "recall": 0.8795180722891566, "f1": 0.8902439024390244, "support": 249 }, "17": { "precision": 0.896551724137931, "recall": 0.9285714285714286, "f1": 0.9122807017543859, "support": 112 }, "18": { "precision": 0.8798449612403101, "recall": 0.8598484848484849, "f1": 0.8697318007662835, "support": 528 }, "19": { "precision": 0.8086124401913876, "recall": 0.8578680203045685, "f1": 0.8325123152709359, "support": 197 } }, "confusion": { "0": { "0": 56, "1": 0, "2": 1, "3": 0, "4": 0, "5": 1, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 2, "15": 1, "16": 0, "17": 0, "18": 4, "19": 8, "INVALID": 0 }, "1": { "0": 1, "1": 176, "2": 1, "3": 4, "4": 0, "5": 0, "6": 1, "7": 0, "8": 2, "9": 4, "10": 0, "11": 0, "12": 3, "13": 0, "14": 15, "15": 2, "16": 3, "17": 2, "18": 0, "19": 0, "INVALID": 0 }, "2": { "0": 3, "1": 3, "2": 731, "3": 5, "4": 3, "5": 2, "6": 9, "7": 12, "8": 1, "9": 20, "10": 4, "11": 2, "12": 6, "13": 12, "14": 7, "15": 1, "16": 0, "17": 8, "18": 23, "19": 0, "INVALID": 0 }, "3": { "0": 3, "1": 2, "2": 1, "3": 62, "4": 0, "5": 0, "6": 2, "7": 0, "8": 0, "9": 3, "10": 0, "11": 0, "12": 0, "13": 1, "14": 0, "15": 1, "16": 0, "17": 0, "18": 2, "19": 0, "INVALID": 0 }, "4": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 94, "5": 2, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "5": { "0": 1, "1": 0, "2": 2, "3": 0, "4": 0, "5": 229, "6": 0, "7": 6, "8": 0, "9": 1, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 3, "19": 0, "INVALID": 0 }, "6": { "0": 1, "1": 0, "2": 2, "3": 0, "4": 0, "5": 0, "6": 126, "7": 1, "8": 0, "9": 4, "10": 1, "11": 0, "12": 0, "13": 0, "14": 8, "15": 1, "16": 0, "17": 1, "18": 1, "19": 0, "INVALID": 0 }, "7": { "0": 2, "1": 2, "2": 7, "3": 0, "4": 0, "5": 0, "6": 1, "7": 147, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "8": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 29, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 1, "16": 0, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "9": { "0": 1, "1": 6, "2": 21, "3": 2, "4": 0, "5": 0, "6": 4, "7": 0, "8": 5, "9": 254, "10": 1, "11": 0, "12": 7, "13": 1, "14": 18, "15": 3, "16": 10, "17": 1, "18": 2, "19": 0, "INVALID": 0 }, "10": { "0": 1, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 2, "7": 0, "8": 0, "9": 0, "10": 9, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "11": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 1, "10": 0, "11": 13, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "12": { "0": 0, "1": 1, "2": 7, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 13, "10": 0, "11": 0, "12": 93, "13": 0, "14": 1, "15": 0, "16": 3, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "13": { "0": 0, "1": 0, "2": 23, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 1, "10": 0, "11": 1, "12": 1, "13": 84, "14": 0, "15": 0, "16": 0, "17": 0, "18": 6, "19": 0, "INVALID": 0 }, "14": { "0": 2, "1": 10, "2": 12, "3": 0, "4": 0, "5": 0, "6": 14, "7": 3, "8": 3, "9": 33, "10": 0, "11": 1, "12": 1, "13": 0, "14": 324, "15": 4, "16": 8, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "15": { "0": 1, "1": 1, "2": 3, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 1, "10": 0, "11": 0, "12": 0, "13": 0, "14": 3, "15": 98, "16": 0, "17": 0, "18": 9, "19": 9, "INVALID": 0 }, "16": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "5": 0, "6": 1, "7": 0, "8": 0, "9": 21, "10": 0, "11": 0, "12": 0, "13": 0, "14": 5, "15": 0, "16": 219, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "17": { "0": 0, "1": 0, "2": 8, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 104, "18": 0, "19": 0, "INVALID": 0 }, "18": { "0": 2, "1": 1, "2": 18, "3": 1, "4": 2, "5": 1, "6": 2, "7": 3, "8": 1, "9": 10, "10": 3, "11": 0, "12": 0, "13": 3, "14": 2, "15": 2, "16": 0, "17": 0, "18": 454, "19": 23, "INVALID": 0 }, "19": { "0": 1, "1": 0, "2": 6, "3": 0, "4": 0, "5": 0, "6": 1, "7": 5, "8": 0, "9": 2, "10": 1, "11": 0, "12": 0, "13": 1, "14": 1, "15": 1, "16": 0, "17": 0, "18": 9, "19": 169, "INVALID": 0 } }, "invalid": 0, "latency": { "requests": 4117, "timed_requests": 4117, "p50_ms": 9.518680046312511, "p95_ms": 9.69213501084596, "p99_ms": 9.956609806977212, "total_request_seconds": 39.19670115597546, "attempted_decisions_per_second": 105.03434928406905 } }, "base_laya": { "cases": 4117, "accuracy": 0.25965508865678893, "macro_f1": 0.31094202991841874, "per_class": { "0": { "precision": 0.25, "recall": 0.0136986301369863, "f1": 0.025974025974025976, "support": 73 }, "1": { "precision": 0.8429752066115702, "recall": 0.4766355140186916, "f1": 0.608955223880597, "support": 214 }, "2": { "precision": 0.7709923664122137, "recall": 0.11854460093896714, "f1": 0.20549338758901323, "support": 852 }, "3": { "precision": 0.5625, "recall": 0.11688311688311688, "f1": 0.1935483870967742, "support": 77 }, "4": { "precision": 0.8804347826086957, "recall": 0.8350515463917526, "f1": 0.8571428571428571, "support": 97 }, "5": { "precision": 0.5056818181818182, "recall": 0.7355371900826446, "f1": 0.5993265993265994, "support": 242 }, "6": { "precision": 0.34196891191709844, "recall": 0.4520547945205479, "f1": 0.3893805309734513, "support": 146 }, "7": { "precision": 0.023873873873873873, "recall": 0.33125, "f1": 0.04453781512605042, "support": 160 }, "8": { "precision": 0.17647058823529413, "recall": 0.5625, "f1": 0.26865671641791045, "support": 32 }, "9": { "precision": 0.8888888888888888, "recall": 0.023809523809523808, "f1": 0.0463768115942029, "support": 336 }, "10": { "precision": 0.1111111111111111, "recall": 0.07692307692307693, "f1": 0.09090909090909091, "support": 13 }, "11": { "precision": 0.2857142857142857, "recall": 0.42857142857142855, "f1": 0.34285714285714286, "support": 14 }, "12": { "precision": 0.35714285714285715, "recall": 0.08403361344537816, "f1": 0.1360544217687075, "support": 119 }, "13": { "precision": 0.5740740740740741, "recall": 0.5344827586206896, "f1": 0.5535714285714286, "support": 116 }, "14": { "precision": 0.5773195876288659, "recall": 0.26987951807228916, "f1": 0.367816091954023, "support": 415 }, "15": { "precision": 0.1891891891891892, "recall": 0.112, "f1": 0.1407035175879397, "support": 125 }, "16": { "precision": 0.7703349282296651, "recall": 0.6465863453815262, "f1": 0.7030567685589519, "support": 249 }, "17": { "precision": 0.625, "recall": 0.22321428571428573, "f1": 0.32894736842105265, "support": 112 }, "18": { "precision": 0.35714285714285715, "recall": 0.00946969696969697, "f1": 0.01845018450184502, "support": 528 }, "19": { "precision": 0.3111111111111111, "recall": 0.28426395939086296, "f1": 0.29708222811671087, "support": 197 } }, "confusion": { "0": { "0": 1, "1": 0, "2": 2, "3": 0, "4": 0, "5": 11, "6": 0, "7": 53, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 2, "19": 4, "INVALID": 0 }, "1": { "0": 0, "1": 102, "2": 0, "3": 0, "4": 0, "5": 0, "6": 3, "7": 49, "8": 15, "9": 0, "10": 0, "11": 0, "12": 1, "13": 0, "14": 42, "15": 0, "16": 1, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "2": { "0": 1, "1": 0, "2": 101, "3": 2, "4": 1, "5": 20, "6": 28, "7": 604, "8": 17, "9": 0, "10": 6, "11": 13, "12": 8, "13": 14, "14": 0, "15": 10, "16": 0, "17": 8, "18": 3, "19": 16, "INVALID": 0 }, "3": { "0": 0, "1": 1, "2": 0, "3": 9, "4": 0, "5": 0, "6": 3, "7": 55, "8": 3, "9": 0, "10": 0, "11": 0, "12": 0, "13": 1, "14": 3, "15": 2, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 81, "5": 1, "6": 1, "7": 11, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 1, "INVALID": 0 }, "5": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 178, "6": 0, "7": 62, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 1, "16": 0, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "6": { "0": 1, "1": 0, "2": 0, "3": 1, "4": 0, "5": 1, "6": 66, "7": 54, "8": 3, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 10, "15": 3, "16": 0, "17": 0, "18": 0, "19": 7, "INVALID": 0 }, "7": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 97, "6": 5, "7": 53, "8": 2, "9": 0, "10": 1, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 2, "INVALID": 0 }, "8": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "5": 0, "6": 1, "7": 7, "8": 18, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 4, "15": 1, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "9": { "0": 0, "1": 3, "2": 2, "3": 1, "4": 1, "5": 0, "6": 9, "7": 245, "8": 13, "9": 8, "10": 0, "11": 0, "12": 6, "13": 10, "14": 14, "15": 1, "16": 19, "17": 3, "18": 0, "19": 1, "INVALID": 0 }, "10": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 1, "6": 2, "7": 3, "8": 4, "9": 0, "10": 1, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 2, "INVALID": 0 }, "11": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 6, "8": 1, "9": 0, "10": 0, "11": 6, "12": 0, "13": 1, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "12": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "5": 3, "6": 1, "7": 83, "8": 1, "9": 0, "10": 0, "11": 1, "12": 10, "13": 9, "14": 0, "15": 2, "16": 4, "17": 0, "18": 1, "19": 1, "INVALID": 0 }, "13": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "5": 0, "6": 3, "7": 43, "8": 1, "9": 0, "10": 0, "11": 0, "12": 0, "13": 62, "14": 0, "15": 0, "16": 0, "17": 1, "18": 0, "19": 3, "INVALID": 0 }, "14": { "0": 0, "1": 6, "2": 1, "3": 1, "4": 0, "5": 3, "6": 28, "7": 223, "8": 5, "9": 0, "10": 1, "11": 0, "12": 1, "13": 0, "14": 112, "15": 4, "16": 24, "17": 1, "18": 0, "19": 5, "INVALID": 0 }, "15": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "5": 2, "6": 0, "7": 83, "8": 2, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 3, "15": 14, "16": 0, "17": 0, "18": 1, "19": 19, "INVALID": 0 }, "16": { "0": 0, "1": 5, "2": 0, "3": 0, "4": 1, "5": 0, "6": 4, "7": 65, "8": 3, "9": 1, "10": 0, "11": 0, "12": 2, "13": 0, "14": 5, "15": 0, "16": 161, "17": 2, "18": 0, "19": 0, "INVALID": 0 }, "17": { "0": 0, "1": 0, "2": 2, "3": 0, "4": 0, "5": 0, "6": 3, "7": 82, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 25, "18": 0, "19": 0, "INVALID": 0 }, "18": { "0": 1, "1": 2, "2": 7, "3": 0, "4": 8, "5": 13, "6": 30, "7": 341, "8": 12, "9": 0, "10": 0, "11": 1, "12": 0, "13": 11, "14": 1, "15": 33, "16": 0, "17": 0, "18": 5, "19": 63, "INVALID": 0 }, "19": { "0": 0, "1": 0, "2": 10, "3": 0, "4": 0, "5": 22, "6": 6, "7": 98, "8": 2, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 3, "16": 0, "17": 0, "18": 0, "19": 56, "INVALID": 0 } }, "invalid": 0, "latency": { "requests": 4117, "timed_requests": 4117, "p50_ms": 9.521719999611378, "p95_ms": 9.661309176590294, "p99_ms": 9.913551851641387, "total_request_seconds": 39.17916601651814, "attempted_decisions_per_second": 105.0813587574644 } }, "same_dataset_finetune": { "cases": 4117, "accuracy": 0.9081855720184601, "macro_f1": 0.899856257900767, "per_class": { "0": { "precision": 0.9193548387096774, "recall": 0.7808219178082192, "f1": 0.8444444444444444, "support": 73 }, "1": { "precision": 0.92, "recall": 0.8598130841121495, "f1": 0.8888888888888888, "support": 214 }, "2": { "precision": 0.9265402843601895, "recall": 0.9178403755868545, "f1": 0.9221698113207547, "support": 852 }, "3": { "precision": 0.9041095890410958, "recall": 0.8571428571428571, "f1": 0.88, "support": 77 }, "4": { "precision": 0.9595959595959596, "recall": 0.979381443298969, "f1": 0.9693877551020408, "support": 97 }, "5": { "precision": 0.9591836734693877, "recall": 0.9710743801652892, "f1": 0.9650924024640657, "support": 242 }, "6": { "precision": 0.9154929577464789, "recall": 0.8904109589041096, "f1": 0.9027777777777778, "support": 146 }, "7": { "precision": 0.8947368421052632, "recall": 0.95625, "f1": 0.9244712990936556, "support": 160 }, "8": { "precision": 0.8529411764705882, "recall": 0.90625, "f1": 0.8787878787878788, "support": 32 }, "9": { "precision": 0.8114285714285714, "recall": 0.8452380952380952, "f1": 0.8279883381924198, "support": 336 }, "10": { "precision": 0.75, "recall": 0.9230769230769231, "f1": 0.8275862068965517, "support": 13 }, "11": { "precision": 0.875, "recall": 1.0, "f1": 0.9333333333333333, "support": 14 }, "12": { "precision": 0.8849557522123894, "recall": 0.8403361344537815, "f1": 0.8620689655172413, "support": 119 }, "13": { "precision": 0.9196428571428571, "recall": 0.8879310344827587, "f1": 0.9035087719298246, "support": 116 }, "14": { "precision": 0.8707482993197279, "recall": 0.9253012048192771, "f1": 0.897196261682243, "support": 415 }, "15": { "precision": 0.9032258064516129, "recall": 0.896, "f1": 0.8995983935742972, "support": 125 }, "16": { "precision": 0.9570815450643777, "recall": 0.8955823293172691, "f1": 0.9253112033195021, "support": 249 }, "17": { "precision": 0.8839285714285714, "recall": 0.8839285714285714, "f1": 0.8839285714285714, "support": 112 }, "18": { "precision": 0.9300567107750473, "recall": 0.9318181818181818, "f1": 0.9309366130558183, "support": 528 }, "19": { "precision": 0.9203980099502488, "recall": 0.9390862944162437, "f1": 0.9296482412060302, "support": 197 } }, "confusion": { "0": { "0": 57, "1": 0, "2": 2, "3": 0, "4": 0, "5": 0, "6": 0, "7": 1, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 1, "14": 2, "15": 0, "16": 0, "17": 0, "18": 3, "19": 7, "INVALID": 0 }, "1": { "0": 1, "1": 184, "2": 0, "3": 3, "4": 0, "5": 0, "6": 0, "7": 0, "8": 1, "9": 3, "10": 0, "11": 0, "12": 0, "13": 0, "14": 16, "15": 1, "16": 2, "17": 3, "18": 0, "19": 0, "INVALID": 0 }, "2": { "0": 1, "1": 2, "2": 782, "3": 1, "4": 3, "5": 2, "6": 2, "7": 7, "8": 0, "9": 14, "10": 0, "11": 1, "12": 4, "13": 2, "14": 4, "15": 0, "16": 2, "17": 9, "18": 15, "19": 1, "INVALID": 0 }, "3": { "0": 0, "1": 0, "2": 3, "3": 66, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 3, "10": 0, "11": 0, "12": 1, "13": 1, "14": 0, "15": 1, "16": 0, "17": 0, "18": 2, "19": 0, "INVALID": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 95, "5": 2, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "5": { "0": 0, "1": 0, "2": 1, "3": 0, "4": 1, "5": 235, "6": 0, "7": 4, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 1, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "6": { "0": 1, "1": 0, "2": 3, "3": 0, "4": 0, "5": 0, "6": 130, "7": 1, "8": 0, "9": 2, "10": 1, "11": 0, "12": 0, "13": 1, "14": 7, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "7": { "0": 0, "1": 0, "2": 4, "3": 0, "4": 0, "5": 3, "6": 0, "7": 153, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "8": { "0": 0, "1": 1, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 29, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 1, "19": 1, "INVALID": 0 }, "9": { "0": 0, "1": 7, "2": 8, "3": 1, "4": 0, "5": 1, "6": 2, "7": 0, "8": 2, "9": 284, "10": 0, "11": 0, "12": 5, "13": 1, "14": 18, "15": 2, "16": 4, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "10": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 1, "7": 0, "8": 0, "9": 0, "10": 12, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "11": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 14, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "12": { "0": 0, "1": 2, "2": 5, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 6, "10": 0, "11": 0, "12": 100, "13": 2, "14": 1, "15": 0, "16": 2, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "13": { "0": 0, "1": 0, "2": 9, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 1, "12": 1, "13": 103, "14": 0, "15": 0, "16": 0, "17": 0, "18": 2, "19": 0, "INVALID": 0 }, "14": { "0": 0, "1": 1, "2": 5, "3": 0, "4": 0, "5": 0, "6": 6, "7": 2, "8": 2, "9": 14, "10": 0, "11": 0, "12": 0, "13": 0, "14": 384, "15": 1, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "15": { "0": 0, "1": 0, "2": 0, "3": 2, "4": 0, "5": 0, "6": 0, "7": 2, "8": 0, "9": 1, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 112, "16": 0, "17": 0, "18": 6, "19": 1, "INVALID": 0 }, "16": { "0": 0, "1": 2, "2": 1, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 15, "10": 0, "11": 0, "12": 0, "13": 0, "14": 6, "15": 0, "16": 223, "17": 1, "18": 1, "19": 0, "INVALID": 0 }, "17": { "0": 0, "1": 0, "2": 12, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 1, "13": 0, "14": 0, "15": 0, "16": 0, "17": 99, "18": 0, "19": 0, "INVALID": 0 }, "18": { "0": 2, "1": 1, "2": 7, "3": 0, "4": 0, "5": 2, "6": 0, "7": 0, "8": 0, "9": 8, "10": 2, "11": 0, "12": 1, "13": 1, "14": 1, "15": 5, "16": 0, "17": 0, "18": 492, "19": 6, "INVALID": 0 }, "19": { "0": 0, "1": 0, "2": 2, "3": 0, "4": 0, "5": 0, "6": 1, "7": 1, "8": 0, "9": 0, "10": 1, "11": 0, "12": 0, "13": 0, "14": 1, "15": 1, "16": 0, "17": 0, "18": 5, "19": 185, "INVALID": 0 } }, "out_of_task_or_abstained_predictions": 0, "model": { "repo_id": "leonas5555/finnews-topic-single-classify", "dataset": "zeroshot/twitter-financial-news-topic", "max_length": 256, "label_policy": "Use the documented 0–19 topic IDs without remapping from observed test labels.", "split_status": "Card describes training on this corpus and validation-based selection but provides no row IDs; independent held-out status is unverified.", "mapping": { "0": "0", "1": "1", "2": "2", "3": "3", "4": "4", "5": "5", "6": "6", "7": "7", "8": "8", "9": "9", "10": "10", "11": "11", "12": "12", "13": "13", "14": "14", "15": "15", "16": "16", "17": "17", "18": "18", "19": "19" }, "reported_accuracy": 0.907943, "revision": "57c632acdedfe5ae4f45836030c99a890e05d22a", "directory": "data/specializations/comparators/leonas5555--finnews-topic-single-classify", "id2label": { "0": "Analyst Update", "1": "Fed | Central Banks", "2": "Company | Product News", "3": "Treasuries | Corporate Debt", "4": "Dividend", "5": "Earnings", "6": "Energy | Oil", "7": "Financials", "8": "Currencies", "9": "General News | Opinion", "10": "Gold | Metals | Materials", "11": "IPO", "12": "Legal | Regulation", "13": "M&A | Investments", "14": "Macro", "15": "Markets", "16": "Politics", "17": "Personnel Change", "18": "Stock Commentary", "19": "Stock Movement" }, "model_card_url": "https://huggingface.co/leonas5555/finnews-topic-single-classify/blob/57c632acdedfe5ae4f45836030c99a890e05d22a/README.md", "status": "selected; local evaluation pending", "metadata_sha256": { "README.md": "3a6e7802889fd5d65acce65f49dab3c249dfb0b77b040144bc715d9ffa57fe0e", "config.json": "66145ca2119a54b2541b6961071fc4c6b9d27c8ca2e4c873ea40bce9f20912c0" } }, "manifest": { "format_version": 1, "sha256": "cf0e37b7479ebb3cb193d0b3e81237d3fbb7c0233cf804fa458608c486fda117", "cases": 4117, "decisions": 4117, "profile": "specialization-v1", "source": { "id": "zeroshot/twitter-financial-news-topic", "revision": "acbc8af2a35ccf0916124efcbe9e6cf25f191012", "license_metadata": [ "mit" ], "url": "https://huggingface.co/datasets/zeroshot/twitter-financial-news-topic", "metadata_only": true }, "split_seed": 42, "candidate_policy": "all 20 labels, fixed order, no gold-dependent shortlist", "grouping": { "method": "connected components; char_wb TF-IDF 3–5 grams; cosine >= threshold", "threshold": 0.9, "edges": 9933, "groups": 18071, "largest_group": 99, "labels_used_for_similarity": false, "limitation": "source template IDs are unavailable; semantic paraphrase independence is unverified" }, "source_file_sha256": { "topic_train.csv": "63b9ea8183728dd5ddc6a57289341556509ddc0a8d037616519319a316b54490", "topic_valid.csv": "67396d6cd4a21c3755de783f20fb5842f12ae8edb9367eed5d2644dd85ca9243" }, "role": "test", "label_counts": { "0": 73, "2": 852, "9": 336, "8": 32, "4": 97, "5": 242, "6": 146, "1": 214, "7": 160, "10": 13, "11": 14, "12": 119, "13": 116, "14": 415, "15": 125, "17": 112, "16": 249, "18": 528, "19": 197, "3": 77 } }, "device": "cuda:1", "dtype": "float32", "batch_size": 32, "truncated_inputs": 0, "successful_predictions": 4117, "forward_seconds": 1.513033033173997, "forward_timing_includes_warmup": true, "same_input_comparison": true, "common_unseen_test_verified": false, "parameters": 109767188, "weights_sha256": "cce36979f172a9dc688001bf796cba2b4dd992219aa74240a0072e8ce44fdd55" }, "simple_comparator": { "cases": 4117, "accuracy": 0.7779936847218849, "macro_f1": 0.7245496546076804, "per_class": { "0": { "precision": 0.975609756097561, "recall": 0.547945205479452, "f1": 0.7017543859649122, "support": 73 }, "1": { "precision": 0.8176795580110497, "recall": 0.6915887850467289, "f1": 0.7493670886075949, "support": 214 }, "2": { "precision": 0.709409594095941, "recall": 0.9025821596244131, "f1": 0.7944214876033058, "support": 852 }, "3": { "precision": 0.9166666666666666, "recall": 0.42857142857142855, "f1": 0.584070796460177, "support": 77 }, "4": { "precision": 0.9795918367346939, "recall": 0.9896907216494846, "f1": 0.9846153846153847, "support": 97 }, "5": { "precision": 0.9613733905579399, "recall": 0.9256198347107438, "f1": 0.9431578947368421, "support": 242 }, "6": { "precision": 0.7622950819672131, "recall": 0.636986301369863, "f1": 0.6940298507462687, "support": 146 }, "7": { "precision": 0.8851351351351351, "recall": 0.81875, "f1": 0.8506493506493507, "support": 160 }, "8": { "precision": 0.8333333333333334, "recall": 0.46875, "f1": 0.6, "support": 32 }, "9": { "precision": 0.5935162094763092, "recall": 0.7083333333333334, "f1": 0.6458616010854816, "support": 336 }, "10": { "precision": 0.4, "recall": 0.15384615384615385, "f1": 0.2222222222222222, "support": 13 }, "11": { "precision": 1.0, "recall": 0.35714285714285715, "f1": 0.5263157894736842, "support": 14 }, "12": { "precision": 0.9078947368421053, "recall": 0.5798319327731093, "f1": 0.7076923076923077, "support": 119 }, "13": { "precision": 0.8780487804878049, "recall": 0.6206896551724138, "f1": 0.7272727272727273, "support": 116 }, "14": { "precision": 0.7073732718894009, "recall": 0.7397590361445783, "f1": 0.7232037691401649, "support": 415 }, "15": { "precision": 0.8247422680412371, "recall": 0.64, "f1": 0.7207207207207207, "support": 125 }, "16": { "precision": 0.8922413793103449, "recall": 0.8313253012048193, "f1": 0.8607068607068608, "support": 249 }, "17": { "precision": 0.9456521739130435, "recall": 0.7767857142857143, "f1": 0.8529411764705882, "support": 112 }, "18": { "precision": 0.7875, "recall": 0.8352272727272727, "f1": 0.8106617647058824, "support": 528 }, "19": { "precision": 0.8488372093023255, "recall": 0.7411167512690355, "f1": 0.7913279132791328, "support": 197 } }, "confusion": { "0": { "0": 40, "1": 0, "2": 13, "3": 0, "4": 0, "5": 2, "6": 0, "7": 1, "8": 0, "9": 2, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 2, "16": 0, "17": 0, "18": 7, "19": 5, "INVALID": 0 }, "1": { "0": 0, "1": 148, "2": 8, "3": 1, "4": 0, "5": 0, "6": 3, "7": 0, "8": 0, "9": 11, "10": 0, "11": 0, "12": 1, "13": 0, "14": 29, "15": 1, "16": 6, "17": 3, "18": 3, "19": 0, "INVALID": 0 }, "2": { "0": 0, "1": 2, "2": 769, "3": 0, "4": 1, "5": 1, "6": 6, "7": 4, "8": 0, "9": 22, "10": 0, "11": 0, "12": 3, "13": 3, "14": 10, "15": 0, "16": 1, "17": 2, "18": 24, "19": 4, "INVALID": 0 }, "3": { "0": 0, "1": 3, "2": 7, "3": 33, "4": 0, "5": 0, "6": 3, "7": 0, "8": 1, "9": 12, "10": 0, "11": 0, "12": 0, "13": 1, "14": 6, "15": 3, "16": 1, "17": 0, "18": 6, "19": 1, "INVALID": 0 }, "4": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 96, "5": 1, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0, "INVALID": 0 }, "5": { "0": 0, "1": 0, "2": 5, "3": 0, "4": 1, "5": 224, "6": 0, "7": 3, "8": 0, "9": 1, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 0, "16": 0, "17": 0, "18": 6, "19": 1, "INVALID": 0 }, "6": { "0": 0, "1": 0, "2": 10, "3": 0, "4": 0, "5": 1, "6": 93, "7": 0, "8": 0, "9": 4, "10": 1, "11": 0, "12": 0, "13": 1, "14": 27, "15": 3, "16": 1, "17": 0, "18": 5, "19": 0, "INVALID": 0 }, "7": { "0": 0, "1": 1, "2": 16, "3": 0, "4": 0, "5": 3, "6": 1, "7": 131, "8": 0, "9": 0, "10": 1, "11": 0, "12": 0, "13": 0, "14": 2, "15": 0, "16": 0, "17": 0, "18": 4, "19": 1, "INVALID": 0 }, "8": { "0": 0, "1": 6, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 15, "9": 4, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 1, "16": 0, "17": 0, "18": 5, "19": 0, "INVALID": 0 }, "9": { "0": 0, "1": 4, "2": 52, "3": 0, "4": 0, "5": 0, "6": 3, "7": 0, "8": 2, "9": 238, "10": 0, "11": 0, "12": 1, "13": 1, "14": 18, "15": 1, "16": 9, "17": 0, "18": 7, "19": 0, "INVALID": 0 }, "10": { "0": 0, "1": 1, "2": 1, "3": 0, "4": 0, "5": 0, "6": 1, "7": 0, "8": 0, "9": 1, "10": 2, "11": 0, "12": 0, "13": 0, "14": 4, "15": 2, "16": 0, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "11": { "0": 0, "1": 0, "2": 3, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 5, "10": 0, "11": 5, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "12": { "0": 0, "1": 1, "2": 24, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 16, "10": 1, "11": 0, "12": 69, "13": 2, "14": 2, "15": 0, "16": 3, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "13": { "0": 0, "1": 0, "2": 32, "3": 0, "4": 0, "5": 0, "6": 1, "7": 0, "8": 0, "9": 4, "10": 0, "11": 0, "12": 0, "13": 72, "14": 0, "15": 0, "16": 0, "17": 0, "18": 7, "19": 0, "INVALID": 0 }, "14": { "0": 0, "1": 8, "2": 33, "3": 1, "4": 0, "5": 0, "6": 8, "7": 1, "8": 0, "9": 39, "10": 0, "11": 0, "12": 0, "13": 0, "14": 307, "15": 0, "16": 4, "17": 0, "18": 13, "19": 1, "INVALID": 0 }, "15": { "0": 0, "1": 4, "2": 4, "3": 1, "4": 0, "5": 0, "6": 1, "7": 1, "8": 0, "9": 5, "10": 0, "11": 0, "12": 0, "13": 0, "14": 11, "15": 80, "16": 0, "17": 0, "18": 15, "19": 3, "INVALID": 0 }, "16": { "0": 0, "1": 2, "2": 13, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 17, "10": 0, "11": 0, "12": 2, "13": 0, "14": 7, "15": 0, "16": 207, "17": 0, "18": 1, "19": 0, "INVALID": 0 }, "17": { "0": 0, "1": 0, "2": 22, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 2, "10": 0, "11": 0, "12": 0, "13": 0, "14": 1, "15": 0, "16": 0, "17": 87, "18": 0, "19": 0, "INVALID": 0 }, "18": { "0": 0, "1": 1, "2": 50, "3": 0, "4": 0, "5": 0, "6": 1, "7": 0, "8": 0, "9": 18, "10": 0, "11": 0, "12": 0, "13": 0, "14": 4, "15": 3, "16": 0, "17": 0, "18": 441, "19": 10, "INVALID": 0 }, "19": { "0": 1, "1": 0, "2": 22, "3": 0, "4": 0, "5": 1, "6": 1, "7": 7, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 2, "14": 3, "15": 1, "16": 0, "17": 0, "18": 13, "19": 146, "INVALID": 0 } }, "invalid": 0 }, "retention": { "typed-decisions": 0.3565, "ag-news": 0.9383333333333334, "emotion": 0.56, "boolq": 0.8283333333333334, "sst5": 0.38666666666666666, "prompt-injections": 0.7068965517241379, "massive-intent.en": 0.7833333333333333, "xnli.en": 0.8633333333333333 }, "independent_external_test": false }