| [ |
| { |
| "variant": "title", |
| "feature_dim": 1537, |
| "train_rows": 2584, |
| "test_rows": 961, |
| "results": [ |
| { |
| "classifier": "logistic_regression", |
| "accuracy": 0.7741935483870968, |
| "macro_f1": 0.48005410270610865, |
| "report": { |
| "NEI": { |
| "precision": 0.9165545087483177, |
| "recall": 0.9055851063829787, |
| "f1-score": 0.9110367892976589, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.14634146341463414, |
| "recall": 0.16901408450704225, |
| "f1-score": 0.1568627450980392, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.375, |
| "recall": 0.3695652173913043, |
| "f1-score": 0.3722627737226277, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7741935483870968, |
| "macro avg": { |
| "precision": 0.4792986573876506, |
| "recall": 0.4813881360937751, |
| "f1-score": 0.48005410270610865, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7818826581489844, |
| "recall": 0.7741935483870968, |
| "f1-score": 0.7779492021098052, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "linear_svm", |
| "accuracy": 0.7336108220603538, |
| "macro_f1": 0.46400053124097934, |
| "report": { |
| "NEI": { |
| "precision": 0.9194244604316547, |
| "recall": 0.8497340425531915, |
| "f1-score": 0.8832066344160332, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.12612612612612611, |
| "recall": 0.19718309859154928, |
| "f1-score": 0.15384615384615385, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.33548387096774196, |
| "recall": 0.37681159420289856, |
| "f1-score": 0.35494880546075086, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7336108220603538, |
| "macro avg": { |
| "precision": 0.4603448191751742, |
| "recall": 0.47457624511587976, |
| "f1-score": 0.46400053124097934, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7769603781405907, |
| "recall": 0.7336108220603538, |
| "f1-score": 0.7534624361680724, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "random_forest", |
| "accuracy": 0.7866805411030177, |
| "macro_f1": 0.3173161100507625, |
| "report": { |
| "NEI": { |
| "precision": 0.7924130663856691, |
| "recall": 1.0, |
| "f1-score": 0.884185773074662, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.25, |
| "recall": 0.014084507042253521, |
| "f1-score": 0.02666666666666667, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.375, |
| "recall": 0.021739130434782608, |
| "f1-score": 0.0410958904109589, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7866805411030177, |
| "macro avg": { |
| "precision": 0.4724710221285564, |
| "recall": 0.34527454582567874, |
| "f1-score": 0.3173161100507625, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.6923981539251021, |
| "recall": 0.7866805411030177, |
| "f1-score": 0.6997630255589921, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "hist_gradient_boosting", |
| "accuracy": 0.8241415192507804, |
| "macro_f1": 0.473575753311604, |
| "report": { |
| "NEI": { |
| "precision": 0.9020807833537332, |
| "recall": 0.9800531914893617, |
| "f1-score": 0.9394518801784576, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.07894736842105263, |
| "recall": 0.04225352112676056, |
| "f1-score": 0.05504587155963303, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.49056603773584906, |
| "recall": 0.37681159420289856, |
| "f1-score": 0.4262295081967213, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8241415192507804, |
| "macro avg": { |
| "precision": 0.49053139650354494, |
| "recall": 0.46637276893967367, |
| "f1-score": 0.473575753311604, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.782172867271019, |
| "recall": 0.8241415192507804, |
| "f1-score": 0.8004118032321349, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "mlp", |
| "accuracy": 0.8147762747138397, |
| "macro_f1": 0.4808120084601326, |
| "report": { |
| "NEI": { |
| "precision": 0.9143576826196473, |
| "recall": 0.9654255319148937, |
| "f1-score": 0.9391979301423027, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.10909090909090909, |
| "recall": 0.08450704225352113, |
| "f1-score": 0.09523809523809523, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.45535714285714285, |
| "recall": 0.3695652173913043, |
| "f1-score": 0.408, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8147762747138397, |
| "macro avg": { |
| "precision": 0.49293524485589973, |
| "recall": 0.4731659305199063, |
| "f1-score": 0.4808120084601326, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7889507987405984, |
| "recall": 0.8147762747138397, |
| "f1-score": 0.8005647744317549, |
| "support": 961.0 |
| } |
| } |
| } |
| ], |
| "best_classifier": "mlp", |
| "best_macro_f1": 0.4808120084601326, |
| "best_accuracy": 0.8147762747138397 |
| }, |
| { |
| "variant": "abstract", |
| "feature_dim": 1537, |
| "train_rows": 2584, |
| "test_rows": 961, |
| "results": [ |
| { |
| "classifier": "logistic_regression", |
| "accuracy": 0.8033298647242456, |
| "macro_f1": 0.5115093142453775, |
| "report": { |
| "NEI": { |
| "precision": 0.924802110817942, |
| "recall": 0.9321808510638298, |
| "f1-score": 0.928476821192053, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.16901408450704225, |
| "recall": 0.16901408450704225, |
| "f1-score": 0.16901408450704225, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.44696969696969696, |
| "recall": 0.427536231884058, |
| "f1-score": 0.43703703703703706, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8033298647242456, |
| "macro avg": { |
| "precision": 0.5135952974315604, |
| "recall": 0.50957705581831, |
| "f1-score": 0.5115093142453775, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8003465197886686, |
| "recall": 0.8033298647242456, |
| "f1-score": 0.8017957134729812, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "linear_svm", |
| "accuracy": 0.7492195629552549, |
| "macro_f1": 0.4689766317485898, |
| "report": { |
| "NEI": { |
| "precision": 0.923728813559322, |
| "recall": 0.8696808510638298, |
| "f1-score": 0.8958904109589041, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.12121212121212122, |
| "recall": 0.16901408450704225, |
| "f1-score": 0.1411764705882353, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.35064935064935066, |
| "recall": 0.391304347826087, |
| "f1-score": 0.3698630136986301, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7492195629552549, |
| "macro avg": { |
| "precision": 0.4651967618069313, |
| "recall": 0.47666642779898627, |
| "f1-score": 0.4689766317485898, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7821433286079928, |
| "recall": 0.7492195629552549, |
| "f1-score": 0.7645933551959121, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "random_forest", |
| "accuracy": 0.7866805411030177, |
| "macro_f1": 0.31749495160115226, |
| "report": { |
| "NEI": { |
| "precision": 0.7949260042283298, |
| "recall": 1.0, |
| "f1-score": 0.8857479387514723, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.14285714285714285, |
| "recall": 0.014084507042253521, |
| "f1-score": 0.02564102564102564, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.375, |
| "recall": 0.021739130434782608, |
| "f1-score": 0.0410958904109589, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7866805411030177, |
| "macro avg": { |
| "precision": 0.43759438236182424, |
| "recall": 0.34527454582567874, |
| "f1-score": 0.31749495160115226, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.6864487120942363, |
| "recall": 0.7866805411030177, |
| "f1-score": 0.7009096728806787, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "hist_gradient_boosting", |
| "accuracy": 0.8293444328824141, |
| "macro_f1": 0.49004755671422334, |
| "report": { |
| "NEI": { |
| "precision": 0.9114713216957606, |
| "recall": 0.9720744680851063, |
| "f1-score": 0.9407979407979408, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.05405405405405406, |
| "recall": 0.028169014084507043, |
| "f1-score": 0.037037037037037035, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.5245901639344263, |
| "recall": 0.463768115942029, |
| "f1-score": 0.49230769230769234, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8293444328824141, |
| "macro avg": { |
| "precision": 0.496705179894747, |
| "recall": 0.48800386603721413, |
| "f1-score": 0.49004755671422334, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7925678609531744, |
| "recall": 0.8293444328824141, |
| "f1-score": 0.8096234574902629, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "mlp", |
| "accuracy": 0.8220603537981269, |
| "macro_f1": 0.491425996254973, |
| "report": { |
| "NEI": { |
| "precision": 0.9211195928753181, |
| "recall": 0.9627659574468085, |
| "f1-score": 0.94148244473342, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.08695652173913043, |
| "recall": 0.056338028169014086, |
| "f1-score": 0.06837606837606838, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.4806201550387597, |
| "recall": 0.4492753623188406, |
| "f1-score": 0.46441947565543074, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8220603537981269, |
| "macro avg": { |
| "precision": 0.49623208988440276, |
| "recall": 0.4894597826448877, |
| "f1-score": 0.491425996254973, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7962345767753032, |
| "recall": 0.8220603537981269, |
| "f1-score": 0.8084697054471198, |
| "support": 961.0 |
| } |
| } |
| } |
| ], |
| "best_classifier": "logistic_regression", |
| "best_macro_f1": 0.5115093142453775, |
| "best_accuracy": 0.8033298647242456 |
| }, |
| { |
| "variant": "title_abstract", |
| "feature_dim": 1537, |
| "train_rows": 2584, |
| "test_rows": 961, |
| "results": [ |
| { |
| "classifier": "logistic_regression", |
| "accuracy": 0.8012486992715921, |
| "macro_f1": 0.5331477034083011, |
| "report": { |
| "NEI": { |
| "precision": 0.9425444596443229, |
| "recall": 0.9162234042553191, |
| "f1-score": 0.9291975724881996, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.2, |
| "recall": 0.23943661971830985, |
| "f1-score": 0.21794871794871795, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.4413793103448276, |
| "recall": 0.463768115942029, |
| "f1-score": 0.45229681978798586, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8012486992715921, |
| "macro avg": { |
| "precision": 0.5279745899963835, |
| "recall": 0.539809379971886, |
| "f1-score": 0.5331477034083011, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8157167309886754, |
| "recall": 0.8012486992715921, |
| "f1-score": 0.8081663835756785, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "linear_svm", |
| "accuracy": 0.7523413111342352, |
| "macro_f1": 0.4806390901572748, |
| "report": { |
| "NEI": { |
| "precision": 0.9381294964028777, |
| "recall": 0.8670212765957447, |
| "f1-score": 0.9011748445058743, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.1320754716981132, |
| "recall": 0.19718309859154928, |
| "f1-score": 0.15819209039548024, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.35625, |
| "recall": 0.41304347826086957, |
| "f1-score": 0.3825503355704698, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7523413111342352, |
| "macro avg": { |
| "precision": 0.47548498936699696, |
| "recall": 0.4924159511493879, |
| "f1-score": 0.4806390901572748, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7950189800057545, |
| "recall": 0.7523413111342352, |
| "f1-score": 0.7718075627421659, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "random_forest", |
| "accuracy": 0.7866805411030177, |
| "macro_f1": 0.32168631731308234, |
| "report": { |
| "NEI": { |
| "precision": 0.7938689217758985, |
| "recall": 0.9986702127659575, |
| "f1-score": 0.8845700824499411, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.125, |
| "recall": 0.014084507042253521, |
| "f1-score": 0.02531645569620253, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.5714285714285714, |
| "recall": 0.028985507246376812, |
| "f1-score": 0.05517241379310345, |
| "support": 138.0 |
| }, |
| "accuracy": 0.7866805411030177, |
| "macro avg": { |
| "precision": 0.49676583106815664, |
| "recall": 0.34724674235152925, |
| "f1-score": 0.32168631731308234, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7125094402004356, |
| "recall": 0.7866805411030177, |
| "f1-score": 0.7019853938191825, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "hist_gradient_boosting", |
| "accuracy": 0.8335067637877211, |
| "macro_f1": 0.5090131959830478, |
| "report": { |
| "NEI": { |
| "precision": 0.9192938209331651, |
| "recall": 0.9694148936170213, |
| "f1-score": 0.9436893203883495, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.1, |
| "recall": 0.056338028169014086, |
| "f1-score": 0.07207207207207207, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.53125, |
| "recall": 0.4927536231884058, |
| "f1-score": 0.5112781954887218, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8335067637877211, |
| "macro avg": { |
| "precision": 0.5168479403110551, |
| "recall": 0.5061688483248137, |
| "f1-score": 0.5090131959830478, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8030400138831845, |
| "recall": 0.8335067637877211, |
| "f1-score": 0.8171986233367321, |
| "support": 961.0 |
| } |
| } |
| }, |
| { |
| "classifier": "mlp", |
| "accuracy": 0.8335067637877211, |
| "macro_f1": 0.5239862361702394, |
| "report": { |
| "NEI": { |
| "precision": 0.9427083333333334, |
| "recall": 0.9627659574468085, |
| "f1-score": 0.9526315789473684, |
| "support": 752.0 |
| }, |
| "REFUTES": { |
| "precision": 0.125, |
| "recall": 0.09859154929577464, |
| "f1-score": 0.11023622047244094, |
| "support": 71.0 |
| }, |
| "SUPPORTS": { |
| "precision": 0.5109489051094891, |
| "recall": 0.5072463768115942, |
| "f1-score": 0.509090909090909, |
| "support": 138.0 |
| }, |
| "accuracy": 0.8335067637877211, |
| "macro avg": { |
| "precision": 0.5262190794809408, |
| "recall": 0.5228679611847258, |
| "f1-score": 0.5239862361702394, |
| "support": 961.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8202940848821813, |
| "recall": 0.8335067637877211, |
| "f1-score": 0.8267016279672318, |
| "support": 961.0 |
| } |
| } |
| } |
| ], |
| "best_classifier": "logistic_regression", |
| "best_macro_f1": 0.5331477034083011, |
| "best_accuracy": 0.8012486992715921 |
| } |
| ] |
