Publish fine-tuned GreBerta results

src/lib/data.ts

@@ -1,6 +1,7 @@
 import { baselineConfusions, burrowsDeltaConfusions, gormanD2Confusions, grebertaConfusions, grebertaSelectedCharConfusions, metreBaselineConfusions, metreBurrowsDeltaConfusions, metreCharNgramConfusions, metreGrebertaConfusions, metreLineConfusions, metreSyntaxConfusions, metreSyntaxPlusMetreConfusions, selectedCharNgramConfusions, type ConfusionSet } from './confusions';
 import { burrowsLogregAuthorAccuracy, burrowsLogregConfusions, metreBurrowsLogregAuthorAccuracy, metreBurrowsLogregConfusions } from './burrowsLogregDiagnostics';
 import { metreTfidfLemmaAuthorAccuracy, metreTfidfLemmaConfusions, tfidfLemmaAuthorAccuracy, tfidfLemmaConfusions } from './tfidfDiagnostics';
+import { grebertaFinetuneAuthorAccuracy, grebertaFinetuneConfusions } from './grebertaFinetuneDiagnostics';
 
 export type TaskKey = 'sentence1' | 'sentence10' | 'sentence50' | 'sentence100';
 
@@ -169,6 +170,16 @@ export const results: ResultRow[] = [
     },
     confusions: grebertaConfusions
   },
+  {
+    model: 'GreBerta (fine-tuned)',
+    url: 'https://github.com/Urdatorn/sphragis_models',
+    dimensionality: '125,991,185 trainable parameters',
+    classifier: 'End-to-end softmax classifier',
+    status: 'Best sentence-1 checkpoint selected by validation macro-F1; larger tasks average constituent logits; test evaluated once',
+    scores: { sentence1: 85.16662012054707, sentence10: 99.33317135437139, sentence50: 100, sentence100: 100 },
+    authorAccuracy: grebertaFinetuneAuthorAccuracy,
+    confusions: grebertaFinetuneConfusions
+  },
   {
     model: 'Char 4-gram',
     url: 'https://github.com/Urdatorn/sphragis_models',

src/lib/grebertaFinetuneDiagnostics.ts

@@ -0,0 +1,47 @@
+import type { ConfusionSet } from './confusions';
+import type { ResultRow } from './data';
+
+export const grebertaFinetuneAuthorAccuracy: ResultRow['authorAccuracy'] = {
+  Xenophon:{sentence1:91.66666666666666,sentence10:100,sentence50:100,sentence100:100},
+  Demosthenes:{sentence1:91.33333333333333,sentence10:100,sentence50:100,sentence100:100},
+  'Homeric-Iliad':{sentence1:90,sentence10:100,sentence50:100,sentence100:100},
+  'Homeric-Odyssey':{sentence1:80.5,sentence10:95,sentence50:100,sentence100:100},
+  Athenaeus:{sentence1:75.5,sentence10:100,sentence50:100,sentence100:100},
+  Sophocles:{sentence1:85.66666666666667,sentence10:100,sentence50:100,sentence100:100},
+  Herodotus:{sentence1:98.42857142857143,sentence10:100,sentence50:100,sentence100:100},
+  'Diodorus Siculus':{sentence1:92,sentence10:100,sentence50:100,sentence100:100},
+  Josephus:{sentence1:81,sentence10:100,sentence50:100,sentence100:100},
+  'Dionysius of Halicarnassus':{sentence1:83,sentence10:100,sentence50:100,sentence100:100},
+  Plutarch:{sentence1:79,sentence10:100,sentence50:100,sentence100:100},
+  Aeschylus:{sentence1:68.66666666666667,sentence10:93.33333333333333,sentence50:100,sentence100:100},
+  Polybius:{sentence1:94.33333333333334,sentence10:100,sentence50:100,sentence100:100},
+  Procopius:{sentence1:89,sentence10:100,sentence50:100,sentence100:100},
+  Aristophanes:{sentence1:80,sentence10:100,sentence50:100,sentence100:100},
+  Thucydides:{sentence1:85,sentence10:100,sentence50:100,sentence100:100},
+  Plato:{sentence1:74,sentence10:100,sentence50:100,sentence100:100}
+};
+
+export const grebertaFinetuneConfusions: ConfusionSet = {
+  sentence1:{
+    Xenophon:{Xenophon:550,'Diodorus Siculus':4,Athenaeus:6,Plato:14,Aristophanes:7,Demosthenes:7,Plutarch:4,Thucydides:4,Procopius:1,Aeschylus:1,'Dionysius of Halicarnassus':1,Polybius:1},
+    Demosthenes:{Demosthenes:274,Xenophon:14,Aristophanes:5,Sophocles:1,Polybius:3,Plato:1,Athenaeus:2},
+    'Homeric-Iliad':{'Homeric-Iliad':630,'Homeric-Odyssey':65,Athenaeus:1,Sophocles:2,Aeschylus:2},
+    'Homeric-Odyssey':{'Homeric-Odyssey':483,'Homeric-Iliad':112,Xenophon:1,Aeschylus:1,Herodotus:2,Plato:1},
+    Athenaeus:{Athenaeus:151,Demosthenes:4,Aeschylus:1,Xenophon:13,Aristophanes:4,Plutarch:7,Plato:3,'Homeric-Odyssey':2,'Diodorus Siculus':4,'Dionysius of Halicarnassus':2,Sophocles:4,Polybius:3,Herodotus:1,Josephus:1},
+    Sophocles:{Sophocles:257,Demosthenes:3,Athenaeus:3,'Homeric-Iliad':4,Xenophon:1,Aeschylus:26,Plato:3,Polybius:1,Herodotus:2},
+    Herodotus:{Herodotus:689,Xenophon:4,'Homeric-Odyssey':1,'Dionysius of Halicarnassus':1,Athenaeus:2,Sophocles:1,Procopius:1,'Homeric-Iliad':1},
+    'Diodorus Siculus':{'Diodorus Siculus':92,Polybius:5,Athenaeus:1,Plutarch:2},
+    Josephus:{Josephus:81,Polybius:4,'Dionysius of Halicarnassus':6,Plutarch:3,Demosthenes:2,Athenaeus:3,Herodotus:1},
+    'Dionysius of Halicarnassus':{'Dionysius of Halicarnassus':83,Plato:1,Plutarch:3,Polybius:3,Demosthenes:2,Herodotus:1,'Diodorus Siculus':1,Athenaeus:4,Thucydides:1,Xenophon:1},
+    Plutarch:{Plutarch:79,'Dionysius of Halicarnassus':3,Athenaeus:5,Xenophon:3,'Diodorus Siculus':3,Sophocles:1,Plato:2,Herodotus:1,Polybius:1,Demosthenes:1,'Homeric-Odyssey':1},
+    Aeschylus:{Aeschylus:206,Sophocles:89,'Homeric-Iliad':1,Aristophanes:1,'Homeric-Odyssey':1,Athenaeus:2},
+    Polybius:{Polybius:283,Demosthenes:2,'Diodorus Siculus':8,'Dionysius of Halicarnassus':1,Josephus:3,Athenaeus:2,Plutarch:1},
+    Procopius:{Procopius:89,'Diodorus Siculus':1,Josephus:2,Xenophon:5,Thucydides:1,Athenaeus:1,Herodotus:1},
+    Aristophanes:{Aristophanes:80,Demosthenes:7,Sophocles:7,Xenophon:3,Athenaeus:1,Aeschylus:1,'Homeric-Odyssey':1},
+    Thucydides:{Thucydides:85,Aeschylus:1,Xenophon:8,'Dionysius of Halicarnassus':1,Procopius:2,'Diodorus Siculus':1,Polybius:1,Athenaeus:1},
+    Plato:{Plato:74,Aeschylus:1,Athenaeus:1,Xenophon:19,Sophocles:3,Aristophanes:1,Demosthenes:1}
+  },
+  sentence10:{Xenophon:{Xenophon:60},Demosthenes:{Demosthenes:30},'Homeric-Iliad':{'Homeric-Iliad':70},'Homeric-Odyssey':{'Homeric-Odyssey':57,'Homeric-Iliad':3},Athenaeus:{Athenaeus:20},Sophocles:{Sophocles:30},Herodotus:{Herodotus:70},'Diodorus Siculus':{'Diodorus Siculus':10},Josephus:{Josephus:10},'Dionysius of Halicarnassus':{'Dionysius of Halicarnassus':10},Plutarch:{Plutarch:10},Aeschylus:{Aeschylus:28,Sophocles:2},Polybius:{Polybius:30},Procopius:{Procopius:10},Aristophanes:{Aristophanes:10},Thucydides:{Thucydides:10},Plato:{Plato:10}},
+  sentence50:{Xenophon:{Xenophon:12},Demosthenes:{Demosthenes:6},'Homeric-Iliad':{'Homeric-Iliad':14},'Homeric-Odyssey':{'Homeric-Odyssey':12},Athenaeus:{Athenaeus:4},Sophocles:{Sophocles:6},Herodotus:{Herodotus:14},'Diodorus Siculus':{'Diodorus Siculus':2},Josephus:{Josephus:2},'Dionysius of Halicarnassus':{'Dionysius of Halicarnassus':2},Plutarch:{Plutarch:2},Aeschylus:{Aeschylus:6},Polybius:{Polybius:6},Procopius:{Procopius:2},Aristophanes:{Aristophanes:2},Thucydides:{Thucydides:2},Plato:{Plato:2}},
+  sentence100:{Xenophon:{Xenophon:6},Demosthenes:{Demosthenes:3},'Homeric-Iliad':{'Homeric-Iliad':7},'Homeric-Odyssey':{'Homeric-Odyssey':6},Athenaeus:{Athenaeus:2},Sophocles:{Sophocles:3},Herodotus:{Herodotus:7},'Diodorus Siculus':{'Diodorus Siculus':1},Josephus:{Josephus:1},'Dionysius of Halicarnassus':{'Dionysius of Halicarnassus':1},Plutarch:{Plutarch:1},Aeschylus:{Aeschylus:3},Polybius:{Polybius:3},Procopius:{Procopius:1},Aristophanes:{Aristophanes:1},Thucydides:{Thucydides:1},Plato:{Plato:1}}
+};