File size: 1,445 Bytes
41e1660 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | """A tiny character-level tokenizer for Turkish names.
Every token is a single character. The vocabulary is built directly from the
names file, so it contains exactly the characters that appear in the data
(29 Turkish letters + the newline "\n", which we use as the start/end-of-name
marker).
Usage:
tok = CharTokenizer.from_file("temiz_isimler.txt")
ids = tok.encode("ali") # -> [..]
tok.decode(ids) # -> "ali"
tok.newline_id # id of "\n", the name separator / stop token
"""
class CharTokenizer:
def __init__(self, chars: list[str]):
self.chars = chars
self.stoi = {ch: i for i, ch in enumerate(chars)} # char -> id
self.itos = {i: ch for i, ch in enumerate(chars)} # id -> char
self.vocab_size = len(chars)
# The newline both separates names and marks end-of-sequence (EOS).
self.newline_id = self.stoi["\n"]
self.eos_id = self.newline_id
@classmethod
def from_file(cls, path: str) -> "CharTokenizer":
text = open(path, encoding="utf-8").read()
if "\n" not in text: # make sure the stop token always exists
text += "\n"
return cls(sorted(set(text)))
def encode(self, s: str) -> list[int]:
return [self.stoi[c] for c in s]
def decode(self, ids: list[int]) -> str:
return "".join(self.itos[i] for i in ids)
|