token_train.py

import os
from tokenizer import train_sentencepiece
from config import vocab_size
DATA_DIR = "./training_data"

text_files = [
    os.path.join(DATA_DIR, f)
    for f in os.listdir(DATA_DIR)
    if f.endswith(".txt")
]

print(f"Found {len(text_files)} text files")

if not text_files:
    raise ValueError(
        f"No .txt files found in {DATA_DIR}"
    )

tokenizer = train_sentencepiece(text_files,vocab_size=vocab_size)