File size: 380 Bytes
786df5b
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
{
  "name": "latex-tokenizer-v0.1",
  "version": "0.1",
  "vocab_size_config": 131072,
  "vocab_size_trained": 4144,
  "vocab_size_export": 131072,
  "vocab_padded": true,
  "smoke": false,
  "byte_fallback": true,
  "pretrained_load": false,
  "seed": 42,
  "note": "SP pieces from corpus; unused_* pads to 131072 for embedding table. Expand corpus later to fill real pieces."
}