Hugging Face's logo

crumb-ai
/
crumb-python-bpe-tokenizer

Model card Files
xet
 Community
ahmed-farhanur-rashid commited on
Commit
acced0b
·
verified ·
1 Parent(s): ce0a424

Upload 5 files

Browse files
Files changed (5) hide show
  1. crumb_metadata.json +38 -0
  2. merges.txt +0 -0
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +69 -0
  5. vocab.json +0 -0
crumb_metadata.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
3
+ "script_version": "v7.4.0",
4
+ "vocab_size": 32768,
5
+ "domain": "python_source_code",
6
+ "corpus": "stack-dedup-python-parquet-27GB",
7
+ "min_frequency": 2,
8
+ "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
9
+ "protected_keywords": 38,
10
+ "protected_dunders": 21,
11
+ "protected_builtins": 73,
12
+ "protected_exceptions": 69,
13
+ "protected_stdlib": 125,
14
+ "protected_ds_tokens": 119,
15
+ "total_never_split": 474,
16
+ "special_tokens": 20,
17
+ "reserved_future_tokens": 4,
18
+ "target_model_sizes": [
19
+ "124M",
20
+ "250M",
21
+ "370M",
22
+ "500M"
23
+ ],
24
+ "fim_tokens": [
25
+ "<fim_prefix>",
26
+ "<fim_suffix>",
27
+ "<fim_middle>"
28
+ ],
29
+ "notebook_tokens": [
30
+ "<cell>",
31
+ "</cell>",
32
+ "<cell_output>",
33
+ "</cell_output>"
34
+ ],
35
+ "comments_in_training": true,
36
+ "docstrings_in_training": true,
37
+ "training_date": "2026-05-11"
38
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "extra_special_tokens": [
6
+ "<func>",
7
+ "</func>",
8
+ "<class>",
9
+ "</class>",
10
+ "<module>",
11
+ "</module>",
12
+ "<cell>",
13
+ "</cell>",
14
+ "<cell_output>",
15
+ "</cell_output>",
16
+ "<traceback>",
17
+ "</traceback>",
18
+ "<fim_prefix>",
19
+ "<fim_suffix>",
20
+ "<fim_middle>",
21
+ "<comment>",
22
+ "<docstring>",
23
+ "</docstring>",
24
+ "<type_hint>"
25
+ ],
26
+ "mask_token": "<mask>",
27
+ "model_max_length": 1000000000000000019884624838656,
28
+ "pad_token": "<pad>",
29
+ "tokenizer_class": "TokenizersBackend",
30
+ "unk_token": "<unk>",
31
+ "crumb_metadata": {
32
+ "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
33
+ "script_version": "v7.4.0",
34
+ "vocab_size": 32768,
35
+ "domain": "python_source_code",
36
+ "corpus": "stack-dedup-python-parquet-27GB",
37
+ "min_frequency": 2,
38
+ "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
39
+ "protected_keywords": 38,
40
+ "protected_dunders": 21,
41
+ "protected_builtins": 73,
42
+ "protected_exceptions": 69,
43
+ "protected_stdlib": 125,
44
+ "protected_ds_tokens": 119,
45
+ "total_never_split": 474,
46
+ "special_tokens": 20,
47
+ "reserved_future_tokens": 4,
48
+ "target_model_sizes": [
49
+ "124M",
50
+ "250M",
51
+ "370M",
52
+ "500M"
53
+ ],
54
+ "fim_tokens": [
55
+ "<fim_prefix>",
56
+ "<fim_suffix>",
57
+ "<fim_middle>"
58
+ ],
59
+ "notebook_tokens": [
60
+ "<cell>",
61
+ "</cell>",
62
+ "<cell_output>",
63
+ "</cell_output>"
64
+ ],
65
+ "comments_in_training": true,
66
+ "docstrings_in_training": true,
67
+ "training_date": "2026-05-11"
68
+ }
69
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff