Hugging Face's logo

trohrbaugh
/
Hy3-heretic

Text Generation
Transformers
Safetensors
hy_v3
hunyuan
hy3
Mixture of Experts
heretic
uncensored
decensored
abliterated
conversational
Model card Files
xet
 Community
2
trohrbaugh commited on
Commit
aaae574
·
verified ·
1 Parent(s): 6e84e10

Upload tokenizer

Browse files
Files changed (3) hide show
  1. chat_template.jinja +222 -0
  2. tokenizer.json +0 -0
  3. tokenizer_config.json +12 -0
chat_template.jinja ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {#- ----------‑‑‑ special token variables ‑‑‑---------- -#}
2
+ {%- set HYTK = ':opensource' %}
3
+ {%- set eos_token = '<｜hy_eos{}｜>'.format(HYTK) %}
4
+ {%- set bos_token = '<｜hy_begin_of_sentence{}｜>'.format(HYTK) %}
5
+ {%- set pad_token = '<｜hy_pad{}｜>'.format(HYTK) %}
6
+ {%- set user_token = '<｜hy_User{}｜>'.format(HYTK) %}
7
+ {%- set assistant_token = '<｜hy_Assistant{}｜>'.format(HYTK) %}
8
+ {%- set think_begin_token = '<think{}>'.format(HYTK) %}
9
+ {%- set think_end_token = '</think{}>'.format(HYTK) %}
10
+ {%- set toolcalls_begin_token = '<tool_calls{}>'.format(HYTK) %}
11
+ {%- set toolcalls_end_token = '</tool_calls{}>'.format(HYTK) %}
12
+ {%- set toolcall_begin_token = '<tool_call{}>'.format(HYTK) %}
13
+ {%- set toolcall_end_token = '</tool_call{}>'.format(HYTK) %}
14
+ {%- set toolsep_token = '<tool_sep{}>'.format(HYTK) %}
15
+ {%- set argkey_begin_token = '<arg_key{}>'.format(HYTK) %}
16
+ {%- set argkey_end_token = '</arg_key{}>'.format(HYTK) %}
17
+ {%- set argvalue_begin_token = '<arg_value{}>'.format(HYTK) %}
18
+ {%- set argvalue_end_token = '</arg_value{}>'.format(HYTK) %}
19
+ {%- set toolresponses_begin_token = '<tool_responses{}>'.format(HYTK) %}
20
+ {%- set toolresponses_end_token = '</tool_responses{}>'.format(HYTK) %}
21
+ {%- set toolresponse_begin_token = '<tool_response{}>'.format(HYTK) %}
22
+ {%- set toolresponse_end_token = '</tool_response{}>'.format(HYTK) %}
23
+ {%- set reasoning_mode_token = '<｜reasoning_mode{}｜>'.format(HYTK) %}
24
+
25
+ {#- ----------‑‑‑ hyperparameters variables ‑‑‑---------- -#}
26
+ {%- if not add_generation_prompt is defined %}
27
+ {%- set add_generation_prompt = false %}
28
+ {%- endif %}
29
+ {%- if not preserved_thinking is defined %}
30
+ {%- if not tools %}
31
+ {%- set preserved_thinking = false %}
32
+ {%- else %}
33
+ {%- set preserved_thinking = true %}
34
+ {%- endif %}
35
+ {%- endif %}
36
+ {%- if not is_training is defined %}
37
+ {%- set is_training = false %}
38
+ {%- endif %}
39
+
40
+ {%- if not reasoning_effort is defined %}
41
+ {%- set reasoning_effort = 'no_think' %}
42
+ {%- elif reasoning_effort not in ['high', 'low', 'no_think'] %}
43
+ {%- if reasoning_effort is none %}
44
+ {{- raise_exception('reasoning_effort error : None, should be no_think/low/high') }}
45
+ {%- else %}
46
+ {{- raise_exception('reasoning_effort error : ' + reasoning_effort + ', should be no_think/low/high') }}
47
+ {%- endif %}
48
+ {%- endif %}
49
+
50
+ {%- if fallback_strategy is defined and fallback_strategy == 'reasoning_toolcall_retry' %}
51
+ {%- set reasoning_effort = 'high' %}
52
+ {%- set add_generation_prompt = false %}
53
+ {%- endif %}
54
+ {%- if not raw_last_assistant is defined %}
55
+ {%- set raw_last_assistant = false %}
56
+ {%- endif %}
57
+
58
+ {%- macro visible_text(content) -%}
59
+ {%- if content is string -%}
60
+ {{- content }}
61
+ {%- elif content is iterable and content is not mapping -%}
62
+ {%- for item in content -%}
63
+ {%- if item is mapping and item.type == 'text' -%}
64
+ {{- item.text }}
65
+ {%- elif item is string -%}
66
+ {{- item }}
67
+ {%- endif -%}
68
+ {%- endfor -%}
69
+ {%- elif content is none -%}
70
+ {{- '' }}
71
+ {%- else -%}
72
+ {{- content }}
73
+ {%- endif -%}
74
+ {%- endmacro -%}
75
+
76
+ {%- set ns = namespace(last_user_index=-1) %}
77
+ {%- set sp_ns = namespace(system_prompt='', is_first_sp=true) %}
78
+ {%- for message in messages %}
79
+ {%- if message['role'] == 'system' %}
80
+ {%- set sp_ns.system_prompt = sp_ns.system_prompt + visible_text(message['content']) %}
81
+ {%- endif %}
82
+ {%- if message['role'] == 'user' %}
83
+ {%- set ns.last_user_index = loop.index0 %}
84
+ {%- endif %}
85
+ {%- endfor %}
86
+ {%- if reasoning_effort is defined and reasoning_effort is string and reasoning_effort != '' and not tools %}
87
+ {%- set sp_ns.system_prompt = sp_ns.system_prompt + reasoning_mode_token + 'reasoning_effort:' + reasoning_effort %}
88
+ {%- endif %}
89
+ {{- bos_token }}
90
+ {{- sp_ns.system_prompt }}
91
+ {%- if tools %}
92
+ {%- if sp_ns.system_prompt != '' %}
93
+ {{- '\n\n# Tools\n\nYou may call one or more functions to assist with the user query.' }}
94
+ {%- else %}
95
+ {{- '# Tools\n\nYou may call one or more functions to assist with the user query.' }}
96
+ {%- endif %}
97
+ {{- '\n\nYou are provided with function signatures within <tools></tools> XML tags:' }}
98
+ {{- '\n<tools>\n' }}
99
+ {%- for tool in tools %}
100
+ {%- if loop.index0 > 0 %}
101
+ {{- '\n' }}
102
+ {%- endif %}
103
+ {{- tool | tojson }}
104
+ {%- endfor %}
105
+ {{- '\n</tools>\n\n' }}
106
+ {{- 'For function call returns, you should first print ' + toolcalls_begin_token + '\n' }}
107
+ {{- 'For each function call, you should return object like:\n' }}
108
+ {{- toolcall_begin_token + '{function-name}' + toolsep_token + '\n' }}
109
+ {{- argkey_begin_token + '{arg-key-1}' + argkey_end_token + '\n' }}
110
+ {{- argvalue_begin_token + '{arg-value-1}' + argvalue_end_token + '\n' }}
111
+ {{- argkey_begin_token + '{arg-key-2}' + argkey_end_token + '\n' }}
112
+ {{- argvalue_begin_token + '{arg-value-2}' + argvalue_end_token + '\n' }}
113
+ {{- '...\n' }}
114
+ {{- toolcall_end_token + '\n' }}
115
+ {%- if reasoning_effort is defined and reasoning_effort is string and reasoning_effort != '' %}
116
+ {{- 'At the end of function call returns, you should print ' + toolcalls_end_token + reasoning_mode_token + 'reasoning_effort:' + reasoning_effort }}
117
+ {%- else %}
118
+ {{- 'At the end of function call returns, you should print ' + toolcalls_end_token }}
119
+ {%- endif %}
120
+ {%- endif %}
121
+
122
+ {%- set prev_ns = namespace(is_tool=false, is_tool_first=true) %}
123
+ {%- set last_ns = namespace(last_is_assistant=false) %}
124
+ {%- for message in messages %}
125
+ {%- if message['role'] == 'user' %}
126
+ {%- if prev_ns.is_tool %}
127
+ {{- toolresponses_end_token }}
128
+ {%- endif %}
129
+ {{- user_token + visible_text(message['content']) }}
130
+ {%- set prev_ns.is_tool = false %}
131
+ {%- endif %}
132
+ {%- if message['role'] == 'assistant' %}
133
+ {%- if is_training %}
134
+ {%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
135
+ {%- set rc = message['reasoning_content'] %}
136
+ {%- elif 'reasoning' in message and message['reasoning'] is string %}
137
+ {%- set rc = message['reasoning'] %}
138
+ {%- else %}
139
+ {%- set rc = none %}
140
+ {%- endif %}
141
+ {%- if rc is not none %}
142
+ {%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
143
+ {%- else %}
144
+ {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
145
+ {%- endif %}
146
+ {%- else %}
147
+ {%- if ((preserved_thinking is defined and preserved_thinking) or loop.index0 > ns.last_user_index) %}
148
+ {%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
149
+ {%- set rc = message['reasoning_content'] %}
150
+ {%- elif 'reasoning' in message and message['reasoning'] is string %}
151
+ {%- set rc = message['reasoning'] %}
152
+ {%- else %}
153
+ {%- set rc = none %}
154
+ {%- endif %}
155
+ {%- if rc is not none %}
156
+ {%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
157
+ {%- else %}
158
+ {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
159
+ {%- endif %}
160
+ {%- else %}
161
+ {%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
162
+ {%- endif %}
163
+ {%- endif %}
164
+ {%- if prev_ns.is_tool %}
165
+ {{- toolresponses_end_token }}
166
+ {%- endif %}
167
+ {{- assistant_token }}
168
+ {%- if message['tool_calls'] is defined and message['tool_calls'] %}
169
+ {%- set prev_ns.is_tool_first = true %}
170
+ {{- content }}
171
+ {{- toolcalls_begin_token + '\n' }}
172
+ {%- for tool in message['tool_calls'] %}
173
+ {%- set arguments = tool['function']['arguments'] %}
174
+ {{- toolcall_begin_token + tool['function']['name'] + toolsep_token + '\n' }}
175
+ {%- for key, value in arguments.items() %}
176
+ {{- argkey_begin_token + key + argkey_end_token + '\n' }}
177
+ {%- if value is not string %}
178
+ {%- set value = value | tojson(ensure_ascii=False) %}
179
+ {%- endif %}
180
+ {{- argvalue_begin_token + value + argvalue_end_token + '\n' }}
181
+ {%- endfor %}
182
+ {{- toolcall_end_token + '\n' }}
183
+ {%- endfor %}
184
+ {{- toolcalls_end_token + eos_token }}
185
+ {%- else %}
186
+ {%- if loop.last and raw_last_assistant %}
187
+ {{- visible_text(message['content']) }}
188
+ {%- elif not loop.last or is_training %}
189
+ {{- content + eos_token }}
190
+ {%- else %}
191
+ {{- content }}
192
+ {%- endif %}
193
+ {%- endif %}
194
+ {%- set prev_ns.is_tool = false %}
195
+ {%- endif %}
196
+ {%- if message['role'] == 'tool' %}
197
+ {%- set prev_ns.is_tool = true %}
198
+ {%- if prev_ns.is_tool_first %}
199
+ {{- toolresponses_begin_token + '\n' }}
200
+ {%- set prev_ns.is_tool_first = false %}
201
+ {%- endif %}
202
+ {{- toolresponse_begin_token + '\n' + visible_text(message['content']) + '\n' + toolresponse_end_token + '\n' }}
203
+ {%- endif %}
204
+ {%- if loop.last and message['role'] == 'assistant' %}
205
+ {%- set last_ns.last_is_assistant = true %}
206
+ {%- endif %}
207
+
208
+ {%- endfor %}
209
+ {%- if prev_ns.is_tool %}
210
+ {{- toolresponses_end_token }}
211
+ {%- endif %}
212
+ {%- if add_generation_prompt %}
213
+ {%- if not last_ns.last_is_assistant %}
214
+ {%- if reasoning_effort is defined and reasoning_effort in ['low', 'high'] %}
215
+ {{- assistant_token + think_begin_token }}
216
+ {%- elif reasoning_effort is defined and reasoning_effort == 'no_think' %}
217
+ {{- assistant_token + think_begin_token + think_end_token }}
218
+ {%- else %}
219
+ {{- assistant_token }}
220
+ {%- endif %}
221
+ {%- endif %}
222
+ {%- endif %}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<｜hy_begin_of_sentence:opensource｜>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<｜hy_eos:opensource｜>",
6
+ "is_local": true,
7
+ "local_files_only": false,
8
+ "model_max_length": 1000000000000000019884624838656,
9
+ "pad_token": "<｜hy_pad:opensource｜>",
10
+ "token_suffix": ":opensource",
11
+ "tokenizer_class": "TokenizersBackend"
12
+ }