Spaces:
Running
Running
Added new Help section with common use-case examples
Browse files
app.py
CHANGED
@@ -37,7 +37,7 @@ def human_readable_metadata(
|
|
37 |
key: str,
|
38 |
typ: int,
|
39 |
val: Any,
|
40 |
-
) -> tuple[str,
|
41 |
typ = GGUFValueType(typ).name
|
42 |
|
43 |
if typ == 'ARRAY':
|
@@ -68,134 +68,472 @@ def human_readable_metadata(
|
|
68 |
|
69 |
with gr.Blocks(
|
70 |
) as blocks:
|
71 |
-
with gr.
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
|
|
79 |
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
|
|
|
|
|
|
|
|
|
|
85 |
|
86 |
-
|
87 |
-
|
88 |
-
scale = 1,
|
89 |
)
|
90 |
|
91 |
-
|
92 |
-
|
93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
94 |
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
info = "Search by metadata key name",
|
101 |
allow_custom_value = True,
|
102 |
visible = False,
|
103 |
)
|
104 |
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
info = "Select data type",
|
110 |
type = "index",
|
111 |
visible = False,
|
112 |
)
|
113 |
|
114 |
-
|
115 |
-
|
116 |
-
"
|
117 |
-
|
|
|
118 |
visible = False,
|
119 |
)
|
120 |
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
info = "Search by token name",
|
131 |
-
type = "index",
|
132 |
-
allow_custom_value = True,
|
133 |
visible = False,
|
134 |
)
|
135 |
|
136 |
-
|
137 |
-
|
138 |
-
label = "
|
139 |
-
|
140 |
-
|
|
|
141 |
visible = False,
|
142 |
)
|
143 |
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
visible = False,
|
150 |
)
|
151 |
|
152 |
-
|
153 |
-
|
|
|
154 |
visible = False,
|
155 |
)
|
156 |
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
161 |
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
row_count = (1, "fixed"),
|
166 |
-
height = "1rem",
|
167 |
-
interactive = False,
|
168 |
-
visible = False,
|
169 |
-
)
|
170 |
|
171 |
-
|
172 |
-
None,
|
173 |
-
label = "Metadata Changes",
|
174 |
-
color_map = {"add": "green", "rem": "red"},
|
175 |
-
interactive = False,
|
176 |
-
visible = False,
|
177 |
-
)
|
178 |
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
visible = False,
|
183 |
-
)
|
184 |
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
|
200 |
meta_state = gr.State() # init_state
|
201 |
# BUG: For some reason using gr.State initial value turns tuple to list?
|
|
|
37 |
key: str,
|
38 |
typ: int,
|
39 |
val: Any,
|
40 |
+
) -> tuple[str, str, Any]:
|
41 |
typ = GGUFValueType(typ).name
|
42 |
|
43 |
if typ == 'ARRAY':
|
|
|
68 |
|
69 |
with gr.Blocks(
|
70 |
) as blocks:
|
71 |
+
with gr.Tab("Editor"):
|
72 |
+
with gr.Row():
|
73 |
+
hf_search = HuggingfaceHubSearch(
|
74 |
+
label = "Search Huggingface Hub",
|
75 |
+
placeholder = "Search for models on Huggingface",
|
76 |
+
search_type = "model",
|
77 |
+
sumbit_on_select = True,
|
78 |
+
scale = 2,
|
79 |
+
)
|
80 |
|
81 |
+
hf_branch = gr.Dropdown(
|
82 |
+
None,
|
83 |
+
label = "Branch",
|
84 |
+
scale = 1,
|
85 |
+
)
|
86 |
+
|
87 |
+
gr.LoginButton(
|
88 |
+
"Sign in to access gated/private repos",
|
89 |
+
scale = 1,
|
90 |
+
)
|
91 |
|
92 |
+
hf_file = FileExplorer(
|
93 |
+
visible=False,
|
|
|
94 |
)
|
95 |
|
96 |
+
with gr.Row():
|
97 |
+
with gr.Column():
|
98 |
+
meta_keys = gr.Dropdown(
|
99 |
+
None,
|
100 |
+
label = "Modify Metadata",
|
101 |
+
info = "Search by metadata key name",
|
102 |
+
allow_custom_value = True,
|
103 |
+
visible = False,
|
104 |
+
)
|
105 |
+
|
106 |
+
with gr.Column():
|
107 |
+
meta_types = gr.Dropdown(
|
108 |
+
[e.name for e in GGUFValueType],
|
109 |
+
label = "Metadata Type",
|
110 |
+
info = "Select data type",
|
111 |
+
type = "index",
|
112 |
+
visible = False,
|
113 |
+
)
|
114 |
+
|
115 |
+
with gr.Column():
|
116 |
+
btn_delete = gr.Button(
|
117 |
+
"Remove Key",
|
118 |
+
variant = "stop",
|
119 |
+
visible = False,
|
120 |
+
)
|
121 |
+
|
122 |
+
meta_boolean = gr.Checkbox(
|
123 |
+
label = "Boolean",
|
124 |
+
info = "Click to update value",
|
125 |
+
visible = False,
|
126 |
+
)
|
127 |
|
128 |
+
with gr.Row():
|
129 |
+
meta_token_select = gr.Dropdown(
|
130 |
+
label = "Select token",
|
131 |
+
info = "Search by token name",
|
132 |
+
type = "index",
|
|
|
133 |
allow_custom_value = True,
|
134 |
visible = False,
|
135 |
)
|
136 |
|
137 |
+
meta_token_type = gr.Dropdown(
|
138 |
+
[e.name for e in TokenType],
|
139 |
+
label = "Token type",
|
140 |
+
info = "Select token type",
|
|
|
141 |
type = "index",
|
142 |
visible = False,
|
143 |
)
|
144 |
|
145 |
+
meta_lookup = gr.Dropdown(
|
146 |
+
label = "Lookup token",
|
147 |
+
info = "Search by token name",
|
148 |
+
type = "index",
|
149 |
+
allow_custom_value = True,
|
150 |
visible = False,
|
151 |
)
|
152 |
|
153 |
+
meta_number = gr.Number(
|
154 |
+
label = "Number",
|
155 |
+
info = "Enter to update value",
|
156 |
+
visible = False,
|
157 |
+
)
|
158 |
|
159 |
+
meta_string = gr.Textbox(
|
160 |
+
label = "String",
|
161 |
+
info = "Enter to update value (Shift+Enter for new line)",
|
|
|
|
|
|
|
162 |
visible = False,
|
163 |
)
|
164 |
|
165 |
+
meta_array = gr.Matrix(
|
166 |
+
None,
|
167 |
+
label = "Unsupported",
|
168 |
+
row_count = (1, "fixed"),
|
169 |
+
height = "1rem",
|
170 |
+
interactive = False,
|
171 |
visible = False,
|
172 |
)
|
173 |
|
174 |
+
meta_changes = gr.HighlightedText(
|
175 |
+
None,
|
176 |
+
label = "Metadata Changes",
|
177 |
+
color_map = {"add": "green", "rem": "red"},
|
178 |
+
interactive = False,
|
179 |
visible = False,
|
180 |
)
|
181 |
|
182 |
+
btn_download = gr.Button(
|
183 |
+
"Download GGUF",
|
184 |
+
variant = "primary",
|
185 |
visible = False,
|
186 |
)
|
187 |
|
188 |
+
file_meta = gr.Matrix(
|
189 |
+
None,
|
190 |
+
col_count = (3, "fixed"),
|
191 |
+
headers = [
|
192 |
+
"Metadata Name",
|
193 |
+
"Type",
|
194 |
+
"Value",
|
195 |
+
],
|
196 |
+
datatype = ["str", "str", "str"],
|
197 |
+
column_widths = ["35%", "15%", "50%"],
|
198 |
+
wrap = True,
|
199 |
+
interactive = False,
|
200 |
+
visible = False,
|
201 |
+
)
|
202 |
|
203 |
+
with gr.Tab("Help"):
|
204 |
+
gr.Markdown(
|
205 |
+
"""# Huggingface GGUF Editor
|
|
|
|
|
|
|
|
|
|
|
206 |
|
207 |
+
An advanced GGUF editor, reading GGUF files directly from Huggingface repositories and applying changes to your own copies.
|
|
|
|
|
|
|
|
|
|
|
|
|
208 |
|
209 |
+
Below you will find a collection of example use-cases to show you how to perform a few common GGUF editing operations:
|
210 |
+
""",
|
211 |
+
)
|
|
|
|
|
212 |
|
213 |
+
with gr.Column(render = False) as example_group:
|
214 |
+
example_description = gr.Markdown(
|
215 |
+
visible = False,
|
216 |
+
)
|
217 |
+
|
218 |
+
with gr.Row():
|
219 |
+
with gr.Column():
|
220 |
+
example_keys = gr.Dropdown(
|
221 |
+
allow_custom_value = True,
|
222 |
+
visible = False,
|
223 |
+
)
|
224 |
+
|
225 |
+
with gr.Column():
|
226 |
+
example_types = gr.Dropdown(
|
227 |
+
allow_custom_value = True,
|
228 |
+
visible = False,
|
229 |
+
)
|
230 |
+
|
231 |
+
with gr.Column():
|
232 |
+
example_delete = gr.Button(
|
233 |
+
interactive = False,
|
234 |
+
visible = False,
|
235 |
+
)
|
236 |
+
|
237 |
+
example_boolean = gr.Checkbox(
|
238 |
+
visible = False,
|
239 |
+
)
|
240 |
+
|
241 |
+
with gr.Row():
|
242 |
+
example_token_select = gr.Dropdown(
|
243 |
+
allow_custom_value = True,
|
244 |
+
visible = False,
|
245 |
+
)
|
246 |
+
|
247 |
+
example_token_type = gr.Dropdown(
|
248 |
+
allow_custom_value = True,
|
249 |
+
visible = False,
|
250 |
+
)
|
251 |
+
|
252 |
+
example_number = gr.Number(
|
253 |
+
visible = False,
|
254 |
+
)
|
255 |
+
|
256 |
+
example_string = gr.Textbox(
|
257 |
+
visible = False,
|
258 |
+
)
|
259 |
+
|
260 |
+
example_components = [
|
261 |
+
example_description,
|
262 |
+
example_keys,
|
263 |
+
example_types,
|
264 |
+
example_delete,
|
265 |
+
example_boolean,
|
266 |
+
example_token_select,
|
267 |
+
example_token_type,
|
268 |
+
example_number,
|
269 |
+
example_string,
|
270 |
+
]
|
271 |
+
example_defaults = {
|
272 |
+
example_description: dict(
|
273 |
+
value = "",
|
274 |
+
visible = False,
|
275 |
+
),
|
276 |
+
example_keys: dict(
|
277 |
+
value = "",
|
278 |
+
label = meta_keys.label,
|
279 |
+
info = "Select this metadata key",
|
280 |
+
visible = False,
|
281 |
+
),
|
282 |
+
example_types: dict(
|
283 |
+
value = "",
|
284 |
+
label = meta_types.label,
|
285 |
+
info = "This will have the correct type set automatically",
|
286 |
+
visible = False,
|
287 |
+
),
|
288 |
+
example_delete: dict(
|
289 |
+
value = btn_delete.value,
|
290 |
+
variant = btn_delete.variant,
|
291 |
+
visible = False,
|
292 |
+
),
|
293 |
+
example_boolean: dict(
|
294 |
+
value = False,
|
295 |
+
label = meta_boolean.label,
|
296 |
+
info = "",
|
297 |
+
visible = False,
|
298 |
+
),
|
299 |
+
example_token_select: dict(
|
300 |
+
value = "",
|
301 |
+
label = meta_token_select.label,
|
302 |
+
visible = False,
|
303 |
+
),
|
304 |
+
example_token_type: dict(
|
305 |
+
value = "",
|
306 |
+
label = meta_token_type.label,
|
307 |
+
visible = False,
|
308 |
+
),
|
309 |
+
example_number: dict(
|
310 |
+
value = 0,
|
311 |
+
precision = 0,
|
312 |
+
label = meta_number.label,
|
313 |
+
info = "",
|
314 |
+
visible = False,
|
315 |
+
),
|
316 |
+
example_string: dict(
|
317 |
+
value = "",
|
318 |
+
label = meta_string.label,
|
319 |
+
info = "",
|
320 |
+
visible = False,
|
321 |
+
),
|
322 |
+
}
|
323 |
+
example_properties = [
|
324 |
+
dict(
|
325 |
+
label = 'Fix "missing pre-tokenizer type" warning',
|
326 |
+
outputs = {
|
327 |
+
example_description: dict(
|
328 |
+
value = """## Fixing Pre-Tokenizer warning
|
329 |
+
|
330 |
+
Custom Pre-Tokenization was added to `llama.cpp` April 29th 2024, and since then basically every model using BPE tokenization need support added to `llama.cpp` to work correctly.
|
331 |
+
|
332 |
+
Models converted using the conversion script before the support for this specific model was added will either be missing the pre-tokenizer metadata or be set incorrectly to `default`.
|
333 |
+
|
334 |
+
See the models list in [llama.cpp/convert_hf_to_gguf_update.py](https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf_update.py#L67) to find out which pre-tokenizer to choose.
|
335 |
+
|
336 |
+
Setting the correct pre-tokenizer is often enough to fix the model's tokenizer, however if it has been quantized using an `imatrix` it should be re-quantized for best performance.
|
337 |
+
|
338 |
+
Removing this metadata key from a model will cause `llama.cpp` to output a warning if BPE tokenization is used, it currently has no effect on any other tokenizers.
|
339 |
+
""",
|
340 |
+
visible = True,
|
341 |
+
),
|
342 |
+
example_keys: dict(
|
343 |
+
value = "tokenizer.ggml.pre",
|
344 |
+
visible = True,
|
345 |
+
),
|
346 |
+
example_types: dict(
|
347 |
+
value = GGUFValueType.STRING.name,
|
348 |
+
visible = True,
|
349 |
+
),
|
350 |
+
example_delete: dict(
|
351 |
+
visible = True,
|
352 |
+
),
|
353 |
+
example_string: dict(
|
354 |
+
info = "Fill in pre-tokenizer name, can be f.ex. deepseek-llm, command-r, tekken, etc. you will need to do some research to find the correct one",
|
355 |
+
value = "llama-bpe",
|
356 |
+
visible = True,
|
357 |
+
),
|
358 |
+
},
|
359 |
+
),
|
360 |
+
dict(
|
361 |
+
label = "Add missing (Fill-in-Middle, EOT, etc) or change incorrect (BOS, EOS, etc) tokens",
|
362 |
+
outputs = {
|
363 |
+
example_description: dict(
|
364 |
+
value = """## Add missing/change incorrect tokens
|
365 |
+
|
366 |
+
Sometimes converted models will be missing declarations of important tokens like EOT, Fill-in-Middle (prefix, suffix, middle) for various reasons.
|
367 |
+
Other times they may have the incorrect tokens set as BOS, EOS, etc. Either way, missing or incorrectly declared tokens means inference will not work as expected.
|
368 |
+
|
369 |
+
Token declaration is made with the metadata key(s) named "tokenizer.ggml.`token name`\_token\_id" which contains the ID (index number) of the token in the token list (`tokenizer.ggml.tokens`).
|
370 |
+
|
371 |
+
A recurring issue is misconfigured EOS/EOT/EOM tokens, the need to set each of these and what they should be will vary between models, but the effect when these are incorrect is usually the same;
|
372 |
+
infinte generation responses, ie. inference does not know when to stop. Typically this would be because f.ex. EOS has been set to <|endoftext|> instead of <|im\_end|> (again, model specific, just an example).
|
373 |
+
|
374 |
+
Another issue, mainly for code models, is that Fill-in-Middle tokens have not been declared (note; not all models have or use such tokens), causing sub-par results for filling in blanks in code/text.
|
375 |
+
There are 3 main metadata keys that need to be present for this; tokenizer.ggml.`prefix`\_token\_id, `suffix` and `middle`, sometimes also EOT/EOM if it differs from EOS in this mode.
|
376 |
+
They are usually named fim\_`something` or just `PRE`, `SUF` and `MID`, take extra care with DeepSeek-based models where prefix is (...fim...)`begin`, suffix is `hole` and middle is `end`.
|
377 |
+
""",
|
378 |
+
visible = True,
|
379 |
+
),
|
380 |
+
example_keys: dict(
|
381 |
+
value = "tokenizer.ggml.prefix_token_id",
|
382 |
+
info = "Select or enter any metadata key ending with _token_id",
|
383 |
+
visible = True,
|
384 |
+
),
|
385 |
+
example_types: dict(
|
386 |
+
value = GGUFValueType.UINT32.name,
|
387 |
+
visible = True,
|
388 |
+
),
|
389 |
+
example_token_select: dict(
|
390 |
+
value = "<fim_prefix>",
|
391 |
+
label = meta_lookup.label,
|
392 |
+
info = "You can search for the correct token by parts of its name here, then select the correct one from the list of options",
|
393 |
+
visible = True,
|
394 |
+
),
|
395 |
+
example_number: dict(
|
396 |
+
value = 92295,
|
397 |
+
info = "The token ID will be automatically filled in when you select the token, but you can also fill in the ID directly",
|
398 |
+
visible = True,
|
399 |
+
),
|
400 |
+
},
|
401 |
+
),
|
402 |
+
dict(
|
403 |
+
label = "Setting the correct token type for a token",
|
404 |
+
outputs = {
|
405 |
+
example_description: dict(
|
406 |
+
value = """## Changing a token's type
|
407 |
+
|
408 |
+
A common issue is not declaring special control tokens as such, leading to bad tokenization of them when used (usually in the chat template), causing poor responses from the model.
|
409 |
+
|
410 |
+
Take f.ex. a model with an incorrectly configured <|im\_start|> token as a normal token instead of a special control token, given the following prompt:
|
411 |
+
```
|
412 |
+
<|im_start|>Hello World<|im_end|>
|
413 |
+
```
|
414 |
+
|
415 |
+
This prompt would then be incorrectly tokenized as follows:
|
416 |
+
```
|
417 |
+
27 ('<')
|
418 |
+
91 ('|')
|
419 |
+
318 ('im')
|
420 |
+
4906 ('_start')
|
421 |
+
91 ('|')
|
422 |
+
29 ('>')
|
423 |
+
9707 ('Hello')
|
424 |
+
4337 (' World')
|
425 |
+
151645 ('<|im_end|>')
|
426 |
+
```
|
427 |
+
|
428 |
+
instead of:
|
429 |
+
```
|
430 |
+
151644 ('<|im_start|>')
|
431 |
+
9707 ('Hello')
|
432 |
+
4337 (' World')
|
433 |
+
151645 ('<|im_end|>')
|
434 |
+
```
|
435 |
+
|
436 |
+
Take care to also adjust the value for this token in `tokenizer.ggml.scores` (if it exists) similarly to other special control tokens.
|
437 |
+
|
438 |
+
**WARNING**: Even though you have the option to, you should never remove the `tokenizer.ggml.token_type` key!
|
439 |
+
""",
|
440 |
+
visible = True,
|
441 |
+
),
|
442 |
+
example_keys: dict(
|
443 |
+
value = "tokenizer.ggml.token_type",
|
444 |
+
visible = True,
|
445 |
+
),
|
446 |
+
example_types: dict(
|
447 |
+
value = GGUFValueType.INT32.name,
|
448 |
+
visible = True,
|
449 |
+
),
|
450 |
+
example_delete: dict(
|
451 |
+
visible = True,
|
452 |
+
),
|
453 |
+
example_token_select: dict(
|
454 |
+
value = "<|im_start|>",
|
455 |
+
info = "You can search for the token by parts of its name here, then select it from the list of options",
|
456 |
+
visible = True,
|
457 |
+
),
|
458 |
+
example_token_type: dict(
|
459 |
+
value = TokenType.CONTROL.name,
|
460 |
+
info = "Select the appropriate token type, in this case we set it as a special control token",
|
461 |
+
visible = True,
|
462 |
+
),
|
463 |
+
},
|
464 |
+
),
|
465 |
+
dict(
|
466 |
+
label = "Updating or adding a chat template",
|
467 |
+
outputs = {
|
468 |
+
example_description: dict(
|
469 |
+
value = """## Modifying the Chat Template
|
470 |
+
|
471 |
+
The chat template is a very important part of the model metadata as this provides a template for how to format the conversation prompt to the model.
|
472 |
+
It's not uncommon for these to have bugs (or sometimes just be plain wrong), requiring you to update them to be able to prompt the model correctly.
|
473 |
+
|
474 |
+
It's also possible to have multiple chat templates for different purposes, the main ones being RAG and Tools, but you can create any additional template you want.
|
475 |
+
The standard metadata key for RAG is `tokenizer.chat_template.rag` and Tools is `tokenizer.chat_template.tool_use`, any metadata key added starting with `tokenizer.chat_template.` will be added as a custom chat template.
|
476 |
+
|
477 |
+
Any framework based on `llama-cpp-python` will let you select which chat template to use with the `chat_format` option, available as `chat_template.default`, `chat_template.rag`, `chat_template.tool_use`, etc...
|
478 |
+
""",
|
479 |
+
visible = True,
|
480 |
+
),
|
481 |
+
example_keys: dict(
|
482 |
+
value = "tokenizer.chat_template",
|
483 |
+
info = 'Select this or enter any key starting with "tokenizer.chat_template."',
|
484 |
+
visible = True,
|
485 |
+
),
|
486 |
+
example_types: dict(
|
487 |
+
value = GGUFValueType.STRING.name,
|
488 |
+
visible = True,
|
489 |
+
),
|
490 |
+
example_delete: dict(
|
491 |
+
visible = True,
|
492 |
+
),
|
493 |
+
example_string: dict(
|
494 |
+
info = "Paste in the updated chat template or make changes here. Using an external Jinja2 editor is recommended",
|
495 |
+
value = "{%- for message in messages %}\n {{- '<|' + message['role'] + '|>\\n' }}\n {{- message['content'] + eos_token }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|assistant|>\\n' }}\n{%- endif %}",
|
496 |
+
visible = True,
|
497 |
+
),
|
498 |
+
},
|
499 |
+
),
|
500 |
+
]
|
501 |
+
|
502 |
+
examples = gr.Dataset(
|
503 |
+
label = "Choose an example",
|
504 |
+
type = "index",
|
505 |
+
samples = [[]] * len(example_properties),
|
506 |
+
sample_labels = [x["label"] for x in example_properties],
|
507 |
+
)
|
508 |
+
|
509 |
+
@gr.on(
|
510 |
+
triggers = [
|
511 |
+
examples.click,
|
512 |
+
],
|
513 |
+
inputs = [
|
514 |
+
examples,
|
515 |
+
],
|
516 |
+
outputs = [
|
517 |
+
] + example_components,
|
518 |
+
show_progress = "hidden",
|
519 |
+
)
|
520 |
+
def show_example(
|
521 |
+
value: int,
|
522 |
+
):
|
523 |
+
outputs = example_properties[value]["outputs"]
|
524 |
+
non_outputs = example_components - outputs.keys()
|
525 |
+
all_outputs = dict(((k, type(k)(**(example_defaults[k] | v))) for k, v in outputs.items()))
|
526 |
+
|
527 |
+
for output in non_outputs:
|
528 |
+
all_outputs[output] = type(output)(**example_defaults[output])
|
529 |
+
|
530 |
+
return all_outputs
|
531 |
+
|
532 |
+
for k, v in example_defaults.items():
|
533 |
+
for prop, val in v.items():
|
534 |
+
setattr(k, prop, val)
|
535 |
+
|
536 |
+
example_group.render()
|
537 |
|
538 |
meta_state = gr.State() # init_state
|
539 |
# BUG: For some reason using gr.State initial value turns tuple to list?
|