CISCai commited on
Commit
73fd84b
·
verified ·
1 Parent(s): 84afff1

Added new Help section with common use-case examples

Browse files
Files changed (1) hide show
  1. app.py +434 -96
app.py CHANGED
@@ -37,7 +37,7 @@ def human_readable_metadata(
37
  key: str,
38
  typ: int,
39
  val: Any,
40
- ) -> tuple[str, int | str, Any]:
41
  typ = GGUFValueType(typ).name
42
 
43
  if typ == 'ARRAY':
@@ -68,134 +68,472 @@ def human_readable_metadata(
68
 
69
  with gr.Blocks(
70
  ) as blocks:
71
- with gr.Row():
72
- hf_search = HuggingfaceHubSearch(
73
- label = "Search Huggingface Hub",
74
- placeholder = "Search for models on Huggingface",
75
- search_type = "model",
76
- sumbit_on_select = True,
77
- scale = 2,
78
- )
 
79
 
80
- hf_branch = gr.Dropdown(
81
- None,
82
- label = "Branch",
83
- scale = 1,
84
- )
 
 
 
 
 
85
 
86
- gr.LoginButton(
87
- "Sign in to access gated/private repos",
88
- scale = 1,
89
  )
90
 
91
- hf_file = FileExplorer(
92
- visible=False,
93
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
 
95
- with gr.Row():
96
- with gr.Column():
97
- meta_keys = gr.Dropdown(
98
- None,
99
- label = "Modify Metadata",
100
- info = "Search by metadata key name",
101
  allow_custom_value = True,
102
  visible = False,
103
  )
104
 
105
- with gr.Column():
106
- meta_types = gr.Dropdown(
107
- [e.name for e in GGUFValueType],
108
- label = "Metadata Type",
109
- info = "Select data type",
110
  type = "index",
111
  visible = False,
112
  )
113
 
114
- with gr.Column():
115
- btn_delete = gr.Button(
116
- "Remove Key",
117
- variant = "stop",
 
118
  visible = False,
119
  )
120
 
121
- meta_boolean = gr.Checkbox(
122
- label = "Boolean",
123
- info = "Click to update value",
124
- visible = False,
125
- )
126
 
127
- with gr.Row():
128
- meta_token_select = gr.Dropdown(
129
- label = "Select token",
130
- info = "Search by token name",
131
- type = "index",
132
- allow_custom_value = True,
133
  visible = False,
134
  )
135
 
136
- meta_token_type = gr.Dropdown(
137
- [e.name for e in TokenType],
138
- label = "Token type",
139
- info = "Select token type",
140
- type = "index",
 
141
  visible = False,
142
  )
143
 
144
- meta_lookup = gr.Dropdown(
145
- label = "Lookup token",
146
- info = "Search by token name",
147
- type = "index",
148
- allow_custom_value = True,
149
  visible = False,
150
  )
151
 
152
- meta_number = gr.Number(
153
- info = "Enter to update value",
 
154
  visible = False,
155
  )
156
 
157
- meta_string = gr.Textbox(
158
- info = "Enter to update value (Shift+Enter for new line)",
159
- visible = False,
160
- )
 
 
 
 
 
 
 
 
 
 
161
 
162
- meta_array = gr.Matrix(
163
- None,
164
- label = "Unsupported",
165
- row_count = (1, "fixed"),
166
- height = "1rem",
167
- interactive = False,
168
- visible = False,
169
- )
170
 
171
- meta_changes = gr.HighlightedText(
172
- None,
173
- label = "Metadata Changes",
174
- color_map = {"add": "green", "rem": "red"},
175
- interactive = False,
176
- visible = False,
177
- )
178
 
179
- btn_download = gr.Button(
180
- "Download GGUF",
181
- variant = "primary",
182
- visible = False,
183
- )
184
 
185
- file_meta = gr.Matrix(
186
- None,
187
- col_count = (3, "fixed"),
188
- headers = [
189
- "Metadata Name",
190
- "Type",
191
- "Value",
192
- ],
193
- datatype = ["str", "str", "str"],
194
- column_widths = ["35%", "15%", "50%"],
195
- wrap = True,
196
- interactive = False,
197
- visible = False,
198
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
  meta_state = gr.State() # init_state
201
  # BUG: For some reason using gr.State initial value turns tuple to list?
 
37
  key: str,
38
  typ: int,
39
  val: Any,
40
+ ) -> tuple[str, str, Any]:
41
  typ = GGUFValueType(typ).name
42
 
43
  if typ == 'ARRAY':
 
68
 
69
  with gr.Blocks(
70
  ) as blocks:
71
+ with gr.Tab("Editor"):
72
+ with gr.Row():
73
+ hf_search = HuggingfaceHubSearch(
74
+ label = "Search Huggingface Hub",
75
+ placeholder = "Search for models on Huggingface",
76
+ search_type = "model",
77
+ sumbit_on_select = True,
78
+ scale = 2,
79
+ )
80
 
81
+ hf_branch = gr.Dropdown(
82
+ None,
83
+ label = "Branch",
84
+ scale = 1,
85
+ )
86
+
87
+ gr.LoginButton(
88
+ "Sign in to access gated/private repos",
89
+ scale = 1,
90
+ )
91
 
92
+ hf_file = FileExplorer(
93
+ visible=False,
 
94
  )
95
 
96
+ with gr.Row():
97
+ with gr.Column():
98
+ meta_keys = gr.Dropdown(
99
+ None,
100
+ label = "Modify Metadata",
101
+ info = "Search by metadata key name",
102
+ allow_custom_value = True,
103
+ visible = False,
104
+ )
105
+
106
+ with gr.Column():
107
+ meta_types = gr.Dropdown(
108
+ [e.name for e in GGUFValueType],
109
+ label = "Metadata Type",
110
+ info = "Select data type",
111
+ type = "index",
112
+ visible = False,
113
+ )
114
+
115
+ with gr.Column():
116
+ btn_delete = gr.Button(
117
+ "Remove Key",
118
+ variant = "stop",
119
+ visible = False,
120
+ )
121
+
122
+ meta_boolean = gr.Checkbox(
123
+ label = "Boolean",
124
+ info = "Click to update value",
125
+ visible = False,
126
+ )
127
 
128
+ with gr.Row():
129
+ meta_token_select = gr.Dropdown(
130
+ label = "Select token",
131
+ info = "Search by token name",
132
+ type = "index",
 
133
  allow_custom_value = True,
134
  visible = False,
135
  )
136
 
137
+ meta_token_type = gr.Dropdown(
138
+ [e.name for e in TokenType],
139
+ label = "Token type",
140
+ info = "Select token type",
 
141
  type = "index",
142
  visible = False,
143
  )
144
 
145
+ meta_lookup = gr.Dropdown(
146
+ label = "Lookup token",
147
+ info = "Search by token name",
148
+ type = "index",
149
+ allow_custom_value = True,
150
  visible = False,
151
  )
152
 
153
+ meta_number = gr.Number(
154
+ label = "Number",
155
+ info = "Enter to update value",
156
+ visible = False,
157
+ )
158
 
159
+ meta_string = gr.Textbox(
160
+ label = "String",
161
+ info = "Enter to update value (Shift+Enter for new line)",
 
 
 
162
  visible = False,
163
  )
164
 
165
+ meta_array = gr.Matrix(
166
+ None,
167
+ label = "Unsupported",
168
+ row_count = (1, "fixed"),
169
+ height = "1rem",
170
+ interactive = False,
171
  visible = False,
172
  )
173
 
174
+ meta_changes = gr.HighlightedText(
175
+ None,
176
+ label = "Metadata Changes",
177
+ color_map = {"add": "green", "rem": "red"},
178
+ interactive = False,
179
  visible = False,
180
  )
181
 
182
+ btn_download = gr.Button(
183
+ "Download GGUF",
184
+ variant = "primary",
185
  visible = False,
186
  )
187
 
188
+ file_meta = gr.Matrix(
189
+ None,
190
+ col_count = (3, "fixed"),
191
+ headers = [
192
+ "Metadata Name",
193
+ "Type",
194
+ "Value",
195
+ ],
196
+ datatype = ["str", "str", "str"],
197
+ column_widths = ["35%", "15%", "50%"],
198
+ wrap = True,
199
+ interactive = False,
200
+ visible = False,
201
+ )
202
 
203
+ with gr.Tab("Help"):
204
+ gr.Markdown(
205
+ """# Huggingface GGUF Editor
 
 
 
 
 
206
 
207
+ An advanced GGUF editor, reading GGUF files directly from Huggingface repositories and applying changes to your own copies.
 
 
 
 
 
 
208
 
209
+ Below you will find a collection of example use-cases to show you how to perform a few common GGUF editing operations:
210
+ """,
211
+ )
 
 
212
 
213
+ with gr.Column(render = False) as example_group:
214
+ example_description = gr.Markdown(
215
+ visible = False,
216
+ )
217
+
218
+ with gr.Row():
219
+ with gr.Column():
220
+ example_keys = gr.Dropdown(
221
+ allow_custom_value = True,
222
+ visible = False,
223
+ )
224
+
225
+ with gr.Column():
226
+ example_types = gr.Dropdown(
227
+ allow_custom_value = True,
228
+ visible = False,
229
+ )
230
+
231
+ with gr.Column():
232
+ example_delete = gr.Button(
233
+ interactive = False,
234
+ visible = False,
235
+ )
236
+
237
+ example_boolean = gr.Checkbox(
238
+ visible = False,
239
+ )
240
+
241
+ with gr.Row():
242
+ example_token_select = gr.Dropdown(
243
+ allow_custom_value = True,
244
+ visible = False,
245
+ )
246
+
247
+ example_token_type = gr.Dropdown(
248
+ allow_custom_value = True,
249
+ visible = False,
250
+ )
251
+
252
+ example_number = gr.Number(
253
+ visible = False,
254
+ )
255
+
256
+ example_string = gr.Textbox(
257
+ visible = False,
258
+ )
259
+
260
+ example_components = [
261
+ example_description,
262
+ example_keys,
263
+ example_types,
264
+ example_delete,
265
+ example_boolean,
266
+ example_token_select,
267
+ example_token_type,
268
+ example_number,
269
+ example_string,
270
+ ]
271
+ example_defaults = {
272
+ example_description: dict(
273
+ value = "",
274
+ visible = False,
275
+ ),
276
+ example_keys: dict(
277
+ value = "",
278
+ label = meta_keys.label,
279
+ info = "Select this metadata key",
280
+ visible = False,
281
+ ),
282
+ example_types: dict(
283
+ value = "",
284
+ label = meta_types.label,
285
+ info = "This will have the correct type set automatically",
286
+ visible = False,
287
+ ),
288
+ example_delete: dict(
289
+ value = btn_delete.value,
290
+ variant = btn_delete.variant,
291
+ visible = False,
292
+ ),
293
+ example_boolean: dict(
294
+ value = False,
295
+ label = meta_boolean.label,
296
+ info = "",
297
+ visible = False,
298
+ ),
299
+ example_token_select: dict(
300
+ value = "",
301
+ label = meta_token_select.label,
302
+ visible = False,
303
+ ),
304
+ example_token_type: dict(
305
+ value = "",
306
+ label = meta_token_type.label,
307
+ visible = False,
308
+ ),
309
+ example_number: dict(
310
+ value = 0,
311
+ precision = 0,
312
+ label = meta_number.label,
313
+ info = "",
314
+ visible = False,
315
+ ),
316
+ example_string: dict(
317
+ value = "",
318
+ label = meta_string.label,
319
+ info = "",
320
+ visible = False,
321
+ ),
322
+ }
323
+ example_properties = [
324
+ dict(
325
+ label = 'Fix "missing pre-tokenizer type" warning',
326
+ outputs = {
327
+ example_description: dict(
328
+ value = """## Fixing Pre-Tokenizer warning
329
+
330
+ Custom Pre-Tokenization was added to `llama.cpp` April 29th 2024, and since then basically every model using BPE tokenization need support added to `llama.cpp` to work correctly.
331
+
332
+ Models converted using the conversion script before the support for this specific model was added will either be missing the pre-tokenizer metadata or be set incorrectly to `default`.
333
+
334
+ See the models list in [llama.cpp/convert_hf_to_gguf_update.py](https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf_update.py#L67) to find out which pre-tokenizer to choose.
335
+
336
+ Setting the correct pre-tokenizer is often enough to fix the model's tokenizer, however if it has been quantized using an `imatrix` it should be re-quantized for best performance.
337
+
338
+ Removing this metadata key from a model will cause `llama.cpp` to output a warning if BPE tokenization is used, it currently has no effect on any other tokenizers.
339
+ """,
340
+ visible = True,
341
+ ),
342
+ example_keys: dict(
343
+ value = "tokenizer.ggml.pre",
344
+ visible = True,
345
+ ),
346
+ example_types: dict(
347
+ value = GGUFValueType.STRING.name,
348
+ visible = True,
349
+ ),
350
+ example_delete: dict(
351
+ visible = True,
352
+ ),
353
+ example_string: dict(
354
+ info = "Fill in pre-tokenizer name, can be f.ex. deepseek-llm, command-r, tekken, etc. you will need to do some research to find the correct one",
355
+ value = "llama-bpe",
356
+ visible = True,
357
+ ),
358
+ },
359
+ ),
360
+ dict(
361
+ label = "Add missing (Fill-in-Middle, EOT, etc) or change incorrect (BOS, EOS, etc) tokens",
362
+ outputs = {
363
+ example_description: dict(
364
+ value = """## Add missing/change incorrect tokens
365
+
366
+ Sometimes converted models will be missing declarations of important tokens like EOT, Fill-in-Middle (prefix, suffix, middle) for various reasons.
367
+ Other times they may have the incorrect tokens set as BOS, EOS, etc. Either way, missing or incorrectly declared tokens means inference will not work as expected.
368
+
369
+ Token declaration is made with the metadata key(s) named "tokenizer.ggml.`token name`\_token\_id" which contains the ID (index number) of the token in the token list (`tokenizer.ggml.tokens`).
370
+
371
+ A recurring issue is misconfigured EOS/EOT/EOM tokens, the need to set each of these and what they should be will vary between models, but the effect when these are incorrect is usually the same;
372
+ infinte generation responses, ie. inference does not know when to stop. Typically this would be because f.ex. EOS has been set to <|endoftext|> instead of <|im\_end|> (again, model specific, just an example).
373
+
374
+ Another issue, mainly for code models, is that Fill-in-Middle tokens have not been declared (note; not all models have or use such tokens), causing sub-par results for filling in blanks in code/text.
375
+ There are 3 main metadata keys that need to be present for this; tokenizer.ggml.`prefix`\_token\_id, `suffix` and `middle`, sometimes also EOT/EOM if it differs from EOS in this mode.
376
+ They are usually named fim\_`something` or just `PRE`, `SUF` and `MID`, take extra care with DeepSeek-based models where prefix is (...fim...)`begin`, suffix is `hole` and middle is `end`.
377
+ """,
378
+ visible = True,
379
+ ),
380
+ example_keys: dict(
381
+ value = "tokenizer.ggml.prefix_token_id",
382
+ info = "Select or enter any metadata key ending with _token_id",
383
+ visible = True,
384
+ ),
385
+ example_types: dict(
386
+ value = GGUFValueType.UINT32.name,
387
+ visible = True,
388
+ ),
389
+ example_token_select: dict(
390
+ value = "<fim_prefix>",
391
+ label = meta_lookup.label,
392
+ info = "You can search for the correct token by parts of its name here, then select the correct one from the list of options",
393
+ visible = True,
394
+ ),
395
+ example_number: dict(
396
+ value = 92295,
397
+ info = "The token ID will be automatically filled in when you select the token, but you can also fill in the ID directly",
398
+ visible = True,
399
+ ),
400
+ },
401
+ ),
402
+ dict(
403
+ label = "Setting the correct token type for a token",
404
+ outputs = {
405
+ example_description: dict(
406
+ value = """## Changing a token's type
407
+
408
+ A common issue is not declaring special control tokens as such, leading to bad tokenization of them when used (usually in the chat template), causing poor responses from the model.
409
+
410
+ Take f.ex. a model with an incorrectly configured <|im\_start|> token as a normal token instead of a special control token, given the following prompt:
411
+ ```
412
+ <|im_start|>Hello World<|im_end|>
413
+ ```
414
+
415
+ This prompt would then be incorrectly tokenized as follows:
416
+ ```
417
+ 27 ('<')
418
+ 91 ('|')
419
+ 318 ('im')
420
+ 4906 ('_start')
421
+ 91 ('|')
422
+ 29 ('>')
423
+ 9707 ('Hello')
424
+ 4337 (' World')
425
+ 151645 ('<|im_end|>')
426
+ ```
427
+
428
+ instead of:
429
+ ```
430
+ 151644 ('<|im_start|>')
431
+ 9707 ('Hello')
432
+ 4337 (' World')
433
+ 151645 ('<|im_end|>')
434
+ ```
435
+
436
+ Take care to also adjust the value for this token in `tokenizer.ggml.scores` (if it exists) similarly to other special control tokens.
437
+
438
+ **WARNING**: Even though you have the option to, you should never remove the `tokenizer.ggml.token_type` key!
439
+ """,
440
+ visible = True,
441
+ ),
442
+ example_keys: dict(
443
+ value = "tokenizer.ggml.token_type",
444
+ visible = True,
445
+ ),
446
+ example_types: dict(
447
+ value = GGUFValueType.INT32.name,
448
+ visible = True,
449
+ ),
450
+ example_delete: dict(
451
+ visible = True,
452
+ ),
453
+ example_token_select: dict(
454
+ value = "<|im_start|>",
455
+ info = "You can search for the token by parts of its name here, then select it from the list of options",
456
+ visible = True,
457
+ ),
458
+ example_token_type: dict(
459
+ value = TokenType.CONTROL.name,
460
+ info = "Select the appropriate token type, in this case we set it as a special control token",
461
+ visible = True,
462
+ ),
463
+ },
464
+ ),
465
+ dict(
466
+ label = "Updating or adding a chat template",
467
+ outputs = {
468
+ example_description: dict(
469
+ value = """## Modifying the Chat Template
470
+
471
+ The chat template is a very important part of the model metadata as this provides a template for how to format the conversation prompt to the model.
472
+ It's not uncommon for these to have bugs (or sometimes just be plain wrong), requiring you to update them to be able to prompt the model correctly.
473
+
474
+ It's also possible to have multiple chat templates for different purposes, the main ones being RAG and Tools, but you can create any additional template you want.
475
+ The standard metadata key for RAG is `tokenizer.chat_template.rag` and Tools is `tokenizer.chat_template.tool_use`, any metadata key added starting with `tokenizer.chat_template.` will be added as a custom chat template.
476
+
477
+ Any framework based on `llama-cpp-python` will let you select which chat template to use with the `chat_format` option, available as `chat_template.default`, `chat_template.rag`, `chat_template.tool_use`, etc...
478
+ """,
479
+ visible = True,
480
+ ),
481
+ example_keys: dict(
482
+ value = "tokenizer.chat_template",
483
+ info = 'Select this or enter any key starting with "tokenizer.chat_template."',
484
+ visible = True,
485
+ ),
486
+ example_types: dict(
487
+ value = GGUFValueType.STRING.name,
488
+ visible = True,
489
+ ),
490
+ example_delete: dict(
491
+ visible = True,
492
+ ),
493
+ example_string: dict(
494
+ info = "Paste in the updated chat template or make changes here. Using an external Jinja2 editor is recommended",
495
+ value = "{%- for message in messages %}\n {{- '<|' + message['role'] + '|>\\n' }}\n {{- message['content'] + eos_token }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|assistant|>\\n' }}\n{%- endif %}",
496
+ visible = True,
497
+ ),
498
+ },
499
+ ),
500
+ ]
501
+
502
+ examples = gr.Dataset(
503
+ label = "Choose an example",
504
+ type = "index",
505
+ samples = [[]] * len(example_properties),
506
+ sample_labels = [x["label"] for x in example_properties],
507
+ )
508
+
509
+ @gr.on(
510
+ triggers = [
511
+ examples.click,
512
+ ],
513
+ inputs = [
514
+ examples,
515
+ ],
516
+ outputs = [
517
+ ] + example_components,
518
+ show_progress = "hidden",
519
+ )
520
+ def show_example(
521
+ value: int,
522
+ ):
523
+ outputs = example_properties[value]["outputs"]
524
+ non_outputs = example_components - outputs.keys()
525
+ all_outputs = dict(((k, type(k)(**(example_defaults[k] | v))) for k, v in outputs.items()))
526
+
527
+ for output in non_outputs:
528
+ all_outputs[output] = type(output)(**example_defaults[output])
529
+
530
+ return all_outputs
531
+
532
+ for k, v in example_defaults.items():
533
+ for prop, val in v.items():
534
+ setattr(k, prop, val)
535
+
536
+ example_group.render()
537
 
538
  meta_state = gr.State() # init_state
539
  # BUG: For some reason using gr.State initial value turns tuple to list?