support chat_template.json (and bump dependencies)

README.md

@@ -4,7 +4,7 @@ emoji: 💬📝
 colorFrom: purple
 colorTo: indigo
 sdk: gradio
-sdk_version: 5.6.0
+sdk_version: 5.38.0
 python_version: 3.11
 app_file: app.py
 pinned: false

app.py

@@ -14,7 +14,7 @@ hfapi = HfApi()
 
 class ModelFiles(StrEnum):
     CHAT_TEMPLATE_JSON = "chat_template.json"
-    TOKENIZER_CHAT_TEMPLATE = "tokenizer_chat_template.jinja"
+    TOKENIZER_CHAT_TEMPLATE = "chat_template.jinja"
     TOKENIZER_CONFIG = "tokenizer_config.json"
     TOKENIZER_INVERSE_TEMPLATE = "inverse_template.jinja"
 
@@ -325,21 +325,21 @@ example_values = [
     "content": [
       {
         "type": "text",
-        "content": "Can this animal"
+        "text": "Can this animal"
       },
       {
         "type": "image"
       },
       {
         "type": "text",
-        "content": "make this sound"
+        "text": "make this sound"
       },
       {
         "type": "audio"
       },
       {
         "type": "text",
-        "content": "while moving like this?"
+        "text": "while moving like this?"
       },
       {
         "type": "video"
@@ -379,7 +379,7 @@ class TokenizerConfig():
     @chat_template.setter
     def chat_template(self, value: str | list | None):
         if not value:
-            self.chat_templates.clear()
+            self.chat_templates = {}
         elif isinstance(value, str):
             self.chat_templates = {
                 "default": value,
@@ -711,6 +711,9 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
         org_template_tool_use = ""
         org_template_rag = ""
 
+        tokenizer_chat_template = info.get(ModelFiles.TOKENIZER_CHAT_TEMPLATE, {})
+        org_template = tokenizer_chat_template.get("data", org_template)
+
         for config_file_name in (ModelFiles.CHAT_TEMPLATE_JSON, ModelFiles.TOKENIZER_CONFIG):
             config_file = info.get(config_file_name, {})
             org_config = config_file.get("data")
@@ -719,7 +722,7 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
             if org_content and ("chat_template" in org_content or not org_template):
                 tokenizer_config = TokenizerConfig(org_content)
 
-                org_template = tokenizer_config.chat_templates.get("default") or ""
+                org_template = org_template or tokenizer_config.chat_templates.get("default") or ""
                 org_template_tool_use = tokenizer_config.chat_templates.get("tool_use") or ""
                 org_template_rag = tokenizer_config.chat_templates.get("rag") or ""
                 # org_template_inverse = tokenizer_config.inverse_template or ""
@@ -738,9 +741,6 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
                     for token in unified_diff(new_config.splitlines(keepends = True), org_config.splitlines(keepends = True), fromfile = config_file_name, tofile = config_file_name)
                 ]
 
-        tokenizer_chat_template = info.get(ModelFiles.TOKENIZER_CHAT_TEMPLATE, {})
-        org_template = tokenizer_chat_template.get("data", org_template)
-
         tokenizer_inverse_template = info.get(ModelFiles.TOKENIZER_INVERSE_TEMPLATE, {})
         org_template_inverse = tokenizer_inverse_template.get("data", org_template_inverse)
 
@@ -1171,6 +1171,24 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
                         revision = parent_commit or branch,
                         token = oauth_token.token if oauth_token else False,
                     )
+
+                if ModelFiles.TOKENIZER_CHAT_TEMPLATE not in info and (hfapi.file_exists(
+                    repo,
+                    ModelFiles.TOKENIZER_CHAT_TEMPLATE,
+                    revision = branch,
+                    token = oauth_token.token if oauth_token else False,
+                )):
+                    tokenizer_chat_template = hfapi.hf_hub_download(
+                        repo,
+                        ModelFiles.TOKENIZER_CHAT_TEMPLATE,
+                        revision = parent_commit or branch,
+                        token = oauth_token.token if oauth_token else False,
+                    )
+                    with open(tokenizer_chat_template, "r", encoding = "utf-8") as fp:
+                        template_data = fp.read()
+                        info[ModelFiles.TOKENIZER_CHAT_TEMPLATE] = {
+                            "data": template_data,
+                        }
             except Exception as e:
                 pass
             else:
@@ -1183,6 +1201,8 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
                             "content": template_content,
                         }
                         info["chat_template"] = template_content.get("chat_template")
+                elif ModelFiles.TOKENIZER_CHAT_TEMPLATE in info:
+                    info["chat_template"] = info[ModelFiles.TOKENIZER_CHAT_TEMPLATE].get("data")
 
         pr_details = None
         if branch and branch.startswith("refs/pr/"):
@@ -1216,8 +1236,8 @@ You can freely edit and test GGUF chat template(s) (and are encouraged to do so)
             pr_submit: gr.Button(
                 value = f"Commit to PR #{pr_details.num}" if pr_details else "Create Pull Request",
             ),
-            chat_template: gr.skip() if ModelFiles.CHAT_TEMPLATE_JSON not in info else gr.Code(
-                value = TokenizerConfig(info[ModelFiles.CHAT_TEMPLATE_JSON]["content"]).chat_templates.get("default"),
+            chat_template: gr.skip() if ModelFiles.CHAT_TEMPLATE_JSON not in info and ModelFiles.TOKENIZER_CHAT_TEMPLATE not in info else gr.Code(
+                value = info.get(ModelFiles.TOKENIZER_CHAT_TEMPLATE, {}).get("data") or TokenizerConfig(info[ModelFiles.CHAT_TEMPLATE_JSON]["content"]).chat_templates.get("default"),
             ),
             # inverse_template: gr.skip() if ModelFiles.TOKENIZER_INVERSE_TEMPLATE not in info else gr.Code(
             #     value = info[ModelFiles.TOKENIZER_INVERSE_TEMPLATE]["data"],

requirements.txt

@@ -1,5 +1,5 @@
-# gradio[oauth]==5.6.0
-huggingface_hub==0.26.2
+gradio[oauth]==5.38.0
+huggingface_hub==0.33.4
 # gradio_huggingfacehub_search==0.0.8
-transformers==4.46.1
+transformers==4.53.2
 https://huggingface.co/spaces/CISCai/chat-template-editor/resolve/main/gradio_huggingfacehub_search-0.0.8-py3-none-any.whl