AlexHung29629 commited on
Commit
6524c20
·
verified ·
1 Parent(s): 4bb24b8

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -123,7 +123,7 @@
123
  },
124
  {
125
  "name": "tool_use",
126
- "template": "{{ bos_token }}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|system|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% elif (message['role'] == 'user') %}{% if tools is defined and tools != None and message == user_messages[-1] %}{{'<|function_metadata|>' + '\n'}}{{ tools }}{{'\n'}}{% endif %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{% if message['tool_calls'] == None or message['tool_calls'] == '' %}{{message['content'] + '<|end|>' + '\n'}}{% else %}{{message['content'] + '<|tool_calls|>' + message['tool_calls'] + '<|end|>' + '\n'}}{% endif %}{% elif (message['role'] == 'tool_results') %}{{'<|tool_results|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}"
127
  }
128
  ],
129
  "clean_up_tokenization_spaces": false,
 
123
  },
124
  {
125
  "name": "tool_use",
126
+ "template": "{{ bos_token }}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|system|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% elif (message['role'] == 'user') %}{% if tools is defined and tools != None and message == user_messages[-1] %}{{'<|function_metadata|>' + '\n'}}{{ tools|tojson }}{{'\n'}}{% endif %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{% if not 'tool_calls' in message or message['tool_calls'] == None or message['tool_calls'] == '' %}{{ message['content'] + '<|end|>' + '\n'}}{% else %}{% if 'content' in message %}{{ message['content'] }}{% endif %}{{'<|tool_calls|>' + message['tool_calls']|tojson + '<|end|>' + '\n'}}{% endif %}{% elif (message['role'] == 'tool_results') %}{{'<|tool_results|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}"
127
  }
128
  ],
129
  "clean_up_tokenization_spaces": false,