| --- /home/jon/miniconda3/envs/h2ogpt/lib/python3.10/site-packages/autogen/token_count_utils.py 2024-07-29 21:31:51.630851528 -0700 | |
| +++ /home/jon/token_count_utils.py 2024-07-30 19:13:10.160760647 -0700 | |
| elif "gpt-4" in model: | |
| logger.info("gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") | |
| return _num_token_from_messages(messages, model="gpt-4-0613") | |
| - elif "gemini" in model: | |
| - logger.info("Gemini is not supported in tiktoken. Returning num tokens assuming gpt-4-0613.") | |
| - return _num_token_from_messages(messages, model="gpt-4-0613") | |
| - elif "claude" in model: | |
| - logger.info("Claude is not supported in tiktoken. Returning num tokens assuming gpt-4-0613.") | |
| - return _num_token_from_messages(messages, model="gpt-4-0613") | |
| - elif "mistral-" in model or "mixtral-" in model: | |
| - logger.info("Mistral.AI models are not supported in tiktoken. Returning num tokens assuming gpt-4-0613.") | |
| - return _num_token_from_messages(messages, model="gpt-4-0613") | |
| else: | |
| - raise NotImplementedError( | |
| - f"""_num_token_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" | |
| - ) | |
| + logger.info("%s model is not supported in tiktoken. Returning num tokens assuming gpt-4-0613." % model) | |
| + return _num_token_from_messages(messages, model="gpt-4-0613") | |
| num_tokens = 0 | |
| for message in messages: | |
| num_tokens += tokens_per_message | |