shadowsword commited on
Commit
2076634
·
1 Parent(s): 5dc2d78

Update token estimation method

Browse files
Files changed (1) hide show
  1. dai2.py +4 -2
dai2.py CHANGED
@@ -1,6 +1,7 @@
1
  import discord
2
  import asyncio
3
  import aiohttp
 
4
  from time import time
5
  from discord.ext import commands
6
  from discord import app_commands
@@ -128,8 +129,9 @@ def restart_program():
128
 
129
  # Every 4 characters is an estimated token.. this could be improved
130
  def estimate_tokens(input_string:str):
131
- tokens = [input_string[i:i+4] for i in range(0, len(input_string), 4)]
132
- return len(tokens)
 
133
 
134
  chats = {} # Chat History
135
  queues = [] # Job Queue
 
1
  import discord
2
  import asyncio
3
  import aiohttp
4
+ import re
5
  from time import time
6
  from discord.ext import commands
7
  from discord import app_commands
 
129
 
130
  # Every 4 characters is an estimated token.. this could be improved
131
  def estimate_tokens(input_string:str):
132
+ pattern = r'\s+|\b\w+\b|\W'
133
+ all = re.findall(pattern, input_string)
134
+ return (len([i for i in all if ' ' != i])+1)
135
 
136
  chats = {} # Chat History
137
  queues = [] # Job Queue