ElPlaguister commited on
Commit
47eae50
Β·
1 Parent(s): c8a4b93

Feat Generalize Special Token Replacement

Browse files
Files changed (1) hide show
  1. model.py +3 -2
model.py CHANGED
@@ -9,6 +9,7 @@ class Model:
9
  self.tokenizer = None
10
  self.gen_config = None
11
  self.INPUT_FORMAT = None
 
12
 
13
  def generate(self, inputs:str) -> str:
14
  outputs = inputs
@@ -16,7 +17,7 @@ class Model:
16
 
17
  def chat(self, message, chat_history):
18
  response = self.generate(message)
19
- response = response.replace('#ν™”μž#', 'λ„ˆ')
20
- response = response.replace('#청자#', 'λ„ˆ')
21
  chat_history.append((message, response))
22
  return "", gr.Chatbot(chat_history, label=self.name, bubble_full_width=False)
 
9
  self.tokenizer = None
10
  self.gen_config = None
11
  self.INPUT_FORMAT = None
12
+ self.SPETIAL_TOKENS = ["#ν™”μž#", "#청자#", "#(λ‚¨μž)청자#", "#(λ‚¨μž)ν™”μž#", "#(μ—¬μž)청자#", "(μ—¬μž)ν™”μž"]
13
 
14
  def generate(self, inputs:str) -> str:
15
  outputs = inputs
 
17
 
18
  def chat(self, message, chat_history):
19
  response = self.generate(message)
20
+ for special_token in self.SPETIAL_TOKENS:
21
+ response = response.replace(special_token, 'λ„ˆ')
22
  chat_history.append((message, response))
23
  return "", gr.Chatbot(chat_history, label=self.name, bubble_full_width=False)