r1208 commited on
Commit
2831fcf
·
verified ·
1 Parent(s): dbd00a1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -6
app.py CHANGED
@@ -8,12 +8,12 @@ import os
8
  def main():
9
 
10
  def get_tokens_as_list(word_list):
11
- "Converts a sequence of words into a list of tokens"
12
- tokens_list = []
13
- for word in word_list:
14
- tokenized_word = tokenizer_with_prefix_space([word], add_special_tokens=False).input_ids[0]
15
- tokens_list.append(tokenized_word)
16
- return tokens_list
17
 
18
 
19
  def translate(text):
 
8
  def main():
9
 
10
  def get_tokens_as_list(word_list):
11
+ "Converts a sequence of words into a list of tokens"
12
+ tokens_list = []
13
+ for word in word_list:
14
+ tokenized_word = tokenizer_with_prefix_space([word], add_special_tokens=False).input_ids[0]
15
+ tokens_list.append(tokenized_word)
16
+ return tokens_list
17
 
18
 
19
  def translate(text):