DD0101 commited on
Commit
44c2c53
·
1 Parent(s): f8e5290

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -1
app.py CHANGED
@@ -14,7 +14,10 @@ rdrsegmenter = py_vncorenlp.VnCoreNLP(annotators=["wseg"], save_dir='/home/user/
14
  class MyPipeline(TokenClassificationPipeline):
15
  def preprocess(self, sentence, offset_mapping=None):
16
  truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False
17
- sentence = rdrsegmenter.word_segment(sentence)
 
 
 
18
  model_inputs = self.tokenizer(
19
  sentence,
20
  return_tensors=self.framework,
 
14
  class MyPipeline(TokenClassificationPipeline):
15
  def preprocess(self, sentence, offset_mapping=None):
16
  truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False
17
+
18
+ sentence = " ".join(rdrsegmenter.word_segment(sentence))
19
+
20
+
21
  model_inputs = self.tokenizer(
22
  sentence,
23
  return_tensors=self.framework,