quyip commited on
Commit
d9a5c81
·
1 Parent(s): a2d4cda
Files changed (1) hide show
  1. utils/summary_utils.py +3 -2
utils/summary_utils.py CHANGED
@@ -4,7 +4,7 @@ from transformers import pipeline
4
  from utils.tag_utils import filter_tags
5
 
6
  AiSummaryVersion = 4
7
- summarization_pipeline = pipeline("summarization", model="csebuetnlp/mT5_multilingual_XLSum", max_length=512)
8
  en_translation_pipe = pipeline("translation", model="Helsinki-NLP/opus-mt-mul-en")
9
  text_to_tags_pipe = pipeline('text2text-generation', model='models/text2tags')
10
 
@@ -12,13 +12,14 @@ text_to_tags_pipe = pipeline('text2text-generation', model='models/text2tags')
12
  def summarize(id: str, text: str):
13
  print('================= 1')
14
  print(id)
 
15
  print(text)
16
  if text is None or len(text) < 10:
17
  print('================= 2')
18
  return {
19
  "ver": AiSummaryVersion
20
  }
21
- summary = get_summarization(text) if len(text) > 1000 else text
22
  translated = get_en_translation(summary)
23
  tags = get_tags(translated)
24
  tags = sorted(list(set(tags)))
 
4
  from utils.tag_utils import filter_tags
5
 
6
  AiSummaryVersion = 4
7
+ summarization_pipeline = pipeline("summarization", model="csebuetnlp/mT5_multilingual_XLSum", max_length=512, min_length=50)
8
  en_translation_pipe = pipeline("translation", model="Helsinki-NLP/opus-mt-mul-en")
9
  text_to_tags_pipe = pipeline('text2text-generation', model='models/text2tags')
10
 
 
12
  def summarize(id: str, text: str):
13
  print('================= 1')
14
  print(id)
15
+ print(len(text))
16
  print(text)
17
  if text is None or len(text) < 10:
18
  print('================= 2')
19
  return {
20
  "ver": AiSummaryVersion
21
  }
22
+ summary = get_summarization(text) if len(text) > 2500 else text
23
  translated = get_en_translation(summary)
24
  tags = get_tags(translated)
25
  tags = sorted(list(set(tags)))