update requirements.txt

app.py

@@ -3,6 +3,8 @@ import os
 from typing import AnyStr
 
 import nltk
+from nltk.tokenize import sent_tokenize
+from nltk.tokenize import word_tokenize
 import streamlit as st
 import validators
 from transformers import pipeline
@@ -96,20 +98,20 @@ def main() -> None:
     #         return tuple(summarizer.abstractive_summary(list(summary_sentence)))
 
     def split_text(text: str) -> list:
-        tokens = nltk.tokenize(text)
-        sentences = []
+        sentences = sent_tokenize(text)
         token_count = 0
-        sentence = ""
-        for token in tokens:
-            if token_count < 1024:
-                sentence += "".join(token + " ")
-                token_count += 1
+        text_block = ""
+        result = []
+        for sentence in sentences:
+            tokens = word_tokenize(sentence)
+            if token_count + len(tokens) < 1024:
+                token_count += len(tokens)
+                text_block += " ".join(sentence)
             else:
-                sentences.append(sentence)
-                token_count = 0
-                sentence = ""
-
-        return sentences
+                result.append(text_block)
+                text_block = "".join(sentence)
+                token_count = len(tokens)
+        return result
 
     pipe = create_pipeline()