Update Cleaning Text

app.py

@@ -42,7 +42,16 @@ def preprocess_text(text, slank_formal_df):
     text = text.lower()
     text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)
     text = re.sub(r'\@\w+|\#', '', text)
-    text = re.sub(r'[^\w\s]', '', text)
+    text = re.sub(r'([^\w\s\U0001F000-\U0001F9FF])\1+', r'\1', text)
+    text = re.sub(r'([\U0001F600-\U0001F64F\U0001F900-\U0001F9FF\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F700-\U0001F77F])', r' \1 ', text)
+    text = re.sub(r'([.,])', r' \1 ', text)
+    text = re.sub(r'[&%]', lambda x: f' {x.group()} ', text)
+    text = re.sub(r'(\w)\1{1,}', r'\1\1', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    text = re.sub(r'\s*-\s*', '-', text)
+    text = re.sub(r'(?<=\d)\s*\.\s*(?=\d)', '.', text)
+    text = re.sub(r'(?<=\d)\s*,\s*(?=\d)', ',', text)
+    text = re.sub(r'\s+', ' ', text).strip()
     text = replace_slank_to_formal(text, slank_formal_df)
     tokens = word_tokenize(text)
     preprocessed_text = ' '.join(tokens)