Create data_cleaning.py

data_cleaning.py

@@ -0,0 +1,23 @@
+```python
+def clean(df):
+    stop_words = set(stopwords.words('english'))
+    lemmatizer = WordNetLemmatizer()
+    cleaned_headlines = []
+
+    for headline in df['title']:
+        headline = BeautifulSoup(headline, 'html.parser').get_text()
+        headline = re.sub(r'[^a-zA-Z0-9\s]', '', headline)
+        headline = re.sub(r'\s+', ' ', headline).strip()
+        headline = headline.lower()
+
+        words = headline.split()
+        words = [lemmatizer.lemmatize(word) for word in words if word not in stop_words]
+
+        cleaned_headline = ' '.join(words)
+        cleaned_headlines.append(cleaned_headline)
+
+    df['title'] = cleaned_headlines
+    df.drop_duplicates(subset=['title'], inplace=True)
+
+    return df
+```