add twitter feature

requirements.txt

@@ -2,5 +2,4 @@ torch
 transformers
 snscrape
 pandas
-matplotlib
-numpy
+matplotlib

twitter_scraper.py

@@ -1,28 +1,14 @@
 import snscrape.modules.twitter as sntwitter
 import pandas as pd
 import re
-import tqdm
-import sys
 
 
-def scrape_tweets(query, max_tweets=10, output_path="./scraper/output/" ):
+def scrape_tweets(query, max_tweets=10):
     tweets_list = []
-
-    tweets_list = []
-    if sys.version_info.minor>=8:
-        for i,tweet in tqdm(enumerate(sntwitter.TwitterSearchScraper(query).get_items())):
-            if max_tweets != -1 and i >= int(max_tweets):
-                break
-            tweets_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username, tweet.likeCount, tweet.retweetCount, tweet.replyCount, tweet.quoteCount, tweet.url, tweet.lang])
-
-        df = pd.DataFrame(tweets_list, columns=['Datetime', 'Tweet Id', 'Text', 'Username', 'Likes', 'Retweets', 'Replies', 'Quotes', 'URL', 'Language'])
-        df = df[df["Language"] == "in"]
-    else:
-        for i,tweet in tqdm(enumerate(sntwitter.TwitterSearchScraper(query).get_items())):
-            if max_tweets != -1 and i >= int(max_tweets):
-                break
-            tweets_list.append([tweet.date, tweet.id, tweet.content])
-        df = pd.DataFrame(tweets_list, columns=['Datetime', 'Tweet Id', 'Text'])
+    for i,tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
+        if max_tweets != -1 and i >= int(2):
+            break
+        tweets_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username, tweet.likeCount, tweet.retweetCount, tweet.replyCount, tweet.quoteCount, tweet.url, tweet.lang])
 
     df = pd.DataFrame(tweets_list, columns=['Datetime', 'Tweet Id', 'Text', 'Username', 'Likes', 'Retweets', 'Replies', 'Quotes', 'URL', 'Language'])
     df = df[df["Language"] == "in"]