Upload 7 files

finnlp/data_sources/social_media/_base.py

@@ -0,0 +1,19 @@
+from finnlp.data_sources._base import FinNLP_Downloader
+
+class Social_Media_Downloader(FinNLP_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        pass
+
+    def download(self, start_date, end_date, stock = "all"):
+        pass
+
+    def clean_data(self):
+        pass
+
+    def gather_one_day_news(self,date,stock = "all",delay = 0.1):
+        pass
+
+    def transfer_standard_date_to_nonstandard(self,date):
+        pass

finnlp/data_sources/social_media/finnhub_sentiment.py

@@ -0,0 +1,53 @@
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+from tqdm.notebook import tqdm
+import pandas as pd
+import finnhub
+import time
+
+class Finnhub_Sentiment(Social_Media_Downloader):
+    def __init__(self, args = {}):
+        super().__init__(args)
+        assert "token" in args.keys(), "Please input your finnhub token. Avaliable at https://finnhub.io/dashboard"
+        self.finnhub_client = finnhub.Client(api_key=args["token"])
+        self.delay = args["delay"] if "dalay" in args.keys() else 0.7
+
+    def download_sentiment(self, start_date, end_date, stock = "APPL"):
+        self.reddit = pd.DataFrame()
+        self.twitter = pd.DataFrame()
+        self.date_list = pd.date_range(start_date,end_date)
+        days_each_time = 4
+        date_list = self.date_list
+        # cal total lenth
+        if len(date_list)%days_each_time == 0:
+            total = len(date_list)//days_each_time
+        else:
+            total = len(date_list)//days_each_time+1
+        with tqdm(total=total) as bar:
+            while len(date_list):
+                tmp_date_list = date_list[:days_each_time]
+                date_list = date_list[days_each_time:]
+                tmp_start_date = tmp_date_list[0].strftime("%Y-%m-%d")
+                tmp_end_date = tmp_date_list[-1].strftime("%Y-%m-%d")
+                reddit, _stock_name, twitter = self.gather_one_day_sentiment(tmp_start_date,tmp_end_date,stock = stock )
+                self.reddit = pd.concat([self.reddit,reddit])
+                self.twitter = pd.concat([self.twitter,twitter])
+                bar.update(1)
+        self.reddit = self.reddit.sort_values("atTime")
+        self.twitter = self.twitter.sort_values("atTime")
+
+    def gather_one_day_sentiment(self,start_date, end_date, stock = "APPL"):
+        res  = self.finnhub_client.stock_social_sentiment(stock, _from=start_date, to=end_date)
+        reddit = res["reddit"]
+        symbol = res["symbol"]
+        twitter = res["twitter"]
+        reddit = pd.DataFrame(reddit)
+        # print(reddit)
+        
+        twitter = pd.DataFrame(twitter)
+        try:
+            reddit["atTime"] = pd.to_datetime(reddit["atTime"],errors = "ignore")
+            twitter["atTime"] = pd.to_datetime(twitter["atTime"],errors = "ignore")
+        except:
+            pass
+        time.sleep(self.delay)
+        return reddit,symbol,twitter

finnlp/data_sources/social_media/reddit_streaming.py

@@ -0,0 +1,96 @@
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+
+from tqdm import tqdm
+from lxml import etree
+import requests
+import pandas as pd
+import json
+import base64
+
+class Reddit_Streaming(Social_Media_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_streaming_all(self, rounds = 3):
+        # Download the first page by url
+        base_url = "https://www.reddit.com/r/wallstreetbets/new/"
+        pbar = tqdm(total= rounds, desc= "Downloading by pages...")
+        res = self._request_get(base_url)
+        if res is None:
+            raise ConnectionError
+        
+        # get the info from init page
+        html = etree.HTML(res.text)
+        init = html.xpath("//*[@id='data']/text()")[0]
+        init = json.loads(init[14:][:-1])
+        init = init["posts"]["models"]
+        tmp_df = pd.DataFrame(init).T.reset_index(drop = True)
+        self.dataframe = tmp_df
+        init = [i for i in init if len(i)< 12]
+        last_id = init[-1]
+        last_id = self._encode_base64(last_id)
+        
+        pbar.update(1)
+
+        # fetch other pages
+        if rounds > 1:
+            for _ in range(1,rounds):
+                last_id = self._fatch_other_pages(last_id, pbar)
+
+    def _fatch_other_pages(self, last_page, pbar):
+        url = 'https://gql.reddit.com/'
+        headers = {
+            "referer":"https://www.reddit.com/",
+            "authorization": "Bearer -twjFZkBAlpR8gZnZqsGHvz-G5c49PA",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
+        }
+        data = {
+        "id": "02e3b6d0d0d7",
+        "variables": {
+            "name": "wallstreetbets",
+            "includeIdentity": False,
+            "adContext": {
+            "layout": "CARD",
+            "clientSignalSessionData": {
+                "adsSeenCount": 4,
+                "totalPostsSeenCount": 79,
+                "sessionStartTime": "2023-04-07T15:32:13.933Z",
+            }
+            },
+            "isFake": False,
+            "includeAppliedFlair": False,
+            "includeDevPlatformMetadata": True,
+            "includeRecents": False,
+            "includeTrending": False,
+            "includeSubredditRankings": True,
+            "includeSubredditChannels": False,
+            "isAdHocMulti": False,
+            "isAll": False,
+            "isLoggedOutGatedOptedin": False,
+            "isLoggedOutQuarantineOptedin": False,
+            "isPopular": False,
+            "recentPostIds": [],
+            "subredditNames": [],
+            "sort": "NEW",
+            "pageSize": 25,
+            "after": last_page
+            }
+        }
+        response = self._request_post(url = url, headers= headers, json = data)
+        data = json.loads(response.text)
+        data = data["data"]["subredditInfoByName"]["elements"]["edges"]
+        for d in data:
+            if d["node"]["__typename"] == "SubredditPost":
+                tmp = pd.DataFrame(d).T
+                self.dataframe = pd.concat([self.dataframe, tmp])
+                last_id = tmp.id.values[0]
+        
+        last_id = self._encode_base64(last_id)
+        pbar.update(1)
+
+        return last_id
+
+    def _encode_base64(self,id):
+        return base64.b64encode(id.encode('utf-8')).decode()

finnlp/data_sources/social_media/stocktwits_streaming.py

@@ -0,0 +1,50 @@
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+
+import requests
+import pandas as pd
+from tqdm import tqdm
+import json
+
+class Stocktwits_Streaming(Social_Media_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_streaming_stock(self, stock = "AAPL", rounds = 3):
+        url = f"https://api.stocktwits.com/api/2/streams/symbol/{stock}.json"
+        headers = {
+            'accept': 'application/json',
+            'accept-encoding': 'gzip, deflate, br',
+            'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
+            'authorization': 'OAuth 8a881f43cbc7af061ec2aa35deec9b44f7e3cc09',
+            'dnt': '1',
+            'origin': 'https://stocktwits.com',
+            'referer': 'https://stocktwits.com/',
+            
+            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+        }
+        for i in tqdm(range(rounds)):
+            if i == 0:
+                params = { 
+                "filter":"top",
+                "limit":1000,
+                # "max":410000000,
+                }
+            else:
+                params = { 
+                "filter":"top",
+                "limit":1000,
+                "max":max,
+                }
+            response = self._request_get(url = url, headers=headers, params=params)
+            if response is None:
+                print(f"Fetch data fail. Please check your stock name :{stock} and connections. You may raise an issue if you can't solve this problem")
+                continue
+            else:
+                res = json.loads(response.text)
+                max = res["cursor"]["since"]
+                res = pd.DataFrame(res["messages"])
+                self.dataframe = pd.concat([self.dataframe,res])
+        
+        self.dataframe = self.dataframe.reset_index(drop = True)

finnlp/data_sources/social_media/twitter_date_range.py

@@ -0,0 +1,76 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+
+import requests
+from urllib import parse
+from tqdm import tqdm
+from datetime import datetime,timedelta
+import pandas as pd
+import json
+import time
+
+class Twitter_Date_Range(Social_Media_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_date_range_stock(self, start_date, end_date, stock = "AAPL"):
+        self.date_list = pd.date_range(start_date,end_date)
+        res = pd.DataFrame()
+        for date in tqdm(self.date_list, desc= "Downloading by day... "):
+            tmp = self._gather_one_day(date,stock)
+            res = pd.concat([res,tmp])
+        
+        res.created_at = pd.to_datetime(res.created_at)
+        res = res.sort_values("created_at")
+        res = res.reset_index(drop=True)
+        # res = res.query(f"created_at >= @start_date & created_at <= @end_date")
+        res = res[res.created_at >= start_date][res.created_at <= end_date]
+        res = res.reset_index(drop=True)
+        self.dataframe = res
+
+    def _gather_one_day(self, date, stock = "AAPL", pbar = None ,delay = 0.01):
+        time.sleep(delay)
+        next_date = date + timedelta(days=1)
+        date = datetime.strftime(date, "%Y-%m-%d")
+        next_date = datetime.strftime(next_date, "%Y-%m-%d")
+
+        url = "https://twitter.com/i/api/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&q={}&count=20&query_source=typed_query&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel%2CvoiceInfo"
+        url_token = 'https://api.twitter.com/1.1/guest/activate.json'
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
+            'Accept': '*/*',
+            'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
+            'x-guest-token': '',
+            'x-twitter-client-language': 'zh-cn',
+            'x-twitter-active-user': 'yes',
+            'x-csrf-token': '25ea9d09196a6ba850201d47d7e75733',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+            'authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+            'Referer': 'https://twitter.com/',
+            'Connection': 'keep-alive',
+        }
+
+        q = f'{stock} until:{next_date} since:{date}'
+        token = json.loads(requests.post(url_token, headers = headers).text)['guest_token']
+        print(token)
+        headers['x-guest-token'] = token
+        url = url.format(parse.quote(q))
+        print(url)
+        res = self._request_get(url, headers = headers)
+        print(res)
+        if res is not None:
+            try:
+                res = json.loads(res.text)
+                res = pd.DataFrame(res["globalObjects"]["tweets"]).T.sort_values("created_at")
+            except:
+                res = pd.DataFrame()
+        else:
+            res = pd.DataFrame()
+            
+        return res

finnlp/data_sources/social_media/weibo_date_range.py

@@ -0,0 +1,154 @@
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+
+from tqdm import tqdm
+from lxml import etree
+import pandas as pd
+import numpy as np
+import requests
+import datetime
+import time
+import json
+import re
+
+class Weibo_Date_Range(Social_Media_Downloader):
+    def __init__(self, args = {}):
+        super().__init__(args)
+        if "cookies" not in args.keys():
+            raise ValueError("You need first log in at https://weibo.com/ and then copy you cookies and use it as the [value] of [key] \'cookies\' ")
+        self.cookies = args["cookies"]
+        self.dataframe = pd.DataFrame()
+
+    def download_date_range_stock(self, start_date, end_date, start_hour= 0,end_hour = 0,stock = "茅台", delay = 0.01):
+        self.date_list = pd.date_range(start_date, end_date)
+        for date in tqdm(self.date_list, desc = "Downloading by dates..."):
+            date = date.strftime("%Y-%m-%d")
+            self._gather_one_day(date, start_hour, end_hour, stock, delay)
+        self.dataframe = self.dataframe.reset_index(drop = True)
+
+    def _gather_one_day(self,date,start_hour, end_hour, stock = "茅台", delay = 0.01):
+        if start_hour == 0 and end_hour == 0:
+            start_date = datetime.datetime.strptime(date, "%Y-%m-%d")
+            end_date = start_date + datetime.timedelta(days=1)
+            start_date = start_date.strftime("%Y-%m-%d")
+            end_date = end_date.strftime("%Y-%m-%d")
+        else:
+            start_date = date, end_date = date 
+
+        # first page
+        all_urls = self._gather_first_page(start_date, end_date, start_hour, end_hour, stock, delay)
+        # another pages
+        if len(all_urls)>1:
+            base_url=  "https://s.weibo.com/"
+            for url_new in all_urls:
+                url_new = base_url + url_new
+                self._gather_other_pages(date, url_new, delay)
+         
+    def _gather_first_page(self,start_date, end_date, start_hour, end_hour, stock = "茅台", delay = 0.01):  
+        
+        headers = {
+            "cookie": self.cookies,
+            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0", 
+            }
+        
+        params = {
+            "q": stock,
+            "typeall": "1",
+            "suball": "1",
+            "timescope":f"custom:{start_date}-{start_hour}:{end_date}-{end_hour}",
+            "Refer":"g",
+            "page":"1"
+        }
+
+        url = f"https://s.weibo.com/weibo"
+        resp = self._request_get(url, headers=headers, params = params)
+        
+        if resp is None:
+            return "Error"
+
+        if "passport.weibo.com" in resp.url:
+            raise ValueError("Your cookies is useless. Please first log in at https://weibo.com/ and then copy you cookies and use it as the [value] of [key] \'cookies\' ")
+
+        res = etree.HTML(resp.content)
+        # get all pages
+        all_pages = res.xpath('//*[@id="pl_feedlist_index"]/div[3]/div[1]/span/ul/li//@href')
+        items = res.xpath('//div[@class="card-wrap"]')
+        for i in items:
+            ps = i.xpath('.//div[@class="content"]//p')
+            try:
+                content = ps[0].xpath(".//text()")
+                content = ''.join(content)
+                content = content.replace('\n',"")
+                content = content.replace(' ',"")
+                content = content.replace('\u200b',"")
+            except:
+                continue
+            
+            info = ps[1].xpath(".//text()")
+            try:
+                date_content = info[1]
+                date_content = date_content.replace('\n',"")
+                date_content = date_content.replace(' ',"")
+            except:
+                date_content = np.nan
+
+            try:
+                source = info[3]
+            except:
+                source = np.nan
+            
+            tmp = pd.DataFrame([start_date, date_content, source, content]).T
+            tmp.columns = ["date","date_content", "source", "content"]
+            self.dataframe = pd.concat([self.dataframe, tmp])
+
+        time.sleep(delay)
+
+        return all_pages
+    
+    def _gather_other_pages(self, date, url, delay = 0.01):
+        
+        headers = {
+            "cookie": self.cookies,
+            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0", 
+            }
+        
+        resp = self._request_get(url, headers=headers)
+
+        if resp is None:
+            return "Error"
+
+        if "passport.weibo.com" in resp.url:
+            raise ValueError("Your cookies is useless. Please first log in at https://weibo.com/ and then copy you cookies and use it as the [value] of [key] \'cookies\' ")
+
+        res = etree.HTML(resp.content)
+        # get all pages
+        all_pages = res.xpath('//*[@id="pl_feedlist_index"]/div[3]/div[1]/span/ul/li//@href')
+        items = res.xpath('//div[@class="card-wrap"]')
+        for i in items:
+            ps = i.xpath('.//div[@class="content"]//p')
+            try:
+                content = ps[0].xpath(".//text()")
+                content = ''.join(content)
+                content = content.replace('\n',"")
+                content = content.replace(' ',"")
+                content = content.replace('\u200b',"")
+            except:
+                continue
+            
+            info = ps[1].xpath(".//text()")
+            try:
+                date_content = info[1]
+                date_content = date_content.replace('\n',"")
+                date_content = date_content.replace(' ',"")
+            except:
+                date_content = np.nan
+
+            try:
+                source = info[3]
+            except:
+                source = np.nan
+            
+            tmp = pd.DataFrame([date, date_content, source, content]).T
+            tmp.columns = ["date", "date_content", "source", "content"]
+            self.dataframe = pd.concat([self.dataframe, tmp])
+
+        time.sleep(delay)

finnlp/data_sources/social_media/weibo_streaming.py

@@ -0,0 +1,78 @@
+from finnlp.data_sources.social_media._base import Social_Media_Downloader
+
+from tqdm import tqdm
+from lxml import etree
+import pandas as pd
+import requests
+import time
+import json
+import re
+
+class Weibo_Streaming(Social_Media_Downloader):
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_streaming_stock(self, stock = "茅台", rounds = 3):
+        for r in tqdm(range(rounds), desc="Downloading by page.."):
+            page = r+1
+            self._gather_one_page(page, stock)
+
+    def _gather_one_page(self,page, stock = "茅台", delay = 0.01):
+        headers = {
+            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0"
+            }
+        params = {
+            "containerid": f"100103type=61&q={stock}&t=",
+            "page_type": "searchall",
+            "page":page
+        }
+        url = f"https://m.weibo.cn/api/container/getIndex"
+        resp = self._request_get(url, headers=headers, params = params)
+
+        if resp is None:
+            return "Error"
+        
+        res = json.loads(resp.text)
+        res = res["data"]["cards"]
+        res = pd.DataFrame(res)
+
+        pbar = tqdm(total = res.shape[0], desc = "Processing the text content and downloading the full passage...")
+        res[["content_short","content"]] = res.apply(lambda x:self._process_text(x, pbar, delay), axis= 1, result_type= "expand")
+
+        self.dataframe = pd.concat([self.dataframe, res]) 
+    
+    def _process_text(self,x, pbar, delay = 0.01):
+        text = x["mblog"]["text"]
+        text = etree.HTML(text)
+        content_short = text.xpath(".//text()")
+        content_short = ''.join(content_short)
+        
+        link = text.xpath('.//a/@href')
+        link = [l for l in link if "status" in l ]
+        if len(link) >0:
+            base_url = "https://m.weibo.cn/"
+            url_new = base_url + link[0]
+            headers = {
+                "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0"
+            }
+            resp = self._request_get(url_new, headers= headers) 
+            if resp is None:
+                content = content_short
+            else:
+                res = etree.HTML(resp.content)
+                scripts = res.xpath('//script')
+                content = scripts[2].xpath("text()")
+                pattern=re.compile('"text": "(.+),\n')
+                result = pattern.findall(content[0])
+                content = etree.HTML(result[0])
+                content = content.xpath("//text()")
+                content = ''.join(content)
+        else:
+            content = content_short
+
+        pbar.update(1)
+        time.sleep(delay)
+
+        return content_short, content
+