Upload 5 files

finnlp/data_sources/company_announcement/_base.py

@@ -0,0 +1,22 @@
+from finnlp.data_sources._base import FinNLP_Downloader
+
+class Company_Announcement_Downloader(FinNLP_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        pass
+
+    def download_date_range_all(self, start_date, end_date):
+        pass
+    
+    def download_date_range_stock(self, start_date, end_date, stock = "AAPL"):
+        pass
+    
+    def download_streaming_all(self, rounds = 3):
+        pass
+    
+    def download_streaming_stock(self, stock = None, rounds = 3):
+        pass
+
+    def clean_data(self):
+        pass

finnlp/data_sources/company_announcement/juchao.py

@@ -0,0 +1,138 @@
+from finnlp.data_sources.company_announcement._base import Company_Announcement_Downloader
+
+import requests
+import time
+import json
+import os
+import pandas as pd
+from tqdm import tqdm
+from PyPDF2 import PdfReader
+
+class Juchao_Annoumcement(Company_Announcement_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_date_range_stock(self,start_date, end_date, stock = "000001",max_page = 100, searchkey= "", get_content = False, save_dir = "./tmp/" , delate_pdf = False):
+        self.org_dict = self._get_orgid()
+
+        # download the first page
+        res = self._get_open_page(start_date, end_date, stock, 1, searchkey)
+        total_pages = res["totalpages"]+1
+        
+        if res["announcements"] is None:
+            print(f"Nothing related to your searchkey({searchkey}) is found, you may try another one or just leave it blank")
+        else:
+            tmp_df = self._process_data(res)
+            self.dataframe = pd.concat([self.dataframe, tmp_df])
+
+            page = 2
+            # download other page
+            pbar = tqdm(total=total_pages,desc="Downloading by page...")
+            
+            for _ in range(max_page):
+                res = self._get_open_page(start_date, end_date, stock, page, searchkey) 
+                if res["announcements"] is None:
+                    break
+                tmp_df = self._process_data(res)
+                self.dataframe = pd.concat([self.dataframe, tmp_df])
+                pbar.update(1)
+                page += 1
+            pbar.update(1)
+        # Convert Time
+        self.dataframe.announcementTime = self.dataframe.announcementTime.apply(lambda x:time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(x/1000)))
+        self.dataframe.announcementTime = pd.to_datetime(self.dataframe.announcementTime)
+        
+        if get_content:
+            pbar = tqdm(total=self.dataframe.shape[0], desc="Getting the text data...")
+            self.dataframe[["PDF_path","Content"]] = self.dataframe.apply(lambda x: self._get_pdfs(x,save_dir, delate_pdf, pbar),axis= 1,result_type  = "expand")
+        if delate_pdf:
+            os.removedirs(save_dir)
+
+        self.dataframe = self.dataframe.reset_index(drop = True)
+        
+    def _get_open_page(self,start_date,end_date, stock,page, searchkey):
+        url = "http://www.cninfo.com.cn/new/hisAnnouncement/query?"
+        headers = {
+            "Referer": "http://www.cninfo.com.cn/new/commonUrl/pageOfSearch?url=disclosure/list/search&lastPage=index",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        }
+        data = {
+            "pageNum": page,
+            "pageSize": "30",
+            "column": "szse",
+            "tabName": "fulltext",
+            "plate":"", 
+            "stock":stock + "," + self.org_dict[stock] ,
+            "searchkey": searchkey,
+            "secid":"", 
+            "category":"", 
+            "trade":"", 
+            "seDate": f"{start_date}~{end_date}",
+            "sortName": "", 
+            "sortType": "", 
+            "isHLtitle": "true",
+            }
+        res = requests.post(url = url, headers = headers, data = data)
+        if res.status_code != 200:
+            raise ConnectionError
+        
+        res = json.loads(res.text)
+        return res
+    
+    def _process_data(self,res):
+        if res is None:
+            return res
+        else:
+            return pd.DataFrame(res["announcements"])
+
+    def _get_pdfs(self,x, save_dir, delate_pdf,pbar):
+        os.makedirs(save_dir, exist_ok= True)
+        adjunctUrl = x.adjunctUrl
+        pdf_base_url = "http://static.cninfo.com.cn/"
+        pdf_url = pdf_base_url + adjunctUrl
+        responsepdf = self._request_get(pdf_url)
+        
+
+        if responsepdf is None:
+            pbar.update(1)
+            return ("Failed Download","Failed Download")
+
+        else:
+            # make preparations
+            file_name = x.announcementTitle
+            file_name = "".join(file_name.split("<em>"))
+            file_name = "".join(file_name.split("</em>"))
+            file_name
+            file_name = f"{x.secCode}_{x.secName}_{file_name}.pdf"
+            file_path = os.path.join(save_dir, file_name)
+
+            # save pdf
+            with open(file_path, "wb") as f:
+                f.write(responsepdf.content)
+            
+            # analyze pdf
+            with open(file_path, "rb") as filehandle:
+                pdf = PdfReader(filehandle)
+                text_all = ""
+                for page in pdf.pages:
+                    text = page.extract_text()
+                    text = "".join(text.split("\n"))
+                    text_all += text
+            pbar.update(1)
+
+            if delate_pdf:
+                os.remove(file_path)
+                return ("removed", text_all)
+            else:
+                return (file_path, text_all)          
+
+    def _get_orgid(self):
+        org_dict = {}
+        org_json = self._request_get("http://www.cninfo.com.cn/new/data/szse_stock.json").json()["stockList"]
+
+        for i in range(len(org_json)):
+            org_dict[org_json[i]["code"]] = org_json[i]["orgId"]
+
+        return org_dict

finnlp/data_sources/company_announcement/sec.py

@@ -0,0 +1,145 @@
+from finnlp.data_sources.company_announcement._base import Company_Announcement_Downloader
+
+from tqdm import tqdm
+from lxml import etree
+import pandas as pd
+import requests
+import json
+import time
+
+class SEC_Annoumcement(Company_Announcement_Downloader):
+
+    def __init__(self, args = {}):
+        super().__init__(args)
+        self.dataframe = pd.DataFrame()
+
+    def download_date_range_stock(self, start_date, end_date, stock = "AAPL", delay = 0.1):
+        entityName = self._get_entity_name(stock)
+        # first page
+        total_pages = self._gather_one_page(start_date, end_date, 1, entityName, delay)
+        # other pages
+        if total_pages>1:
+            for page in tqdm(range(1, total_pages), desc="Downloading other page..."):
+                self._gather_one_page(start_date, end_date, page + 1, entityName, delay )
+
+        self.dataframe = self.dataframe.reset_index(drop = True)
+        
+    def _get_entity_name(self, stock = "AAPL"):
+        url = "https://efts.sec.gov/LATEST/search-index"
+        headers = {
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
+        }
+        params = {
+            "keysTyped":stock
+        }
+        resp = self._request_get(url = url, headers= headers, params= params)
+        if resp is None:
+            raise ConnectionError("Can't get entity name")
+        
+        res = json.loads(resp.text)
+        item_list = res["hits"]["hits"]
+        entityName_list = []
+        for item in item_list:
+            c_name_one = item["_source"]["entity_words"]
+            c_name_two = item["_id"].zfill(10)
+            entityName = f"{c_name_one} (CIK {c_name_two})"
+            entityName_list.append(entityName)
+        
+        entityName = entityName_list[0]
+
+        return entityName
+    
+    def _gather_one_page(self, start_date, end_date, page, entityName = "Apple Inc. (AAPL) (CIK 0000320193)", delay = 0.01):
+        from_ = (page-1)*100
+        url = "https://efts.sec.gov/LATEST/search-index"
+        headers = {
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
+        }
+        params = {
+            "dateRange": "all",
+            "entityName": entityName,
+            "startdt": start_date,
+            "enddt": end_date,
+            "from" : from_,
+            "page" : page,
+        }
+
+        resp = self._request_get(url = url, headers= headers, params= params)
+        
+        if resp is None:
+            return 'Error'
+        res = json.loads(resp.text)
+
+        # total
+        total_items = res["hits"]["total"]["value"]
+        if total_items % 100 == 0:
+            total_pages = total_items // 100 
+        else:
+            total_pages = total_items // 100 + 1
+
+        items = res["hits"]["hits"]
+
+        url_base = "https://www.sec.gov/Archives/edgar/data"
+
+        for item in tqdm(items, desc="Downloading by item..." ):
+            url_third = item["_source"]["xsl"]
+            url_second, url_fourth = item["_id"].split(":")
+            url_second = url_second.split("-")
+            url_first = url_second[0]
+            url_first = url_first.strip("0")
+            url_second = ''.join(url_second)
+            url_first, url_second, url_fourth
+
+            if url_third is not None:
+                url_new = f"{url_base}/{url_first}/{url_second}/{url_third}/{url_fourth}"
+            else:
+                url_new = f"{url_base}/{url_first}/{url_second}/{url_fourth}"
+            respn = self._request_get(url = url_new, headers= headers)
+            if respn is None:
+                continue
+            try:
+                res = etree.HTML(respn.text)
+                content = res.xpath("/html/body//text()")
+                content = [c for c in content if c != "\n"]
+                content = "".join(content)
+                
+                _id = item["_id"]
+                ciks = item["_source"]["ciks"]
+                period_ending = item["_source"]["period_ending"]
+                root_form = item["_source"]["root_form"]
+                file_num = item["_source"]["file_num"]
+                display_names = item["_source"]["display_names"]
+                xsl = item["_source"]["xsl"]
+                sequence = item["_source"]["sequence"]
+                file_date = item["_source"]["file_date"]
+                biz_states = item["_source"]["biz_states"]
+                sics = item["_source"]["sics"]
+                form = item["_source"]["form"]
+                adsh = item["_source"]["adsh"]
+                film_num = item["_source"]["film_num"]
+                biz_locations = item["_source"]["biz_locations"]
+                file_type = item["_source"]["file_type"]
+                file_description = item["_source"]["file_description"]
+                inc_states = item["_source"]["inc_states"]
+                ite = item["_source"]["items"]
+
+                data = [
+                    _id, ciks, period_ending, root_form, file_num, display_names, xsl, sequence,
+                    file_date, biz_states, sics, form, adsh, film_num, biz_locations, file_type,
+                    file_description, inc_states, ite, content
+                ]
+                columns = [
+                    "_id", "ciks", "period_ending", "root_form", "file_num", "display_names", "xsl", "sequence",
+                    "file_date", "biz_states", "sics", "form", "adsh", "film_num", "biz_locations", "file_type",
+                    "file_description", "inc_states", "ite", "content"
+                ]
+                tmp = pd.DataFrame(data = data).T
+                tmp.columns = columns
+
+                self.dataframe = pd.concat([self.dataframe, tmp])
+                time.sleep(delay)
+            except:
+                continue
+        
+        return total_pages
+    

finnlp/data_sources/company_announcement/sina.py

@@ -0,0 +1,87 @@
+import time
+import requests
+from lxml import etree
+from tqdm.notebook import tqdm
+import pandas as pd
+
+class Sina_Annoumcement_Downloader:
+
+    def __init__(self, args = {}):
+        pass
+
+    def download(self, stock = "all",max_page = 100):
+        page = 0
+        df = pd.DataFrame()
+        print(f"Getting page: ",end = "")
+        while page < max_page:
+            print(page, end = " ")
+            headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0",
+                'Accept-Encoding':'gzip, deflate, br',}
+            url = f"https://vip.stock.finance.sina.com.cn/corp/view/vCB_AllBulletin.php?stockid={stock}&Page={page}"
+            response = requests.get(url = url,headers=headers)
+            # response.encoding = "GBK"
+            # print(response.content.decode('GBK'))
+            text = response.content.decode('GBK')
+            html = etree.HTML(text)
+
+            # get announcement date
+            date_list = html.xpath("/html/body/div[6]/div[2]/div[2]/table[2]/tr/td[2]/div[1]/ul/text()")
+            if len(date_list) <= 0:
+                break
+            date_list = [date.strip('.\r').strip('.\n').strip('.\xa0').strip(' ') for date in date_list]
+            date_list = [date for date in date_list if len(date) == 10]
+
+
+            # get headlines and urls
+            url_root = "https://vip.stock.finance.sina.com.cn"
+            a_list = html.xpath("/html/body/div[6]/div[2]/div[2]/table[2]/tr/td[2]/div[1]/ul/a")
+            headline_list = [a.xpath("./text()")[0] for a in a_list ]
+            url_list = [url_root + a.xpath("./@href")[0] for a in a_list ]
+            
+            tmp_df = {
+                "date": date_list,
+                "headline": headline_list,
+                "url": url_list,
+            }
+            tmp_df = pd.DataFrame(tmp_df)
+            df = pd.concat([df,tmp_df])
+            page += 1
+        
+        
+        with tqdm(total = df.shape[0],desc = "Getting annoumcement content" ) as pbar:
+            df["content"] = df.apply(lambda x: self.get_content(x,pbar), axis=1 )
+        
+        df = df.reset_index(drop=True)
+
+        return df
+        
+    def get_content(self,x,pbar,delay = 0.1):
+        time.sleep(delay)
+        url = x.url
+        headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0",
+                'Accept-Encoding':'gzip, deflate, br',}
+        response = requests.get(url = url,headers=headers)
+        if response.status_code == 200:
+            try:
+                text = response.content.decode('GBK')
+                html = etree.HTML(text)
+
+                # clean content
+                content_list = html.xpath("//*[@id='content']//text()")
+                content_list = [content.strip('.\t').strip('.\n').strip('.\r') for content in content_list]
+                content_list = [content for content in content_list if len(content) != 0]
+                content = "".join(content_list)
+            except:
+                return "can't get content"
+        else:
+            return "can't get content"
+
+        pbar.update(1)
+
+        return content
+
+    def clean_data(self):
+        pass
+
+    def transfer_standard_date_to_nonstandard(self,date):
+        pass