Hugging Face's logo

Spaces:
KotVasily
/
myfirspace
Sleeping

App Files Community
myfirspace
File size: 4,148 Bytes 
d39b93e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
import requests
import pandas as pd
import time
import pymorphy2
import re 

from datetime import datetime, timedelta
from transformers import pipeline
from bs4 import BeautifulSoup

class NewsData:
    def __init__(self) -> None:
        """
        Парсер статей с ru.investing.com.
        """
        self.urls = [
            ("https://ru.investing.com/news/forex-news/", "forex-news"),
            ("https://ru.investing.com/news/commodities-news/", "commodities-news"),
        ]
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
            'Accept-Language': 'en-US,en;q=0.5',
            'Referer': 'http://google.com'
        }

        self.morph = pymorphy2.MorphAnalyzer() # С помощь него мы будем приводить слова в начальную форму
    
    def get_data(self):
        # Парсим сайты за последнии 7 дней
        date_limit = datetime.now().replace(minute=0, second=0, microsecond=0) - timedelta(days=7*1)
        articles_data = []

        for base_url, tag in self.urls:
            page = 1
            while True:
                url = f"{base_url}{page}/" 
                response = requests.get(url, headers=self.headers)

                if response.status_code == 200:
                    soup = BeautifulSoup(response.content, "html.parser")
                    articles = soup.find_all("a", attrs={"data-test": "article-title-link"})
                    
                    daily_count = 0

                    for article in articles:
                        title = article.text.strip()
                        date_tag = article.find_next("time", attrs={"data-test": "article-publish-date"}) 
                        url = article["href"]

                        if date_tag:
                            publish_date_str = date_tag["datetime"]
                            publish_date = datetime.strptime(publish_date_str, "%Y-%m-%d %H:%M:%S")
                            
                            articles_data.append([title, publish_date_str, tag, url])
                            daily_count += 1
                       
                    page += 1

                    if publish_date < date_limit:
                        print(f"Дата публикации: {publish_date_str} меньше лимита. Остановка.")
                        print(f"Общее количество скачанных статей: {len(articles_data)}")
                        break
                else:
                    if response.status_code == 404:
                        break

                    print(f"Ошибка при запросе страницы {page}, код ответа: {response.status_code}")
                    time.sleep(5)

        return self.get_proccesing_news(articles_data)
    
    def clean_text(self, text):
        text = re.sub(r'[^а-яА-Я\s]', '', text)  
        text = text.lower()  
        text = ' '.join([word for word in text.split() if len(word) >= 3]) 
        text = ' '.join([self.morph.parse(word)[0].normal_form for word in text.split()])
        return text
    
    def get_proccesing_news(self, articles_data):
        # Обрабатываем полученные данные, добавляем сентиментальный анализ.
        sentiment_model = pipeline("sentiment-analysis", model="mxlcw/rubert-tiny2-russian-financial-sentiment", framework="pt")

        news = pd.DataFrame(articles_data, columns=["title", "DATE", "tags", 'url'])
        
        news['title'] = news['title'].ffill().map(self.clean_text)
        news['sentiment'] = news['title'].progress_apply(lambda x: sentiment_model(x)[0]['label'])
        news = pd.get_dummies(news, columns=['sentiment'])

        news['DATE'] = pd.to_datetime(news['DATE'])
        news['day'] = news['DATE'].dt.day
        news['year'] = news['DATE'].dt.year
        news['month'] = news['DATE'].dt.month
        news['hour'] = news['DATE'].dt.hour
        return news