Update news.py

news.py

@@ -3,6 +3,7 @@ import pandas as pd
 import time
 import pymorphy2
 import re 
+import cloudscraper
 
 from datetime import datetime, timedelta
 from transformers import pipeline
@@ -26,7 +27,8 @@ class NewsData:
         }
 
         self.morph = pymorphy2.MorphAnalyzer() # С помощь него мы будем приводить слова в начальную форму
-    
+        self.scraper = cloudscraper.create_scraper()
+        
     def get_data(self):
         # Парсим сайты за последнии 7 дней
         date_limit = datetime.now().replace(minute=0, second=0, microsecond=0) - timedelta(days=7*1)
@@ -36,10 +38,11 @@ class NewsData:
             page = 1
             while True:
                 url = f"{base_url}{page}/" 
-                response = requests.get(url, headers=self.headers)
-
+                #response = requests.get(url, headers=self.headers)
+                responce = scraper.get(url, headers=self.headers)
+                
                 if response.status_code == 200:
-                    soup = BeautifulSoup(response.content, "html.parser")
+                    soup = BeautifulSoup(response.text, "html.parser")
                     articles = soup.find_all("a", attrs={"data-test": "article-title-link"})
                     
                     daily_count = 0