Update Prothom_alo_fully_scraped.py

Prothom_alo_fully_scraped.py

@@ -1,79 +1,79 @@
-def get_data(number):
-    print("Running Prothom_alo_fully_scraped")
-    ##Necessary imports
-    from deep_translator import GoogleTranslator
-    from selenium import webdriver
-    from selenium.webdriver import chrome
-    from selenium.webdriver import ChromeOptions
-    from datetime import datetime, timedelta
-    import re
-    #PROXY = "45.251.231.113:5678"
-    options = ChromeOptions()
-    #options.add_argument('--proxy-server=%s' % PROXY)
-    #options.add_argument("--headless=new")
-    driver = webdriver.Chrome(options=options)
-    ## Finding Elements by XPATH
-    from selenium.webdriver.common.by import By
-    import time, math
-    driver.get("https://www.prothomalo.com/topic/%E0%A6%B8%E0%A7%9C%E0%A6%95-%E0%A6%A6%E0%A7%81%E0%A6%B0%E0%A7%8D%E0%A6%98%E0%A6%9F%E0%A6%A8%E0%A6%BE")
-    time.sleep(15)
-    news_list=[]
-    news_link=[]
-    publish_date=[]
-    if number<=15:
-        txt=driver.find_elements(By.CLASS_NAME, "title-link")
-        date=driver.find_elements(By.TAG_NAME, "time")
-        for i in range(number):
-            news_list.append(txt[i].text)
-            news_link.append(txt[i].get_attribute("href"))
-            publish_date.append(date[i].text)
-        
-    else:
-        clck=int((number-25)/15 + 2)
-        for i in range(clck):
-            print(i)
-            time.sleep(10)
-            last_height = driver.execute_script("return document.body.scrollHeight")
-            driver.execute_script(f"window.scrollTo(0, {last_height-1050})")
-            button=driver.find_elements(By.CLASS_NAME, "tNj8k")
-            button[0].click()
-        time.sleep(5)
-        txt=driver.find_elements(By.CLASS_NAME, "title-link")
-        date=driver.find_elements(By.TAG_NAME, "time")
-        for i in range(number):
-            news_list.append(txt[i].text)
-            news_link.append(txt[i].get_attribute("href"))
-            publish_date.append(date[i].text)
-
-    ###### Scraping Description modified for translation######
-    from deep_translator import GoogleTranslator
-    from goose3 import Goose
-    from datetime import datetime
-    g = Goose()
-    description=[]
-    News_title=[]
-    publish_date=[]
-    for i in range(len(news_link)):
-        print(i)
-        article = g.extract(url=news_link[i])
-        ### Only for Prothom Alo ###
-        # Access the articleBody field
-        data = article.schema
-        article_body = data.get('articleBody')
-        print("Para length", len(article_body))
-        if(len(article_body)>=2200):
-            article_body=article_body[0:2200]
-        bangla_title=article.title
-        english_title = GoogleTranslator(source='auto', target='en').translate(text=bangla_title)
-        News_title.append(english_title)
-        text = GoogleTranslator(source='auto', target='en').translate(text=article_body)
-        description.append(text)
-        publish_date.append(article.publish_date)
-    # Convert the dates to "day-month-year" format
-    formatted_dates = [datetime.fromisoformat(date).strftime('%d-%m-%Y') for date in publish_date]
-
-    #### Converting the list to a pandas dataframe by converting the list to a dictionary  ###
-    dict={'News Title':News_title,'News Link':news_link,'Publish Date':formatted_dates, 'Description':description}
-    import pandas as pd
-    df=pd.DataFrame(dict)
+def get_data(number):
+    print("Running Prothom_alo_fully_scraped")
+    ##Necessary imports
+    from deep_translator import GoogleTranslator
+    from selenium import webdriver
+    from selenium.webdriver import chrome
+    from selenium.webdriver import ChromeOptions
+    from datetime import datetime, timedelta
+    import re
+    #PROXY = "45.251.231.113:5678"
+    options = ChromeOptions()
+    #options.add_argument('--proxy-server=%s' % PROXY)
+    options.add_argument("--headless=new")
+    driver = webdriver.Chrome(options=options)
+    ## Finding Elements by XPATH
+    from selenium.webdriver.common.by import By
+    import time, math
+    driver.get("https://www.prothomalo.com/topic/%E0%A6%B8%E0%A7%9C%E0%A6%95-%E0%A6%A6%E0%A7%81%E0%A6%B0%E0%A7%8D%E0%A6%98%E0%A6%9F%E0%A6%A8%E0%A6%BE")
+    time.sleep(15)
+    news_list=[]
+    news_link=[]
+    publish_date=[]
+    if number<=15:
+        txt=driver.find_elements(By.CLASS_NAME, "title-link")
+        date=driver.find_elements(By.TAG_NAME, "time")
+        for i in range(number):
+            news_list.append(txt[i].text)
+            news_link.append(txt[i].get_attribute("href"))
+            publish_date.append(date[i].text)
+        
+    else:
+        clck=int((number-25)/15 + 2)
+        for i in range(clck):
+            print(i)
+            time.sleep(10)
+            last_height = driver.execute_script("return document.body.scrollHeight")
+            driver.execute_script(f"window.scrollTo(0, {last_height-1050})")
+            button=driver.find_elements(By.CLASS_NAME, "tNj8k")
+            button[0].click()
+        time.sleep(5)
+        txt=driver.find_elements(By.CLASS_NAME, "title-link")
+        date=driver.find_elements(By.TAG_NAME, "time")
+        for i in range(number):
+            news_list.append(txt[i].text)
+            news_link.append(txt[i].get_attribute("href"))
+            publish_date.append(date[i].text)
+
+    ###### Scraping Description modified for translation######
+    from deep_translator import GoogleTranslator
+    from goose3 import Goose
+    from datetime import datetime
+    g = Goose()
+    description=[]
+    News_title=[]
+    publish_date=[]
+    for i in range(len(news_link)):
+        print(i)
+        article = g.extract(url=news_link[i])
+        ### Only for Prothom Alo ###
+        # Access the articleBody field
+        data = article.schema
+        article_body = data.get('articleBody')
+        print("Para length", len(article_body))
+        if(len(article_body)>=2200):
+            article_body=article_body[0:2200]
+        bangla_title=article.title
+        english_title = GoogleTranslator(source='auto', target='en').translate(text=bangla_title)
+        News_title.append(english_title)
+        text = GoogleTranslator(source='auto', target='en').translate(text=article_body)
+        description.append(text)
+        publish_date.append(article.publish_date)
+    # Convert the dates to "day-month-year" format
+    formatted_dates = [datetime.fromisoformat(date).strftime('%d-%m-%Y') for date in publish_date]
+
+    #### Converting the list to a pandas dataframe by converting the list to a dictionary  ###
+    dict={'News Title':News_title,'News Link':news_link,'Publish Date':formatted_dates, 'Description':description}
+    import pandas as pd
+    df=pd.DataFrame(dict)
     return df