Upload 9 files

.env

@@ -0,0 +1,9 @@
+CLIENT_ID=340585044850-o53v0f2eam1is8f50jk4v9jvl4fskasi.apps.googleusercontent.com
+PROJECT_ID=moroccan-news-file-share
+AUTH_URI=https://accounts.google.com/o/oauth2/auth
+TOKEN_URI=https://oauth2.googleapis.com/token
+AUTH_PROVIDER_X509_CERT_URL=https://www.googleapis.com/oauth2/v1/certs
+CLIENT_SECRET=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
+REFRESH_TOKEN=1//04FPKbmAI7iv3CgYIARAAGAQSNwF-L9IrFAlZLzJUUf6c3OYtbJhGm_OO9oa3RW4WuNdDWRIZ8QmjdWI083D7HXxmKj8xPe4gA7w
+REDIRECT_URIS=https://developers.google.com/oauthplayground,http://localhost:8501,http://localhost:8080
+

app.py

@@ -0,0 +1,106 @@
+#web interface
+
+import streamlit as st
+import pandas as pd
+import json
+import importlib
+import google_drive_handle as gdrive
+from dotenv import load_dotenv
+import os
+
+# Load config.json
+with open('config.json') as f:
+    config = json.load(f)
+
+drive = gdrive.authenticate_google_drive()
+processed_files = set()
+st.markdown(
+    """
+    <style>
+        .centered {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            text-align: center;
+        }
+    </style>
+    """,
+    unsafe_allow_html=True
+)
+
+st.markdown("<h1 class='centered'>Moroccan News Aggregator</h1>", unsafe_allow_html=True)
+
+selected_websites = {}
+selected_categories = {}
+
+def save_file_id_mapping(file_id_mapping):
+    with open("file_id_mapping.json", "w") as file:
+        json.dump(file_id_mapping, file)
+
+def load_file_id_mapping():
+    try:
+        with open("file_id_mapping.json", "r") as file:
+            return json.load(file)
+    except FileNotFoundError:
+        return {}  # Return an empty dictionary if the file doesn't exist
+
+file_id_mapping = load_file_id_mapping()
+
+for website, details in config.items():
+    if st.checkbox(website, key=website):
+        # Language selection
+        languages = details.get("languages", {})
+        if languages and len(languages) > 1:
+            language = st.selectbox(f'Choose language for {website}', list(languages.keys()), key=f'lang_{website}')
+            selected_websites[website] = f"{website}_{language}"  # like: hespress_en
+        else:
+            selected_websites[website] = website  # like: akhbarona
+
+        # Category selection
+        categories = languages.get(language, {})
+        if categories:
+            categories = st.multiselect(f'Select categories for {website}', list(categories.keys()), key=f'{website}_categories')
+            selected_categories[website] = categories
+
+# Number of articles input
+num_articles = st.number_input('Number of Articles', min_value=1, max_value=10000, step=1)
+
+# Start scraping button
+if st.button('Start Scraping'):
+    with st.spinner('Scraping in progress...'):
+        progress_bar = st.progress(0)
+        total_tasks = sum(len(categories) for categories in selected_categories.values())
+        completed_tasks = 0
+        for website, module_name in selected_websites.items():
+            scraper_module = importlib.import_module(module_name)
+            for category in selected_categories.get(website, []):
+                category_url = config[website]['languages'][language][category]
+               
+                file_path = scraper_module.scrape_category(category_url, num_articles)
+
+                if file_path:
+                    if file_path not in file_id_mapping:
+                        file_id = gdrive.upload_file_to_drive(drive, file_path)
+                        print(f"Uploading file: {file_path}, File ID: {file_id}")
+                        file_id_mapping[file_path] = file_id
+                        save_file_id_mapping(file_id_mapping)
+                    else:
+                        file_id = file_id_mapping[file_path]
+                        print(f"File already uploaded. Using existing File ID: {file_id}")
+            
+                    if file_id:
+                        download_link = gdrive.get_drive_download_link(drive, file_id)
+                        if download_link:
+                            #st.markdown(f"[Download {website} - {category} data]({download_link})", unsafe_allow_html=True)
+
+                            df = pd.read_csv(file_path)
+                            st.write(f"{website} - {category} Data:")
+                            st.dataframe(df)
+                        else:
+                            st.error(f"Failed to retrieve download link for file ID: {file_id}")
+                    else:
+                        st.error(f"Failed to upload file for {website} - {category}")
+                else:
+                    st.error(f"File not created for {website} - {category}")
+
+        st.success('Scraping Completed!')

config.json

@@ -0,0 +1,54 @@
+{
+  "hespress": {
+    "languages": {
+      "en": {
+        "politics": "https://en.hespress.com/politics",
+        "economy": "https://en.hespress.com/economy",
+        "society": "https://en.hespress.com/society",
+        "culture": "https://en.hespress.com/culture",
+        "sports": "https://en.hespress.com/sports",
+        "mena": "https://en.hespress.com/mena",
+        "international": "https://en.hespress.com/international"
+      },
+      "ar": {
+        "Politique": "https://www.hespress.com/politique",
+        "Economie": "https://www.hespress.com/economie",
+        "Tamazight": "https://www.hespress.com/tamazight",
+        "Sport": "https://www.hespress.com/sport",
+        "Sociètè": "https://www.hespress.com/societe",
+        "Culture": "https://www.hespress.com/art-et-culture",
+        "Mèdias": "https://www.hespress.com/medias",
+        "faits-divers": "https://www.hespress.com/faits-divers",
+        "Automoto": "https://www.hespress.com/automoto",
+        "Regions": "https://www.hespress.com/regions"
+      },
+      "fr": {
+        "Politique": "https://fr.hespress.com/politique",
+        "Economie": "https://fr.hespress.com/economie",
+        "Monde": "https://fr.hespress.com/monde",
+        "Sport": "https://fr.hespress.com/sport",
+        "Sociètè": "https://fr.hespress.com/societe",
+        "Culture": "https://fr.hespress.com/culture",
+        "Mèdias": "https://fr.hespress.com/media",
+        "High-tech": "https://fr.hespress.com/high-tech",
+        "Opinions": "https://fr.hespress.com/opinions",
+        "Regions": "https://fr.hespress.com/regions"
+      }
+    },
+    "module": "hespress"
+  },
+  "akhbarona": {
+    "languages": {
+      "ar": {},
+      "fr": {}
+    },
+    "module": "akhbarona"
+  },
+  "le360": {
+    "languages": {
+      "ar": {},
+      "fr": {}
+    },
+    "module": "le360"
+  }
+}

economy_data_en.csv

@@ -0,0 +1,23 @@
+Title,Date,Category,Content,Link,Image
+Forum Mines Rabat Entreprises fosters discussions on Morocco's industrial sovereignty,Thursday 8 February 2024 - 10:40,economy,"The 24th edition of “Forum Mines Rabat Entreprises,” hosted by the Forum Committee of Rabat’s National School of Mines on February 6-7 to, serving as an opportunity to discuss the key role of expertise in safeguarding Morocco’s industrial sovereignty.
+Held under the theme of “The Kingdom’s New Industrial Strategy: Support for Innovation and Realization of Moroccan Sovereignty,” the forum brought together different experts from the industrial sector to highlight the importance of innovation and explore ways to strengthen Morocco’s industrial sovereignty.
+Among them were Ayoub Daoudi, the General Manager of TE Connectivity North Africa, Laurent Figari, the General Manager of Safran Electronics & Defense, and Abderrazak Ben Saga, Head of the Information and Guidance Division and National Coordination of Career Centers.
+
+
+
+
+In this regard, Daoudi highlighted the two trends and essential drivers that shape all technological evolutions: “all green and all connected.”
+The notion of “all connected” signifies the rise in the presence of artificial intelligence and electronics, explained the expert, leading to a surge in the adoption of automation.
+This connectivity implies that the level of reliability will play an increasingly vital role, he underscored.
+This transformative shift holds inevitable implications for both the industrial sector and professionals, namely engineers and technicians of tomorrow.
+“For this reason, the demand for a higher level of expertise will continue to grow,” underscored the General Manager of TE Connectivity North Africa.
+Speaking of the two essential elements of industrial sovereignty, Daoudi highlighted that “while capital holds its significance, it is the expertise and know-how that play a truly essential role in ensuring industrial sovereignty.”
+They are key elements for building and securing industrial sovereignty from one generation to the next, he stressed.
+Daoudi emphasized the vital need to align our educational system with technological skills in today’s environment. He believes that expertise and technical proficiency are interdependent and inseparable.
+“We no longer need engineers who know everything but master nothing. Instead, a combination of both profiles is necessary,” he stressed.
+The General Manager of TE Connectivity North Africa also acknowledged the fact that Morocco places a high priority on developing knowledge and expertise in all sectors that are pivotal for achieving industrial sovereignty.
+“This technological aspect will enable us to build our ecosystem and adapt to macroeconomic geopolitical changes worldwide,” noted Daoudi.
+On the second day of the forum, the event also featured the participation of the Minister of Industry and Commerce, Ryad Mezzour. His address centered around the Kingdom’s industrial strategy, adding a political and strategic dimension to the forum.
+The 24th edition of “Forum Mines Rabat Entreprises,” served as a platform to enhance interaction between students and the job market by building connections with various national and international companies.
+This program provided an important opportunity for students from many professions to make a more effective transition into professional life.
+It fostered an environment suitable for sharing knowledge and exploring internship and career opportunities, bridging the gap between academia and industry.",https://en.hespress.com/79478-forum-mines-rabat-entreprises-fosters-discussions-on-moroccos-industrial-sovereignty.html,https://e1.hespress.com/wp-content/uploads/2024/02/WhatsApp-Image-2024-02-08-at-20.23.17-scaled-e1707422272796-900x600.jpeg

file_id_mapping.json

@@ -0,0 +1 @@
+{"/home/tajeddine/Desktop/news-scraper/faits-divers_data_ar.csv": "1Bl8U3cDG7dNHD4qmWI6oSwyBHo_6-ZEk", "/home/tajeddine/Desktop/news-scraper/art-et-culture_data_ar.csv": "1a_QbvGU04AQ2jGWTNUJ8140iK6oAtrrk", "C:\\Users\\Lenovo\\Documents\\MDS projects\\news-scraper_updated\\v4\\news-scraper\\economy_data_en.csv": null}

google_drive_handle.py

@@ -0,0 +1,64 @@
+from dotenv import load_dotenv
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+from oauth2client.client import OAuth2Credentials
+import os
+
+load_dotenv()
+
+CLIENT_ID = os.getenv('CLIENT_ID')
+CLIENT_SECRET = os.getenv('CLIENT_SECRET')
+REFRESH_TOKEN = os.getenv('REFRESH_TOKEN')
+REDIRECT_URI = os.getenv('REDIRECT_URIS').split(',')[0]  # Access the first URI
+
+def authenticate_google_drive():
+    gauth = GoogleAuth()
+    gauth.credentials = OAuth2Credentials(None, CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN, None,
+                                         "https://accounts.google.com/o/oauth2/token", None, "web")
+    drive = GoogleDrive(gauth)
+    return drive
+
+drive = authenticate_google_drive()
+
+def upload_file_to_drive(drive, file_path, folder_id=None):
+    if not os.path.exists(file_path):
+        print(f"Cannot upload, file does not exist at path: {file_path}")
+        return None
+
+    try:
+        file_metadata = {'title': os.path.basename(file_path)}
+        if folder_id:
+            file_metadata['parents'] = [{'id': folder_id}]
+
+        upload_file = drive.CreateFile(file_metadata)
+
+        # Check if the file already exists on Google Drive
+        existing_files = drive.ListFile({'q': f"title='{upload_file['title']}'"}).GetList()
+        if existing_files:
+            # File with the same name already exists, update the existing file
+            upload_file = existing_files[0]
+            print(f"File already exists on Drive. Updating file with ID: {upload_file['id']}")
+        else:
+            print("Uploading a new file to Drive.")
+
+        upload_file.SetContentFile(file_path)
+        upload_file.Upload()
+        print(f"File uploaded successfully. File ID: {upload_file['id']}")
+        return upload_file['id']
+    except Exception as e:
+        print(f"An error occurred during file upload: {e}")
+        return None
+
+
+def get_drive_download_link(drive, file_id):
+    try:
+        file = drive.CreateFile({'id': file_id})
+        file.Upload() # Make sure the file exists on Drive
+        file.InsertPermission({
+            'type': 'anyone',
+            'value': 'anyone',
+            'role': 'reader'})
+        return "https://drive.google.com/uc?export=download&id=" + file_id
+    except Exception as e:
+        print(f"Error fetching download link: {e}")
+        return None

hespress_ar.py

@@ -0,0 +1,151 @@
+from selenium import webdriver
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import TimeoutException
+from bs4 import BeautifulSoup
+import time
+import re
+import os
+import requests
+import csv
+from urllib.parse import urljoin
+from google_drive_handle import authenticate_google_drive
+drive = authenticate_google_drive()
+
+
+# Set up Chrome WebDriver with options
+options = ChromeOptions()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--disable-dev-shm-usage')
+
+# Initialize the Chrome WebDriver
+wd = webdriver.Chrome(options=options)
+
+def download_image(img_url):
+    return img_url
+
+def scroll_page(expected_article_count):
+    scroll_pause_time = 2
+    screen_height = wd.execute_script("return window.innerHeight;")
+    scrolled_height = 0
+
+    while True:
+        scrolled_height += screen_height
+        wd.execute_script(f"window.scrollTo(0, {scrolled_height});")
+        time.sleep(scroll_pause_time)
+        new_height = wd.execute_script("return document.body.scrollHeight")
+        if scrolled_height >= new_height:
+            break
+
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        articles = soup.find_all('div', class_='overlay card')
+        if len(articles) >= expected_article_count:
+            break
+
+def scrape_article_details(article_url):
+    try:
+        wd.get(article_url)
+        WebDriverWait(wd, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'article-content')))
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        content_tag = soup.find('div', class_='article-content')
+        content = content_tag.get_text().strip() if content_tag else ""
+        date_tag = soup.find('small', class_='text-muted time')
+        date = date_tag.get_text().strip() if date_tag else ""
+        image_tag = soup.find('img', class_='wp-post-image')
+        image_url = image_tag['src'] if image_tag else None
+        img_url = download_image(urljoin(article_url, image_url)) if image_url else None
+        return content, date, img_url
+    except TimeoutException:
+        print("Timed out waiting for page elements to load")
+        return "", "", None
+    except Exception as e:
+        print(f"An error occurred while scraping article details: {str(e)}")
+        return "", "", None
+
+def scrape_article_details(article_url):
+
+    try:
+        wd.get(article_url)
+        WebDriverWait(wd, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'article-content')))  # Adjusted to wait for article content
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+
+        content_tag = soup.find('div', class_='article-content')
+        content = content_tag.get_text().strip() if content_tag else ""
+
+        date_tag = soup.find('small', class_='text-muted time')
+        date = date_tag.get_text().strip() if date_tag else ""
+
+        image_tag = soup.find('img', class_='wp-post-image')
+        image_url = image_tag['src'] if image_tag else None
+
+        img_url = download_image(urljoin(article_url, image_url)) if image_url else None
+
+        return content, date, img_url
+
+    except TimeoutException:
+        print("Timed out waiting for page elements to load")
+        return "", "", None, ""
+    except Exception as e:
+        print(f"An error occurred while scraping article details: {str(e)}")
+        return "", "", None, ""
+
+def sanitize_filename(filename):
+    return re.sub(r'[^\w\s-]', '', filename).strip().lower().replace(' ', '_')
+
+def scrape_category(category_url, num_articles):
+    print("Attempting to scrape:", category_url)
+    articles_data = []
+    wd.get(category_url)
+    scroll_page(num_articles)
+
+    soup = BeautifulSoup(wd.page_source, 'html.parser')
+    articles = soup.find_all('div', class_='overlay card')
+    for article in articles[:num_articles]:
+        link_tag = article.find('a', class_='stretched-link')
+        link = link_tag['href'] if link_tag else ""
+        title_tag = article.find('h3', class_='card-title')
+        title = title_tag.get_text().strip() if title_tag else ""
+        content, date, img_url = scrape_article_details(link)
+        article_data = {
+            "Title": title,
+            "Date": date,
+            "Category": category_url.split('/')[-1],
+            "Content": content,
+            "Link": link,
+            "Image": img_url
+        }
+        print(f"Scraping article: {title}, Link: {link}")
+        articles_data.append(article_data)
+
+
+    # Save scraped data to a CSV file
+    category_name = sanitize_filename(category_url.split("/")[-1])
+    csv_file_path = os.path.join(os.getcwd(), f'{category_name}_data_ar.csv')
+    file_mode = 'a' if os.path.exists(csv_file_path) else 'w'
+
+    try:
+            with open(csv_file_path, file_mode, newline='', encoding='utf-8') as file:
+                fieldnames = ["Title", "Date", "Category", "Content", "Link", "Image"]
+                writer = csv.DictWriter(file, fieldnames=fieldnames)
+                if file_mode == 'w':
+                    writer.writeheader()
+                for article in articles_data:
+                    writer.writerow(article)
+            print(f"CSV file saved successfully at {csv_file_path}")
+    except IOError as e:
+            print(f"Failed to save file at {csv_file_path}: {e}")
+            return None  # Return None to indicate failure
+
+    # Check if the file exists before uploading
+    
+    if os.path.exists(csv_file_path):
+        print(f"File successfully created at {csv_file_path}")
+        return csv_file_path
+        
+    else:
+        print(f"Failed to create file for {category_url}")
+        return None

hespress_en.py

@@ -0,0 +1,124 @@
+from selenium import webdriver
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import TimeoutException
+from bs4 import BeautifulSoup
+import time
+import re
+import os
+import requests
+import csv
+from urllib.parse import urljoin
+from google_drive_handle import authenticate_google_drive
+drive = authenticate_google_drive()
+
+
+# Set up Chrome WebDriver with options
+options = ChromeOptions()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--disable-dev-shm-usage')
+
+
+wd = webdriver.Chrome(options=options)
+
+
+def download_image(img_url):
+    return img_url
+
+def scroll_page(expected_article_count):
+    scroll_pause_time = 2
+    screen_height = wd.execute_script("return window.innerHeight;")
+    scrolled_height = 0
+
+    while True:
+        scrolled_height += screen_height
+        wd.execute_script(f"window.scrollTo(0, {scrolled_height});")
+        time.sleep(scroll_pause_time)
+        new_height = wd.execute_script("return document.body.scrollHeight")
+        if scrolled_height >= new_height:
+            break
+
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        articles = soup.find_all('div', class_='overlay card')
+        if len(articles) >= expected_article_count:
+            break
+
+def scrape_article_details(article_url):
+    try:
+        wd.get(article_url)
+        WebDriverWait(wd, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'article-content')))
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        content_tag = soup.find('div', class_='article-content')
+        content = content_tag.get_text().strip() if content_tag else ""
+        date_tag = soup.find('small', class_='text-muted time')
+        date = date_tag.get_text().strip() if date_tag else ""
+        image_tag = soup.find('img', class_='wp-post-image')
+        image_url = image_tag['src'] if image_tag else None
+        img_url = download_image(urljoin(article_url, image_url)) if image_url else None
+        return content, date, img_url
+    except TimeoutException:
+        print("Timed out waiting for page elements to load")
+        return "", "", None
+    except Exception as e:
+        print(f"An error occurred while scraping article details: {str(e)}")
+        return "", "", None
+
+def sanitize_filename(filename):
+    return re.sub(r'[^\w\s-]', '', filename).strip().lower().replace(' ', '_')
+
+def scrape_category(category_url, num_articles):
+    print("Attempting to scrape:", category_url)
+    articles_data = []
+    wd.get(category_url)
+    scroll_page(num_articles)
+
+    soup = BeautifulSoup(wd.page_source, 'html.parser')
+    articles = soup.find_all('div', class_='overlay card')
+    for article in articles[:num_articles]:
+        link_tag = article.find('a', class_='stretched-link')
+        link = link_tag['href'] if link_tag else ""
+        title_tag = article.find('h3', class_='card-title')
+        title = title_tag.get_text().strip() if title_tag else ""
+        content, date, img_url = scrape_article_details(link)
+        article_data = {
+            "Title": title,
+            "Date": date,
+            "Category": category_url.split('/')[-1],
+            "Content": content,
+            "Link": link,
+            "Image": img_url
+        }
+        print(f"Scraping article: {title}, Link: {link}")
+        articles_data.append(article_data)
+
+    # Save scraped data to a CSV file
+    category_name = sanitize_filename(category_url.split("/")[-1])
+    csv_file_path = os.path.join(os.getcwd(), f'{category_name}_data_en.csv')
+    file_mode = 'a' if os.path.exists(csv_file_path) else 'w'
+
+    try:
+        with open(csv_file_path, file_mode, newline='', encoding='utf-8') as file:
+            fieldnames = ["Title", "Date", "Category", "Content", "Link", "Image"]
+            writer = csv.DictWriter(file, fieldnames=fieldnames)
+            if file_mode == 'w':
+                writer.writeheader()
+            for article in articles_data:
+                writer.writerow(article)
+        print(f"CSV file saved successfully at {csv_file_path}")
+    except IOError as e:
+        print(f"Failed to save file at {csv_file_path}: {e}")
+        return None  # Return None to indicate failure
+
+    # Check if the file exists before uploading
+    
+    if os.path.exists(csv_file_path):
+        print(f"File successfully created at {csv_file_path}")
+        return csv_file_path
+        
+    else:
+        print(f"Failed to create file for {category_url}")
+        return None

hespress_fr.py

@@ -0,0 +1,151 @@
+from selenium import webdriver
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import TimeoutException
+from bs4 import BeautifulSoup
+import time
+import re
+import os
+import requests
+import csv
+from urllib.parse import urljoin
+from google_drive_handle import authenticate_google_drive
+drive = authenticate_google_drive()
+
+
+# Set up Chrome WebDriver with options
+options = ChromeOptions()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--disable-dev-shm-usage')
+
+# Initialize the Chrome WebDriver
+wd = webdriver.Chrome(options=options)
+
+def download_image(img_url):
+    return img_url
+
+def scroll_page(expected_article_count):
+    scroll_pause_time = 2
+    screen_height = wd.execute_script("return window.innerHeight;")
+    scrolled_height = 0
+
+    while True:
+        scrolled_height += screen_height
+        wd.execute_script(f"window.scrollTo(0, {scrolled_height});")
+        time.sleep(scroll_pause_time)
+        new_height = wd.execute_script("return document.body.scrollHeight")
+        if scrolled_height >= new_height:
+            break
+
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        articles = soup.find_all('div', class_='overlay card')
+        if len(articles) >= expected_article_count:
+            break
+
+def scrape_article_details(article_url):
+    try:
+        wd.get(article_url)
+        WebDriverWait(wd, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'article-content')))
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+        content_tag = soup.find('div', class_='article-content')
+        content = content_tag.get_text().strip() if content_tag else ""
+        date_tag = soup.find('small', class_='text-muted time')
+        date = date_tag.get_text().strip() if date_tag else ""
+        image_tag = soup.find('img', class_='wp-post-image')
+        image_url = image_tag['src'] if image_tag else None
+        img_url = download_image(urljoin(article_url, image_url)) if image_url else None
+        return content, date, img_url
+    except TimeoutException:
+        print("Timed out waiting for page elements to load")
+        return "", "", None
+    except Exception as e:
+        print(f"An error occurred while scraping article details: {str(e)}")
+        return "", "", None
+
+def scrape_article_details(article_url):
+
+    try:
+        wd.get(article_url)
+        WebDriverWait(wd, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'article-content')))  # Adjusted to wait for article content
+        soup = BeautifulSoup(wd.page_source, 'html.parser')
+
+        content_tag = soup.find('div', class_='article-content')
+        content = content_tag.get_text().strip() if content_tag else ""
+
+        date_tag = soup.find('small', class_='text-muted time')
+        date = date_tag.get_text().strip() if date_tag else ""
+
+        image_tag = soup.find('img', class_='wp-post-image')
+        image_url = image_tag['src'] if image_tag else None
+
+        img_url = download_image(urljoin(article_url, image_url)) if image_url else None
+
+        return content, date, img_url
+
+    except TimeoutException:
+        print("Timed out waiting for page elements to load")
+        return "", "", None, ""
+    except Exception as e:
+        print(f"An error occurred while scraping article details: {str(e)}")
+        return "", "", None, ""
+
+def sanitize_filename(filename):
+    return re.sub(r'[^\w\s-]', '', filename).strip().lower().replace(' ', '_')
+
+def scrape_category(category_url, num_articles):
+    print("Attempting to scrape:", category_url)
+    articles_data = []
+    wd.get(category_url)
+    scroll_page(num_articles)
+
+    soup = BeautifulSoup(wd.page_source, 'html.parser')
+    articles = soup.find_all('div', class_='overlay card')
+    for article in articles[:num_articles]:
+        link_tag = article.find('a', class_='stretched-link')
+        link = link_tag['href'] if link_tag else ""
+        title_tag = article.find('h3', class_='card-title')
+        title = title_tag.get_text().strip() if title_tag else ""
+        content, date, img_url = scrape_article_details(link)
+        article_data = {
+            "Title": title,
+            "Date": date,
+            "Category": category_url.split('/')[-1],
+            "Content": content,
+            "Link": link,
+            "Image": img_url
+        }
+        print(f"Scraping article: {title}, Link: {link}")
+        articles_data.append(article_data)
+
+
+    # Save scraped data to a CSV file
+    category_name = sanitize_filename(category_url.split("/")[-1])
+    csv_file_path = os.path.join(os.getcwd(), f'{category_name}_data_fr.csv')
+    file_mode = 'a' if os.path.exists(csv_file_path) else 'w'
+
+    try:
+            with open(csv_file_path, file_mode, newline='', encoding='utf-8') as file:
+                fieldnames = ["Title", "Date", "Category", "Content", "Link", "Image"]
+                writer = csv.DictWriter(file, fieldnames=fieldnames)
+                if file_mode == 'w':
+                    writer.writeheader()
+                for article in articles_data:
+                    writer.writerow(article)
+            print(f"CSV file saved successfully at {csv_file_path}")
+    except IOError as e:
+            print(f"Failed to save file at {csv_file_path}: {e}")
+            return None  # Return None to indicate failure
+
+    # Check if the file exists before uploading
+    
+    if os.path.exists(csv_file_path):
+        print(f"File successfully created at {csv_file_path}")
+        return csv_file_path
+        
+    else:
+        print(f"Failed to create file for {category_url}")
+        return None