File size: 1,287 Bytes
22a4d09 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
import os
import logging
from bs4 import BeautifulSoup
import requests
from openpyxl import Workbook
from app.models.product import Product
from app.services.scraping_service import ScrapingService
from app.utils.logger import logger
logger.info("Starting scraping process")
# Set up scraping service
scraping_service = ScrapingService()
# Get list of websites to scrape
websites = ["https://example.com", "https://example.net"]
# Scrape each website
for website in websites:
logger.info(f"Scraping {website}")
html = requests.get(website).text
soup = BeautifulSoup(html, "html.parser")
products = []
for product in soup.find_all("div", {"class": "product"}):
product_name = product.find("h2", {"class": "product-name"}).text.strip()
product_price = product.find("span", {"class": "product-price"}).text.strip()
products.append(Product(product_name, product_price))
logger.info(f"Found {len(products)} products on {website}")
# Save products to Excel file
wb = Workbook()
ws = wb.active
ws.title = "Products"
ws.append(["Product Name", "Product Price"])
for product in products:
ws.append([product.name, product.price])
wb.save(f"{website}.xlsx")
logger.info(f"Saved products to {website}.xlsx") |