Spaces:

Viraj2307
/

ND-Scrapper

Sleeping

File size: 9,387 Bytes

9e6917b

import json
import requests
from bs4 import BeautifulSoup
import mongoengine as me
import certifi
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import List
from urllib.parse import urljoin
import uvicorn

# MongoDB connection settings
db_name = 'Deliveroo'
host = 'ac-xvwfer4-shard-00-00.xrv1il9.mongodb.net'
port = 27017
username = 'sahlhubdeliveroo'
password = 'vp5bLCjufKdarP1W'


me.disconnect()
# Connect to the MongoDB database
me.connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where())

# Define the MongoDB document models
class DeliverooLink(me.Document):
    href = me.StringField(required=True)
 
class MenuOption(me.Document):
    name = me.StringField(required=True)
    price = me.StringField(required=True)
    option_groups = me.ListField(me.ReferenceField('MenuOptionGroup'))
    location = me.StringField(required=True)
    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)

class MenuOptionGroup(me.Document):
    group_name = me.StringField(required=True)
    options = me.ListField(me.ReferenceField(MenuOption))
    min_quantity = me.IntField(required=True)
    max_quantity = me.IntField(required=True)
    location = me.StringField(required=True)
    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)

class MenuItem(me.Document):
    category_name = me.StringField(required=True)
    name = me.StringField(required=True)
    description = me.StringField()
    image_url = me.StringField()
    price = me.StringField(required=True)
    option_groups = me.ListField(me.ReferenceField(MenuOptionGroup))
    location = me.StringField(required=True)
    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)

class MenuCategory(me.Document):
    name = me.StringField(required=True)
    items = me.ListField(me.ReferenceField(MenuItem))
    location = me.StringField(required=True)
    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)

# FastAPI app

# Pydantic model for input validation
# class URLRequest(BaseModel):
#     city: str
#     location: str
#     url: str

# @app.post("/fetch_and_process_data")
# async def fetch_and_process_data(request: URLRequest):
#     city = request.city.lower()
#     location = request.location.lower()
#     url = request.url
#     print(f"Fetching restaurant data for {city}, {location}...")

#     print(f"Processing restaurant data from URL: {url}...")
#     data = fetch_restaurant_data(url, location)

#     if not data:
#         raise HTTPException(status_code=404, detail="No data found for the specified URL.")

#     print("Saving data to MongoDB...")
#     save_data_to_db(data, location)

#     print("Data has been processed and saved successfully.")
#     return {"message": "Data has been processed and saved successfully."}

def fetch_restaurant_data(url, location):
    headers = {
        'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
    }

    response = requests.get(url, headers=headers)

    if response.status_code != 200:
        print(f"Failed to fetch the URL: {url}")
        return None

    soup = BeautifulSoup(response.content, 'html.parser')
    script_tag = soup.find('script', id='__NEXT_DATA__')

    if not script_tag:
        print("Script tag not found")
        return None

    json_data = json.loads(script_tag.string)
    json_data = json_data['props']['initialState']['menuPage']['menu']['meta']

    items = json_data['items']
    categories = json_data['categories']
    category_map = {category['id']: category['name'] for category in categories}
    modifier_groups = json_data['modifierGroups']

    modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}

    items_with_modifiers = []
    for item in items:
        category_id = item['categoryId']
        category_name = category_map.get(category_id, 'Unknown')
        
        # Skip items with the "Unknown" category
        if category_name == 'Unknown':
            continue

        item_with_modifiers = {
            "id": item['id'],
            "category_id": category_id,
            "category_name": category_name,
            "name": item['name'],
            "description": item.get('description', ''),
            "price": item['price']['formatted'],
            "img_url": item.get('image').get('url', '') if item.get('image') else '',
            "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
            "location": location,
            "restro_ref_id": url
        }
        items_with_modifiers.append(item_with_modifiers)

    return items_with_modifiers

def save_data_to_json(data, filename='restaurants.json'):
    with open(filename, 'w') as f:
        json.dump(data, f, indent=4)

def save_data_to_db(data, location):
    # Dictionary to hold categories and associated items
    category_dict = {}

    for item_data in data:
        category_name = item_data['category_name']
        name = item_data['name']
        description = item_data.get('description', '')
        image_url = item_data.get('img_url', '')
        price = item_data['price']
        option_groups_data = item_data.get('modifier_groups', [])
        restro_ref_id_url = item_data['restro_ref_id']

        restro_link = DeliverooLink.objects(href=restro_ref_id_url).first()
        if not restro_link:
            restro_link = DeliverooLink(href=restro_ref_id_url)
            restro_link.save()

        # Create or retrieve MenuOption instances
        option_groups = []
        for group_data in option_groups_data:
            group_name = group_data['name']
            min_quantity = group_data.get('minSelection', 0)
            max_quantity = group_data.get('maxSelection', 1)
            options_data = group_data.get('modifierOptions', [])

            option_group = MenuOptionGroup.objects(
                group_name=group_name, min_quantity=min_quantity, max_quantity=max_quantity, location=location, restro_ref_id=restro_link).first()

            if not option_group:
                option_group = MenuOptionGroup(
                    group_name=group_name,
                    min_quantity=min_quantity,
                    max_quantity=max_quantity,
                    location=location,
                    restro_ref_id=restro_link
                )
                option_group.save()

            options = []
            for option_data in options_data:
                option_name = option_data['name']
                option_price = option_data['price']['formatted']

                option = MenuOption.objects(name=option_name, price=option_price, location=location, restro_ref_id=restro_link).first()
                if not option:
                    option = MenuOption(name=option_name, price=option_price, location=location, restro_ref_id=restro_link)
                    option.save()

                if option not in option_group.options:
                    option_group.options.append(option)
                    option_group.save()

                if option_group not in option.option_groups:
                    option.option_groups.append(option_group)
                    option.save()

                options.append(option)

            option_groups.append(option_group)
        
        # Create or retrieve MenuItem instance
        menu_item = MenuItem.objects(
            category_name=category_name, name=name, location=location, restro_ref_id=restro_link).first()
        if not menu_item:
            menu_item = MenuItem(
                category_name=category_name,
                name=name,
                description=description,
                image_url=image_url,
                price=price,
                option_groups=option_groups,
                location=location,
                restro_ref_id=restro_link
            )
            menu_item.save()
        else:
            # Ensure option groups are updated for existing menu items
            for option_group in option_groups:
                if option_group not in menu_item.option_groups:
                    menu_item.option_groups.append(option_group)
            menu_item.save()

        # Store the menu item under its category
        if category_name not in category_dict:
            category_dict[category_name] = []
        category_dict[category_name].append(menu_item)

    # Create or retrieve MenuCategory instances and associate items
    for category_name, items in category_dict.items():
        menu_category = MenuCategory.objects(name=category_name, location=location, restro_ref_id=restro_link).first()
        if not menu_category:
            menu_category = MenuCategory(name=category_name, location=location, restro_ref_id=restro_link)
            menu_category.save()

        for menu_item in items:
            if menu_item not in menu_category.items:
                menu_category.items.append(menu_item)
                menu_category.save()

        print(f"Saved item '{name}' in category '{category_name}' for location '{location}'")