import json import requests from bs4 import BeautifulSoup import mongoengine as me import certifi from fastapi import FastAPI, HTTPException from pydantic import BaseModel from typing import List from urllib.parse import urljoin import uvicorn # MongoDB connection settings db_name = 'Deliveroo' host = 'ac-xvwfer4-shard-00-00.xrv1il9.mongodb.net' port = 27017 username = 'sahlhubdeliveroo' password = 'vp5bLCjufKdarP1W' me.disconnect() # Connect to the MongoDB database me.connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where()) # Define the MongoDB document models class DeliverooLink(me.Document): href = me.StringField(required=True) class MenuOption(me.Document): name = me.StringField(required=True) price = me.StringField(required=True) option_groups = me.ListField(me.ReferenceField('MenuOptionGroup')) location = me.StringField(required=True) restro_ref_id = me.ReferenceField(DeliverooLink, required=True) class MenuOptionGroup(me.Document): group_name = me.StringField(required=True) options = me.ListField(me.ReferenceField(MenuOption)) min_quantity = me.IntField(required=True) max_quantity = me.IntField(required=True) location = me.StringField(required=True) restro_ref_id = me.ReferenceField(DeliverooLink, required=True) class MenuItem(me.Document): category_name = me.StringField(required=True) name = me.StringField(required=True) description = me.StringField() image_url = me.StringField() price = me.StringField(required=True) option_groups = me.ListField(me.ReferenceField(MenuOptionGroup)) location = me.StringField(required=True) restro_ref_id = me.ReferenceField(DeliverooLink, required=True) class MenuCategory(me.Document): name = me.StringField(required=True) items = me.ListField(me.ReferenceField(MenuItem)) location = me.StringField(required=True) restro_ref_id = me.ReferenceField(DeliverooLink, required=True) # FastAPI app # Pydantic model for input validation # class URLRequest(BaseModel): # city: str # location: str # url: str # @app.post("/fetch_and_process_data") # async def fetch_and_process_data(request: URLRequest): # city = request.city.lower() # location = request.location.lower() # url = request.url # print(f"Fetching restaurant data for {city}, {location}...") # print(f"Processing restaurant data from URL: {url}...") # data = fetch_restaurant_data(url, location) # if not data: # raise HTTPException(status_code=404, detail="No data found for the specified URL.") # print("Saving data to MongoDB...") # save_data_to_db(data, location) # print("Data has been processed and saved successfully.") # return {"message": "Data has been processed and saved successfully."} def fetch_restaurant_data(url, location): headers = { 'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc' } response = requests.get(url, headers=headers) if response.status_code != 200: print(f"Failed to fetch the URL: {url}") return None soup = BeautifulSoup(response.content, 'html.parser') script_tag = soup.find('script', id='__NEXT_DATA__') if not script_tag: print("Script tag not found") return None json_data = json.loads(script_tag.string) json_data = json_data['props']['initialState']['menuPage']['menu']['meta'] items = json_data['items'] categories = json_data['categories'] category_map = {category['id']: category['name'] for category in categories} modifier_groups = json_data['modifierGroups'] modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups} items_with_modifiers = [] for item in items: category_id = item['categoryId'] category_name = category_map.get(category_id, 'Unknown') # Skip items with the "Unknown" category if category_name == 'Unknown': continue item_with_modifiers = { "id": item['id'], "category_id": category_id, "category_name": category_name, "name": item['name'], "description": item.get('description', ''), "price": item['price']['formatted'], "img_url": item.get('image').get('url', '') if item.get('image') else '', "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])], "location": location, "restro_ref_id": url } items_with_modifiers.append(item_with_modifiers) return items_with_modifiers def save_data_to_json(data, filename='restaurants.json'): with open(filename, 'w') as f: json.dump(data, f, indent=4) def save_data_to_db(data, location): # Dictionary to hold categories and associated items category_dict = {} for item_data in data: category_name = item_data['category_name'] name = item_data['name'] description = item_data.get('description', '') image_url = item_data.get('img_url', '') price = item_data['price'] option_groups_data = item_data.get('modifier_groups', []) restro_ref_id_url = item_data['restro_ref_id'] restro_link = DeliverooLink.objects(href=restro_ref_id_url).first() if not restro_link: restro_link = DeliverooLink(href=restro_ref_id_url) restro_link.save() # Create or retrieve MenuOption instances option_groups = [] for group_data in option_groups_data: group_name = group_data['name'] min_quantity = group_data.get('minSelection', 0) max_quantity = group_data.get('maxSelection', 1) options_data = group_data.get('modifierOptions', []) option_group = MenuOptionGroup.objects( group_name=group_name, min_quantity=min_quantity, max_quantity=max_quantity, location=location, restro_ref_id=restro_link).first() if not option_group: option_group = MenuOptionGroup( group_name=group_name, min_quantity=min_quantity, max_quantity=max_quantity, location=location, restro_ref_id=restro_link ) option_group.save() options = [] for option_data in options_data: option_name = option_data['name'] option_price = option_data['price']['formatted'] option = MenuOption.objects(name=option_name, price=option_price, location=location, restro_ref_id=restro_link).first() if not option: option = MenuOption(name=option_name, price=option_price, location=location, restro_ref_id=restro_link) option.save() if option not in option_group.options: option_group.options.append(option) option_group.save() if option_group not in option.option_groups: option.option_groups.append(option_group) option.save() options.append(option) option_groups.append(option_group) # Create or retrieve MenuItem instance menu_item = MenuItem.objects( category_name=category_name, name=name, location=location, restro_ref_id=restro_link).first() if not menu_item: menu_item = MenuItem( category_name=category_name, name=name, description=description, image_url=image_url, price=price, option_groups=option_groups, location=location, restro_ref_id=restro_link ) menu_item.save() else: # Ensure option groups are updated for existing menu items for option_group in option_groups: if option_group not in menu_item.option_groups: menu_item.option_groups.append(option_group) menu_item.save() # Store the menu item under its category if category_name not in category_dict: category_dict[category_name] = [] category_dict[category_name].append(menu_item) # Create or retrieve MenuCategory instances and associate items for category_name, items in category_dict.items(): menu_category = MenuCategory.objects(name=category_name, location=location, restro_ref_id=restro_link).first() if not menu_category: menu_category = MenuCategory(name=category_name, location=location, restro_ref_id=restro_link) menu_category.save() for menu_item in items: if menu_item not in menu_category.items: menu_category.items.append(menu_item) menu_category.save() print(f"Saved item '{name}' in category '{category_name}' for location '{location}'")