main.py

from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import Optional, Any, Dict, List
import aiohttp
import os
from datetime import datetime, timezone
import json
import re
from google.oauth2.service_account import Credentials as ServiceAccountCredentials
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from dotenv import load_dotenv
import asyncio
import logging

# --- Logging Setup ---
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

app = FastAPI()

# --- Configuration ---
load_dotenv()

# CORS
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"], # Consider restricting in production
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Google Sheets Config
# Spreadsheet containing Scammer and DWC info
SCAMMER_DWC_SPREADSHEET_ID = os.getenv('SCAMMER_DWC_SPREADSHEET_ID', '1sgkhBNGw_r6tBIxvdeXaI0bVmWBeACN4jiw_oDEeXLw')
# Spreadsheet containing Value lists and Dupe list
VALUES_DUPE_SPREADSHEET_ID = os.getenv('VALUES_DUPE_SPREADSHEET_ID', '1Toe07o3P517q8sm9Qb1e5xyFWCuwgskj71IKJwJNfNU')

SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly']

# Sheet Names and Ranges within SCAMMER_DWC_SPREADSHEET_ID
USER_SCAMMER_SHEET = "User Scammer Files"
USER_SCAMMER_RANGE = "B6:G"
SERVER_SCAMMER_SHEET = "Server Scammer Files"
SERVER_SCAMMER_RANGE = "B6:F"
DWC_SHEET = "DWC Servers / Users"
DWC_RANGE = "B6:G"

# Sheet Names and Ranges within VALUES_DUPE_SPREADSHEET_ID
DUPE_LIST_SHEET = "Dupe List"
DUPE_LIST_RANGE = "B2:B"
# Value Categories (Sheet Names)
CATEGORIES = [
    "Vehicles", "Textures", "Colours", "Spoilers",
    "Rims", "Furnitures", "Gun Skins", "Hyperchromes"
]
VALUES_RANGE = 'B6:P' # Range within each category sheet

# Cache Update Interval
CACHE_UPDATE_INTERVAL_SECONDS = int(os.getenv('CACHE_UPDATE_INTERVAL_SECONDS', 60 * 5)) # 5 minutes default

# Webhook URLs
SCAMMER_WEBHOOK_URL = os.getenv("SCAMMER_WEBHOOK_URL")
VALUE_WEBHOOK_URL = os.getenv("VALUE_WEBHOOK_URL")


# --- Global Cache ---
cache = {
    "values": {},             # Dict mapping category name to list of items
    "value_changes": {},      # Dict mapping category name to list of recent changes (for API endpoint)
    "user_scammers": [],
    "server_scammers": [],
    "dwc": [],
    "dupes": [],              # List of duped usernames
    "last_updated": None,     # Timestamp of the last successful/partial update
    "is_ready": False,        # Is the cache populated at least once?
    "service_available": True # Is the Google Sheets service reachable?
}
# --- Google Sheets Initialization ---
sheets_service = None # Initialize as None

def quote_sheet_name(name: str) -> str:
    """Adds single quotes around a sheet name if it needs them."""
    if not name:
        return "''"
    # Simple check: if it contains spaces or non-alphanumeric chars (excluding _)
    if not re.match(r"^[a-zA-Z0-9_]+$", name):
        # Escape existing single quotes within the name
        escaped_name = name.replace("'", "''")
        return f"'{escaped_name}'"
    return name

def init_google_sheets(scopes=SCOPES):
    """Initialize Google Sheets credentials from environment variable or file"""
    global sheets_service, cache
    try:
        creds_json_str = os.getenv('CREDENTIALS_JSON')
        if creds_json_str:
            logger.info("Attempting to load credentials from CREDENTIALS_JSON environment variable.")
            creds_json = json.loads(creds_json_str)
            creds = ServiceAccountCredentials.from_service_account_info(
                creds_json,
                scopes=scopes
            )
            sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False) # Disable discovery cache
            logger.info("Google Sheets service initialized successfully from ENV VAR.")
            cache["service_available"] = True
            return sheets_service
        else:
            logger.info("CREDENTIALS_JSON environment variable not found. Falling back to file.")
            raise ValueError("CREDENTIALS_JSON not set") # Trigger fallback explicitly

    except Exception as e:
        logger.warning(f"Error initializing Google Sheets from ENV VAR: {e}. Trying file...")
        try:
            # Fallback to loading credentials from file 'credentials.json'
            creds_file = 'credentials.json'
            if os.path.exists(creds_file):
                 logger.info(f"Attempting to load credentials from file '{creds_file}'")
                 creds = ServiceAccountCredentials.from_service_account_file(
                    creds_file,
                    scopes=scopes
                 )
                 sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False)
                 logger.info("Google Sheets service initialized successfully from file.")
                 cache["service_available"] = True
                 return sheets_service
            else:
                 logger.error(f"Credentials file '{creds_file}' not found.")
                 raise FileNotFoundError(f"'{creds_file}' not found")

        except Exception as file_e:
            logger.error(f"Error loading credentials from file: {file_e}")
            logger.critical("Google Sheets service could not be initialized. API will be limited.")
            cache["service_available"] = False
            sheets_service = None
            return None

# Initialize on module load
init_google_sheets()


# --- Helper Functions (Data Extraction & Formatting) ---

def extract_drive_id(url):
    if not url or not isinstance(url, str): return None
    match = re.search(r'https://drive\.google\.com/file/d/([^/]+)', url)
    return match.group(1) if match else None

def convert_to_thumbnail_url(drive_url):
    drive_id = extract_drive_id(drive_url)
    return f"https://drive.google.com/thumbnail?id={drive_id}&sz=w1000" if drive_id else drive_url

def extract_image_url(formula, drive_url=None):
    # Priority to explicit drive_url if provided
    if drive_url and isinstance(drive_url, str) and 'drive.google.com' in drive_url:
        return convert_to_thumbnail_url(drive_url)
    if not formula or not isinstance(formula, str): return ''
    # Handle direct URLs
    if formula.startswith('http://') or formula.startswith('https://'):
        return formula
    # Handle =IMAGE("...") formula
    if formula.startswith('=IMAGE('):
        match = re.search(r'=IMAGE\("([^"]+)"', formula)
        if match: return match.group(1)
    return '' # Return empty string if no valid URL found

def format_currency(value: Any) -> Optional[str]:
    if value is None or str(value).strip() == '': return 'N/A'
    try:
        num_str = str(value).replace('$', '').replace(',', '').strip()
        if not num_str or num_str.lower() == 'n/a': return 'N/A'
        num = float(num_str)
        return f"${num:,.0f}"
    except (ValueError, TypeError):
        if isinstance(value, str) and not re.match(r'^-?[\d,.$]+\$?$', value.strip()):
             return value.strip() # Return original text if non-numeric-like
        return 'N/A'

def parse_cached_currency(value_str: Optional[str]) -> Optional[float]:
    if value_str is None or value_str is None or str(value_str).strip().lower() == 'n/a':
        return None
    try:
        num_str = str(value_str).replace('$', '').replace(',', '').strip()
        return float(num_str)
    except (ValueError, TypeError):
        return None

def clean_string(value, default='N/A'):
    if value is None: return default
    cleaned = str(value).strip()
    return cleaned if cleaned else default

def clean_string_optional(value):
    if value is None: return None
    cleaned = str(value).strip()
    return cleaned if cleaned and cleaned != '-' else None

def parse_alt_accounts(value):
    if value is None: return []
    raw_string = str(value).strip()
    if not raw_string or raw_string == '-': return []
    return [acc.strip() for acc in raw_string.split(',') if acc.strip()]


# --- Roblox API Helpers (Unchanged) ---
async def get_roblox_user_id(session: aiohttp.ClientSession, username: str):
    if not username: return None
    url = "https://users.roblox.com/v1/usernames/users"
    payload = {"usernames": [username], "excludeBannedUsers": False}
    try:
        async with session.post(url, json=payload) as response:
            if response.status == 200:
                data = await response.json()
                if data and data.get("data") and len(data["data"]) > 0:
                    return data["data"][0].get("id")
            else:
                 logger.warning(f"Roblox API returned status {response.status} for username '{username}'")
            return None
    except asyncio.TimeoutError:
        logger.warning(f"Timeout fetching Roblox User ID for {username}")
        return None
    except aiohttp.ClientError as e:
        logger.warning(f"Network error fetching Roblox User ID for {username}: {e}")
        return None
    except Exception as e:
        logger.error(f"Unexpected exception fetching Roblox User ID for {username}: {e}", exc_info=True)
        return None

async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
    if not user_id: return None
    url = f"https://thumbnails.roblox.com/v1/users/avatar-headshot?userIds={user_id}&size=150x150&format=Png&isCircular=false"
    try:
        async with session.get(url) as response:
            if response.status == 200:
                data = await response.json()
                if data and data.get("data") and len(data["data"]) > 0:
                    return data["data"][0].get("imageUrl")
            else:
                 logger.warning(f"Roblox Thumbnail API returned status {response.status} for user ID {user_id}")
            return None
    except asyncio.TimeoutError:
        logger.warning(f"Timeout fetching Roblox avatar for User ID {user_id}")
        return None
    except aiohttp.ClientError as e:
        logger.warning(f"Network error fetching Roblox avatar for User ID {user_id}: {e}")
        return None
    except Exception as e:
        logger.error(f"Unexpected exception fetching Roblox avatar for User ID {user_id}: {e}", exc_info=True)
        return None


# --- Data Processing Functions ---
# These functions take raw rows from the sheet and process them.

def process_sheet_data(values): # For Value Categories
    if not values: return []
    processed_data = []
    for row in values: # Expected range like B6:P
        if not row or not any(str(cell).strip() for cell in row if cell is not None): continue
        # Skip header-like rows (e.g., "LEVEL 1 | HYPERCHROMES" in column F/index 4)
        if len(row) > 4 and isinstance(row[4], str) and re.search(r'LEVEL \d+ \|', row[4]):
            continue

        # Indices based on B6:P (0-indexed from B)
        icon_formula = row[0] if len(row) > 0 else ''
        name = row[2] if len(row) > 2 else 'N/A'
        value_raw = row[4] if len(row) > 4 else 'N/A'
        duped_value_raw = row[6] if len(row) > 6 else 'N/A'
        market_value_raw = row[8] if len(row) > 8 else 'N/A'
        demand = row[10] if len(row) > 10 else 'N/A'
        notes = row[12] if len(row) > 12 else ''
        drive_url = row[14] if len(row) > 14 else None # Column P

        cleaned_name = clean_string(name)
        if cleaned_name == 'N/A': # Skip rows without a valid name
            continue

        processed_item = {
            'icon': extract_image_url(icon_formula, drive_url),
            'name': cleaned_name,
            'value': format_currency(value_raw),
            'dupedValue': format_currency(duped_value_raw),
            'marketValue': format_currency(market_value_raw),
            'demand': clean_string(demand, 'N/A'),
            'notes': clean_string(notes, '')
        }
        processed_data.append(processed_item)
    return processed_data

def process_user_scammer_data(values): # For User Scammer Sheet
    if not values: return []
    processed_data = []
    for row in values: # Expected range like B6:G
        if not row or len(row) < 2: continue
        # Indices based on B6:G (0-indexed from B)
        discord_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
        roblox_username = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
        if not discord_id and not roblox_username: continue
        processed_item = {
            'discord_id': discord_id,
            'roblox_username': roblox_username,
            'scam_type': clean_string(row[2]) if len(row) > 2 else 'N/A', # Col D
            'explanation': clean_string(row[3]) if len(row) > 3 else 'N/A', # Col E
            'evidence_link': clean_string_optional(row[4]) if len(row) > 4 else None, # Col F
            'alt_accounts': parse_alt_accounts(row[5]) if len(row) > 5 else [], # Col G
            'roblox_avatar_url': None # Will be filled later
        }
        processed_data.append(processed_item)
    return processed_data

def process_server_scammer_data(values): # For Server Scammer Sheet
    if not values: return []
    processed_data = []
    for row in values: # Expected range like B6:F
        if not row or len(row) < 2: continue
        # Indices based on B6:F (0-indexed from B)
        server_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
        server_name = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
        if not server_id and not server_name: continue
        processed_item = {
            'server_id': server_id,
            'server_name': server_name,
            'scam_type': clean_string(row[2]) if len(row) > 2 else 'N/A', # Col D
            'explanation': clean_string(row[3]) if len(row) > 3 else 'N/A', # Col E
            'evidence_link': clean_string_optional(row[4]) if len(row) > 4 else None # Col F
        }
        processed_data.append(processed_item)
    return processed_data

def process_dwc_data(values): # For DWC Sheet
    if not values: return []
    processed_data = []
    for row in values: # Expected range like B6:G
        if not row or len(row) < 3: continue
        # Indices based on B6:G (0-indexed from B)
        user_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
        server_id = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
        roblox_user = clean_string_optional(row[2]) if len(row) > 2 else None # Col D
        if not user_id and not server_id and not roblox_user: continue
        processed_item = {
            'status': 'DWC',
            'discord_user_id': user_id,
            'discord_server_id': server_id,
            'roblox_username': roblox_user,
            'explanation': clean_string(row[3]) if len(row) > 3 else 'N/A', # Col E
            'evidence_link': clean_string_optional(row[4]) if len(row) > 4 else None, # Col F
            'alt_accounts': parse_alt_accounts(row[5]) if len(row) > 5 else [], # Col G
            'roblox_avatar_url': None # Will be filled later
        }
        processed_data.append(processed_item)
    return processed_data

def process_dupe_list_data(values): # For Dupe List Sheet
    if not values: return []
    # Expected range like B2:B
    return [row[0].strip().lower() for row in values if row and len(row)>0 and row[0] and isinstance(row[0], str) and row[0].strip()]


# --- Async Fetching Functions ---

async def fetch_batch_ranges_async(spreadsheet_id: str, ranges: List[str], value_render_option: str = 'FORMATTED_VALUE') -> List[Dict]:
    """Async wrapper to fetch multiple ranges using batchGet and return raw valueRanges."""
    global sheets_service
    if not sheets_service:
        logger.warning(f"Attempted batch fetch from {spreadsheet_id} but Sheets service is unavailable.")
        raise Exception("Google Sheets service not initialized")
    if not ranges:
        logger.warning(f"Batch fetch called with empty ranges for {spreadsheet_id}.")
        return []

    try:
        logger.info(f"Fetching batch ranges from {spreadsheet_id}: {ranges}")
        loop = asyncio.get_event_loop()
        result = await loop.run_in_executor(
            None,
            lambda: sheets_service.spreadsheets().values().batchGet(
                spreadsheetId=spreadsheet_id,
                ranges=ranges,
                valueRenderOption=value_render_option,
                majorDimension='ROWS'
            ).execute()
        )
        value_ranges = result.get('valueRanges', [])
        logger.info(f"Successfully fetched batch data for {len(value_ranges)} ranges from {spreadsheet_id}.")
        return value_ranges # Return the raw list of valueRange objects

    except HttpError as e:
        error_details = json.loads(e.content).get('error', {})
        status = error_details.get('status')
        message = error_details.get('message')
        logger.error(f"Google API HTTP Error during batch fetch for {spreadsheet_id}: Status={status}, Message={message}")
        # Handle potential API key/permission issues explicitly
        if status == 'PERMISSION_DENIED' or status == 'UNAUTHENTICATED':
            logger.critical(f"Authentication/Permission Error accessing {spreadsheet_id}. Please check credentials/API access.")
            cache["service_available"] = False # Mark service as down
            sheets_service = None # Reset service
        elif status == 'NOT_FOUND':
             logger.error(f"Spreadsheet or Range not found error for {spreadsheet_id}. Ranges: {ranges}. Check IDs and Sheet Names.")
        raise e # Re-raise after logging
    except Exception as e:
        logger.error(f"Error during batch fetching from {spreadsheet_id} for ranges {ranges}: {e}", exc_info=True)
        raise e

# --- Webhook Sending ---
async def send_webhook_notification(session: aiohttp.ClientSession, webhook_url: str, embed: Dict):
    """Sends a Discord webhook notification with the provided embed."""
    if not webhook_url:
        # logger.debug("Webhook URL not configured. Skipping notification.") # Optional: Log less verbosely
        return
    if not embed:
        logger.warning("Attempted to send webhook with empty embed.")
        return

    webhook_data = {"embeds": [embed]}
    try:
        async with session.post(webhook_url, json=webhook_data, timeout=aiohttp.ClientTimeout(total=10)) as response:
            if response.status not in [200, 204]:
                logger.warning(f"Failed to send webhook (Status: {response.status}): {await response.text()}")
            # else: # Optional: Log success, can be verbose
            #     logger.info(f"Webhook notification sent successfully to {webhook_url[:30]}...")
    except asyncio.TimeoutError:
        logger.warning(f"Timeout sending webhook to {webhook_url[:30]}...")
    except aiohttp.ClientError as e:
        logger.error(f"Network error sending webhook to {webhook_url[:30]}...: {e}")
    except Exception as e:
        logger.error(f"Unexpected error sending webhook: {e}", exc_info=True)

# --- Background Cache Update Task (Refactored for Batching & Webhooks) ---

async def update_cache_periodically():
    """Fetches data, processes, detects changes/new entries, sends webhooks, and updates cache."""
    global cache
    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: # Overall session timeout
        while True:
            if not cache["service_available"]:
                logger.info("Attempting to re-initialize Google Sheets service...")
                init_google_sheets()
                if not cache["service_available"]:
                    logger.warning("Google Sheets service still unavailable, skipping cache update cycle.")
                    await asyncio.sleep(CACHE_UPDATE_INTERVAL_SECONDS * 2) # Wait longer if service is down
                    continue
                else:
                    logger.info("Google Sheets service re-initialized. Proceeding with cache update.")

            logger.info("Starting cache update cycle...")
            start_time = datetime.now(timezone.utc)
            webhook_tasks = [] # Store webhook sending tasks

            # Prepare temporary storage for fetched data
            fetched_values_categories = {} # { "CategoryName": [items...] }
            new_cache_data = {
                "user_scammers": [],
                "server_scammers": [],
                "dwc": [],
                "dupes": [],
            }
            current_errors = {} # Track errors for specific fetches/sheets

            try:
                # --- Define Ranges and Processors ---
                scammer_dwc_ranges = [
                    f"{quote_sheet_name(USER_SCAMMER_SHEET)}!{USER_SCAMMER_RANGE}",
                    f"{quote_sheet_name(SERVER_SCAMMER_SHEET)}!{SERVER_SCAMMER_RANGE}",
                    f"{quote_sheet_name(DWC_SHEET)}!{DWC_RANGE}",
                ]
                scammer_dwc_processor_map = {
                    USER_SCAMMER_SHEET: process_user_scammer_data,
                    SERVER_SCAMMER_SHEET: process_server_scammer_data,
                    DWC_SHEET: process_dwc_data,
                }
                scammer_dwc_target_key_map = {
                    USER_SCAMMER_SHEET: "user_scammers",
                    SERVER_SCAMMER_SHEET: "server_scammers",
                    DWC_SHEET: "dwc",
                }

                values_dupes_ranges = [f"{quote_sheet_name(DUPE_LIST_SHEET)}!{DUPE_LIST_RANGE}"]
                values_dupes_ranges.extend([f"{quote_sheet_name(cat)}!{VALUES_RANGE}" for cat in CATEGORIES])

                # --- Define Fetch Tasks ---
                fetch_tasks = {
                    "scammer_dwc_batch": fetch_batch_ranges_async(
                        SCAMMER_DWC_SPREADSHEET_ID,
                        scammer_dwc_ranges,
                        value_render_option='FORMATTED_VALUE'
                    ),
                    "values_dupes_batch": fetch_batch_ranges_async(
                        VALUES_DUPE_SPREADSHEET_ID,
                        values_dupes_ranges,
                        value_render_option='FORMATTED_VALUE'
                    )
                }

                # --- Execute Tasks Concurrently ---
                results = await asyncio.gather(*fetch_tasks.values(), return_exceptions=True)
                task_keys = list(fetch_tasks.keys())

                # --- Process Results ---
                raw_scammer_dwc_results = None
                raw_values_dupes_results = None

                for i, result in enumerate(results):
                    key = task_keys[i]
                    if isinstance(result, Exception):
                        logger.error(f"Failed to fetch batch data for {key}: {result}")
                        current_errors[key] = str(result)
                    else:
                        if key == "scammer_dwc_batch":
                            raw_scammer_dwc_results = result
                        elif key == "values_dupes_batch":
                            raw_values_dupes_results = result

                # --- Process Scammer/DWC Results ---
                if raw_scammer_dwc_results is not None:
                    logger.info(f"Processing {len(raw_scammer_dwc_results)} valueRanges from Scammer/DWC sheet...")
                    for vr in raw_scammer_dwc_results:
                        range_str = vr.get('range', '')
                        match = re.match(r"^'?([^'!]+)'?!", range_str)
                        if not match:
                            logger.warning(f"Could not extract sheet name from range '{range_str}' in Scammer/DWC response.")
                            continue
                        sheet_name = match.group(1).replace("''", "'")

                        if sheet_name in scammer_dwc_processor_map:
                            processor = scammer_dwc_processor_map[sheet_name]
                            target_key = scammer_dwc_target_key_map[sheet_name]
                            values = vr.get('values', [])
                            try:
                                processed_data = processor(values)
                                new_cache_data[target_key] = processed_data # Store fetched data temporarily
                                logger.info(f"Processed {len(processed_data)} items for {sheet_name} -> {target_key}")
                            except Exception as e:
                                logger.error(f"Error processing data for {sheet_name} using {processor.__name__}: {e}", exc_info=True)
                                current_errors[f"process_{target_key}"] = str(e)
                        else:
                            logger.warning(f"No processor found for sheet name '{sheet_name}' derived from range '{range_str}' in Scammer/DWC sheet.")

                # --- Process Values/Dupes Results ---
                if raw_values_dupes_results is not None:
                    logger.info(f"Processing {len(raw_values_dupes_results)} valueRanges from Values/Dupes sheet...")
                    for vr in raw_values_dupes_results:
                        range_str = vr.get('range', '')
                        match = re.match(r"^'?([^'!]+)'?!", range_str)
                        if not match:
                            logger.warning(f"Could not extract sheet name from range '{range_str}' in Values/Dupes response.")
                            continue
                        sheet_name = match.group(1).replace("''", "'")

                        values = vr.get('values', [])
                        try:
                            if sheet_name == DUPE_LIST_SHEET:
                                processed_data = process_dupe_list_data(values)
                                new_cache_data["dupes"] = processed_data # Store fetched data temporarily
                                logger.info(f"Processed {len(processed_data)} items for {DUPE_LIST_SHEET} -> dupes")
                            elif sheet_name in CATEGORIES:
                                processed_data = process_sheet_data(values)
                                fetched_values_categories[sheet_name] = processed_data # Store fetched data temporarily
                                logger.info(f"Processed {len(processed_data)} items for Category: {sheet_name}")
                            else:
                                logger.warning(f"Unrecognized sheet name '{sheet_name}' derived from range '{range_str}' in Values/Dupes sheet.")
                        except Exception as e:
                            target_key = "dupes" if sheet_name == DUPE_LIST_SHEET else f"values_{sheet_name}"
                            logger.error(f"Error processing data for {sheet_name}: {e}", exc_info=True)
                            current_errors[f"process_{target_key}"] = str(e)

                # --- Fetch Roblox Avatars (for new data before comparison/webhook) ---
                logger.info("Fetching Roblox avatars for newly processed data...")
                avatar_tasks = []
                # Combine lists needing avatars from the *newly fetched* data
                entries_needing_avatars = new_cache_data.get("user_scammers", []) + new_cache_data.get("dwc", [])
                for entry in entries_needing_avatars:
                     if entry.get('roblox_username'):
                         # Pass the specific entry dict to the update function
                         avatar_tasks.append(fetch_avatar_for_entry_update(session, entry))
                if avatar_tasks:
                    await asyncio.gather(*avatar_tasks) # Exceptions logged within helper
                logger.info(f"Finished fetching avatars for {len(avatar_tasks)} potential new entries.")

                # --- Change Detection & Webhook Preparation (BEFORE Cache Update) ---
                current_time = datetime.now(timezone.utc)
                timestamp_iso = current_time.isoformat()

                # 1. Value Changes (Existing Logic + Webhook Prep)
                detected_value_changes_for_api = {} # For the /api/value-changes endpoint
                if "values" not in cache: cache["values"] = {} # Ensure exists for comparison
                if VALUE_WEBHOOK_URL and not any(k.startswith("process_values_") for k in current_errors) and "values_dupes_batch" not in current_errors:
                    logger.info("Detecting value changes for webhooks...")
                    fields_to_compare = ['value', 'dupedValue', 'marketValue']
                    for category, new_items in fetched_values_categories.items():
                        old_items_dict = {item['name']: item for item in cache["values"].get(category, [])}
                        category_changes_for_api = []

                        for new_item in new_items:
                            item_name = new_item.get('name')
                            if not item_name or item_name == 'N/A': continue

                            old_item = old_items_dict.get(item_name)
                            if old_item: # Check existing item for changes
                                for field in fields_to_compare:
                                    old_val_str = old_item.get(field, 'N/A')
                                    new_val_str = new_item.get(field, 'N/A')
                                    # Simple string comparison is sufficient here as they are formatted consistently
                                    if old_val_str != new_val_str:
                                        logger.info(f"Change detected in {category}: {item_name} - {field}: '{old_val_str}' -> '{new_val_str}'")
                                        change_info = {
                                            "item_name": item_name, "field": field,
                                            "old_value": old_val_str if old_val_str is not None else "N/A",
                                            "new_value": new_val_str if new_val_str is not None else "N/A",
                                            "timestamp": timestamp_iso
                                        }
                                        category_changes_for_api.append(change_info)

                                        # Prepare webhook embed
                                        embed = {
                                            "title": f"Value Update: {item_name} ({category})",
                                            "color": 3447003, # Blue
                                            "fields": [
                                                {"name": "Field Changed", "value": field, "inline": True},
                                                {"name": "Old Value", "value": f"`{change_info['old_value']}`", "inline": True},
                                                {"name": "New Value", "value": f"`{change_info['new_value']}`", "inline": True},
                                                {"name": "Item Notes", "value": new_item.get('notes', 'N/A')[:1020] or 'N/A', "inline": False}, # Limit notes length
                                            ],
                                            "timestamp": timestamp_iso
                                        }
                                        if new_item.get('icon'):
                                            embed["thumbnail"] = {"url": new_item['icon']}

                                        webhook_tasks.append(send_webhook_notification(session, VALUE_WEBHOOK_URL, embed))
                        if category_changes_for_api:
                            detected_value_changes_for_api[category] = category_changes_for_api
                    logger.info(f"Prepared {len(webhook_tasks)} value change webhooks.")
                elif not VALUE_WEBHOOK_URL:
                    logger.info("VALUE_WEBHOOK_URL not set, skipping value change webhook detection.")
                else:
                    logger.warning("Skipping value change webhook detection due to fetch/processing errors.")


                # 2. New Scammers / DWC (New Logic + Webhook Prep)
                if SCAMMER_WEBHOOK_URL and "scammer_dwc_batch" not in current_errors and not any(k.startswith("process_") and k in ["process_user_scammers", "process_server_scammers", "process_dwc"] for k in current_errors):
                    logger.info("Detecting new scammer/DWC entries for webhooks...")
                    initial_webhook_task_count = len(webhook_tasks)

                    # User Scammers
                    old_user_keys = set((item.get('discord_id'), item.get('roblox_username')) for item in cache.get("user_scammers", []))
                    for item in new_cache_data.get("user_scammers", []):
                        key = (item.get('discord_id'), item.get('roblox_username'))
                        if key not in old_user_keys:
                            logger.info(f"New User Scammer detected: Discord={item.get('discord_id')}, Roblox={item.get('roblox_username')}")
                            embed = {
                                "title": "🚨 New User Scammer Added",
                                "color": 15158332, # Red
                                "fields": [
                                    {"name": "Discord ID", "value": f"`{item.get('discord_id', 'N/A')}`", "inline": True},
                                    {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
                                ],
                                "timestamp": timestamp_iso
                            }
                            if item.get('evidence_link'):
                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
                            if item.get('alt_accounts'):
                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
                            if item.get('roblox_avatar_url'):
                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))

                    # Server Scammers
                    old_server_keys = set((item.get('server_id'), item.get('server_name')) for item in cache.get("server_scammers", []))
                    for item in new_cache_data.get("server_scammers", []):
                        key = (item.get('server_id'), item.get('server_name'))
                        if key not in old_server_keys:
                            logger.info(f"New Server Scammer detected: ID={item.get('server_id')}, Name={item.get('server_name')}")
                            embed = {
                                "title": "🚨 New Server Scammer Added",
                                "color": 15158332, # Red
                                "fields": [
                                    {"name": "Server ID", "value": f"`{item.get('server_id', 'N/A')}`", "inline": True},
                                    {"name": "Server Name", "value": f"`{item.get('server_name', 'N/A')}`", "inline": True},
                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
                                ],
                                "timestamp": timestamp_iso
                            }
                            if item.get('evidence_link'):
                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))

                    # DWC Entries
                    old_dwc_keys = set((item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username')) for item in cache.get("dwc", []))
                    for item in new_cache_data.get("dwc", []):
                        key = (item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username'))
                        if key not in old_dwc_keys:
                            logger.info(f"New DWC Entry detected: User={item.get('discord_user_id')}, Server={item.get('discord_server_id')}, Roblox={item.get('roblox_username')}")
                            embed = {
                                "title": "⚠️ New DWC Entry Added",
                                "color": 15105570, # Orange/Dark Yellow
                                "fields": [
                                    {"name": "Discord User ID", "value": f"`{item.get('discord_user_id', 'N/A')}`", "inline": True},
                                    {"name": "Discord Server ID", "value": f"`{item.get('discord_server_id', 'N/A')}`", "inline": True},
                                    {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
                                ],
                                "timestamp": timestamp_iso
                            }
                            if item.get('evidence_link'):
                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
                            if item.get('alt_accounts'):
                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
                            if item.get('roblox_avatar_url'):
                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))

                    logger.info(f"Prepared {len(webhook_tasks) - initial_webhook_task_count} new scammer/DWC webhooks.")
                elif not SCAMMER_WEBHOOK_URL:
                    logger.info("SCAMMER_WEBHOOK_URL not set, skipping new scammer webhook detection.")
                else:
                    logger.warning("Skipping new scammer webhook detection due to fetch/processing errors.")

                # --- Send Webhooks Concurrently ---
                if webhook_tasks:
                    logger.info(f"Sending {len(webhook_tasks)} webhook notifications...")
                    await asyncio.gather(*webhook_tasks)
                    logger.info("Finished sending webhook notifications.")
                else:
                    logger.info("No webhooks to send for this cycle.")


                # --- Final Cache Update ---
                update_occurred = False
                if not current_errors: # Perfect cycle
                    logger.info("Updating full cache (no errors during fetch or processing).")
                    cache["values"] = fetched_values_categories
                    cache["user_scammers"] = new_cache_data["user_scammers"]
                    cache["server_scammers"] = new_cache_data["server_scammers"]
                    cache["dwc"] = new_cache_data["dwc"]
                    cache["dupes"] = new_cache_data["dupes"]
                    cache["value_changes"] = detected_value_changes_for_api # Store the detected changes
                    cache["last_updated"] = current_time
                    cache["is_ready"] = True
                    cache["service_available"] = True # Mark as available on success
                    update_occurred = True
                    logger.info(f"Cache update cycle completed successfully.")
                else: # Errors occurred, attempt partial update
                    logger.warning(f"Cache update cycle completed with errors: {current_errors}. Attempting partial update.")
                    partial_update_details = []

                    # Update values only if the values/dupes batch succeeded AND processing succeeded
                    if "values_dupes_batch" not in current_errors and not any(k.startswith("process_values_") for k in current_errors):
                        # Check if fetched data is different from cache before updating
                        if cache.get("values") != fetched_values_categories:
                             cache["values"] = fetched_values_categories
                             cache["value_changes"] = detected_value_changes_for_api # Update changes along with values
                             partial_update_details.append("values")
                             update_occurred = True
                    else:
                         logger.warning("Skipping update for 'values' due to errors.")


                    # Update dupes only if the values/dupes batch succeeded AND processing succeeded
                    if "values_dupes_batch" not in current_errors and "process_dupes" not in current_errors:
                         if cache.get("dupes") != new_cache_data["dupes"]:
                            cache["dupes"] = new_cache_data["dupes"]
                            partial_update_details.append("dupes")
                            update_occurred = True
                    else:
                         logger.warning("Skipping update for 'dupes' due to errors.")

                    # Update scammer/DWC sections if their batch succeeded AND processing succeeded
                    if "scammer_dwc_batch" not in current_errors:
                        for key in ["user_scammers", "server_scammers", "dwc"]:
                            process_error_key = f"process_{key}"
                            if process_error_key not in current_errors:
                                 if cache.get(key) != new_cache_data[key]:
                                    cache[key] = new_cache_data[key]
                                    partial_update_details.append(key)
                                    update_occurred = True
                            else:
                                logger.warning(f"Skipping update for '{key}' due to processing error.")
                    else:
                         logger.warning("Skipping update for 'user_scammers', 'server_scammers', 'dwc' due to batch fetch error.")


                    if update_occurred:
                         cache["last_updated"] = current_time # Mark partial update time
                         cache["is_ready"] = True # Allow access even if partial
                         # Keep service_available as potentially false if there were fetch errors
                         logger.info(f"Partially updated cache sections: {', '.join(partial_update_details)}")
                    else:
                         logger.error(f"Cache update cycle failed, and no parts could be updated based on errors. Errors: {current_errors}")
                         # Keep cache["is_ready"] as it was. Don't update timestamp.

            except Exception as e:
                logger.exception(f"Critical error during cache update cycle: {e}")
                # If a critical error happens (e.g., network error during fetch), mark service potentially unavailable
                if isinstance(e, (aiohttp.ClientError, HttpError, asyncio.TimeoutError)):
                    logger.warning("Communication error detected, will re-check service availability next cycle.")
                    # service_available might have already been set to False by fetch_batch_ranges_async

            # --- Wait for the next cycle ---
            end_time = datetime.now(timezone.utc)
            duration = (end_time - start_time).total_seconds()
            wait_time = max(10, CACHE_UPDATE_INTERVAL_SECONDS - duration) # Ensure at least 10s wait
            logger.info(f"Cache update cycle duration: {duration:.2f}s. Waiting {wait_time:.2f}s for next cycle.")
            await asyncio.sleep(wait_time)


async def fetch_avatar_for_entry_update(session: aiohttp.ClientSession, entry: dict):
    """Fetches avatar and updates the provided entry dictionary IN PLACE."""
    roblox_username = entry.get('roblox_username')
    if not roblox_username: return

    new_avatar = None # Default to None
    try:
        user_id = await get_roblox_user_id(session, roblox_username)
        if user_id:
            new_avatar = await get_roblox_avatar_url(session, user_id)

    except Exception as e:
        # Log errors but don't stop the main update loop
        logger.warning(f"Failed to fetch avatar for {roblox_username}: {e}")
        # Keep new_avatar as None on error

    finally:
         # Update the entry dict directly (no need to check if changed, just set it)
         entry['roblox_avatar_url'] = new_avatar


# --- FastAPI Startup Event ---
@app.on_event("startup")
async def startup_event():
    """Starts the background cache update task."""
    if not cache["service_available"]:
         logger.warning("Google Sheets service not available at startup. Will attempt re-init in background task.")
    logger.info("Starting background cache update task...")
    # Check for webhook URLs at startup
    if not SCAMMER_WEBHOOK_URL:
        logger.warning("SCAMMER_WEBHOOK_URL environment variable not set. New scammer notifications disabled.")
    if not VALUE_WEBHOOK_URL:
        logger.warning("VALUE_WEBHOOK_URL environment variable not set. Value change notifications disabled.")
    asyncio.create_task(update_cache_periodically())


# --- API Endpoints ---

def check_cache_readiness():
    """Reusable check for API endpoints - Checks cache readiness"""
    if not cache["is_ready"]:
         # Be more specific if service is known to be down
         if not cache["service_available"]:
              raise HTTPException(status_code=503, detail="Service temporarily unavailable due to backend connection issues. Please try again later.")
         else:
              raise HTTPException(status_code=503, detail="Cache is initializing or data is currently unavailable. Please try again shortly.")
    # Optional: Add check for staleness?
    # if cache["last_updated"] and (datetime.now(timezone.utc) - cache["last_updated"]).total_seconds() > CACHE_UPDATE_INTERVAL_SECONDS * 3:
    #      raise HTTPException(status_code=503, detail="Data may be stale. Update in progress or backend issue.")

@app.get("/")
async def root():
    return {"message": "JVC API - Running"}

@app.get("/api/status")
async def get_status():
    """Returns the current status of the cache and service availability"""
    last_updated_iso = cache["last_updated"].isoformat() if cache["last_updated"] else None
    return {
        "cache_ready": cache["is_ready"],
        "sheets_service_available": cache["service_available"],
        "last_updated": last_updated_iso,
        "cached_items": {
            "value_categories": len(cache.get("values", {})),
            "user_scammers": len(cache.get("user_scammers", [])),
            "server_scammers": len(cache.get("server_scammers", [])),
            "dwc_entries": len(cache.get("dwc", [])),
            "duped_usernames": len(cache.get("dupes", [])),
        },
         "value_change_categories_in_last_cycle": len(cache.get("value_changes", {}))
    }

@app.get("/api/values")
async def get_values():
    """Get all values data from cache"""
    check_cache_readiness()
    return cache.get("values", {})

@app.get("/api/values/{category}")
async def get_category_values(category: str):
    """Get values data for a specific category from cache"""
    check_cache_readiness()
    # Case-insensitive matching for category name
    matched_category = next((c for c in cache.get("values", {}).keys() if c.lower() == category.lower()), None)
    if not matched_category:
         # Check if the category *exists* conceptually even if empty
         if category.lower() in [c.lower() for c in CATEGORIES]:
              return {category: []} # Return empty list if category is valid but has no items
         else:
            raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
    return {matched_category: cache.get("values", {}).get(matched_category, [])}


@app.get("/api/value-changes/{category}")
async def get_category_value_changes(category: str):
    """Get detected value changes for a specific category from the last cache update cycle."""
    check_cache_readiness()
    # Case-insensitive matching for category name
    matched_category = next((c for c in cache.get("value_changes", {}).keys() if c.lower() == category.lower()), None)
    if not matched_category:
         # Check if the category *exists* conceptually even if empty
         if category.lower() in [c.lower() for c in CATEGORIES]:
              return {category: []} # Return empty list if category is valid but had no changes
         else:
            raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
    return {matched_category: cache.get("value_changes", {}).get(matched_category, [])}

@app.get("/api/value-changes")
async def get_all_value_changes():
    """Get all detected value changes from the last cache update cycle."""
    check_cache_readiness()
    return cache.get("value_changes", {})

@app.get("/api/scammers")
async def get_scammers():
    """Get all scammer and DWC data (users, servers, dwc) from cache"""
    check_cache_readiness()
    return {
        "users": cache.get("user_scammers", []),
        "servers": cache.get("server_scammers", []),
        "dwc": cache.get("dwc", [])
    }

@app.get("/api/dupes")
async def get_dupes():
    """Get all duped usernames from cache"""
    check_cache_readiness()
    return {"usernames": cache.get("dupes", [])} # Return empty list if not ready or empty


class UsernameCheck(BaseModel):
    username: str

@app.post("/api/check")
async def check_username(data: UsernameCheck):
    """Check if a username is duped using cached data and send webhook"""
    check_cache_readiness() # Use the standard readiness check

    username_to_check = data.username.strip().lower()
    # Ensure dupes list is populated before checking
    dupes_list = cache.get("dupes", [])
    is_duped = username_to_check in dupes_list

    # Webhook notification for checks resulting in "Not Found" (runs in background)
    # Note: This uses the *value* webhook URL per original code, not the scammer one.
    # If this needs to go to a *different* webhook, adjust the env var name.
    if not is_duped:
        webhook_url = os.getenv("WEBHOOK_URL") # Keep original env var name for this specific check
        if webhook_url:
            # Use a new session for this one-off task or pass the main one if safe
            async def send_check_webhook():
                 try:
                    async with aiohttp.ClientSession() as session: # Create a short-lived session
                        embed = {
                                "title": "User Dupe Check - Not Found",
                                "description": f"Username `{data.username}` was checked against the dupe list but was **not** found.",
                                "color": 16776960, # Yellow
                                "timestamp": datetime.now(timezone.utc).isoformat()
                        }
                        await send_webhook_notification(session, webhook_url, embed)
                 except Exception as e:
                     logger.error(f"Error sending dupe check webhook: {e}") # Log errors from the task

            asyncio.create_task(send_check_webhook()) # Fire and forget
        else:
            logger.info("WEBHOOK_URL (for dupe checks) not configured. Skipping notification.")

    return {"username": data.username, "is_duped": is_duped}


@app.get("/health")
def health_check():
    """Provides a health status of the API and its cache."""
    now = datetime.now(timezone.utc)
    status_detail = {"status": "ok", "last_updated": None, "time_since_update_seconds": None}

    if cache["last_updated"]:
        status_detail["last_updated"] = cache["last_updated"].isoformat()
        time_since_update = (now - cache["last_updated"]).total_seconds()
        status_detail["time_since_update_seconds"] = round(time_since_update)
    else:
        status_detail["last_updated"] = None
        status_detail["time_since_update_seconds"] = None


    if not cache["is_ready"]:
        status_detail["status"] = "initializing"
        status_detail["reason"] = "Cache has not been populated yet."
        return status_detail

    if not cache["service_available"]:
        status_detail["status"] = "degraded"
        status_detail["reason"] = "Google Sheets service connection issue detected on last attempt."
        return status_detail

    # Check for staleness only if the cache is ready and service *was* available last check
    # Allow some grace period (e.g., 3 intervals)
    if cache["last_updated"] and time_since_update > CACHE_UPDATE_INTERVAL_SECONDS * 3:
         status_detail["status"] = "degraded"
         status_detail["reason"] = f"Cache potentially stale (last update > {CACHE_UPDATE_INTERVAL_SECONDS * 3} seconds ago)"
         return status_detail

    # If we reach here, status is 'ok'
    return status_detail