Update app.py

app.py

@@ -1,5 +1,5 @@
-from fastapi import FastAPI, HTTPException, status, BackgroundTasks  
-from fastapi.responses import StreamingResponse  
+from fastapi import FastAPI, HTTPException, status, BackgroundTasks, Request, Depends  
+from fastapi.responses import StreamingResponse, JSONResponse  
 import instaloader  
 import requests  
 import os  
@@ -9,6 +9,10 @@ import random
 from functools import wraps  
 import json  
 from datetime import datetime, timedelta  
+from typing import Dict, List, Optional  
+import hashlib  
+import asyncio  
+from starlette.concurrency import run_in_threadpool  
 
 # Configure logging  
 logging.basicConfig(  
@@ -26,13 +30,21 @@ INSTAGRAM_PASSWORD = os.getenv('INSTAGRAM_PASSWORD')
 # Session configuration  
 SESSION_DIR = "/tmp/instagram_sessions"  
 SESSION_FILE = os.path.join(SESSION_DIR, f"session-{INSTAGRAM_USERNAME}") if INSTAGRAM_USERNAME else None  
+CACHE_DIR = "/tmp/instagram_cache"  
 
-# Create session directory if not exists  
+# Create required directories  
 os.makedirs(SESSION_DIR, exist_ok=True)  
+os.makedirs(CACHE_DIR, exist_ok=True)  
 
 # Rate limiting state  
 LAST_REQUEST_TIME = {}  
 COOLDOWN_PERIODS = {}  
+RATE_LIMIT_LOCK = asyncio.Lock()  
+
+# Cache state  
+STORY_CACHE: Dict[str, Dict] = {}  
+CACHE_EXPIRY: Dict[str, datetime] = {}  
+CACHE_LOCK = asyncio.Lock()  
 
 USER_AGENTS = [  
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",  
@@ -42,11 +54,53 @@ USER_AGENTS = [
     "Mozilla/5.0 (iPad; CPU OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1"  
 ]  
 
+# Calculate cache key for a username  
+def get_cache_key(username: str) -> str:  
+    return hashlib.md5(username.lower().encode()).hexdigest()  
+
+# Check if we have a valid cache for a username  
+async def get_cached_stories(username: str) -> Optional[Dict]:  
+    cache_key = get_cache_key(username)  
+    
+    async with CACHE_LOCK:  
+        if cache_key in STORY_CACHE and cache_key in CACHE_EXPIRY:  
+            if datetime.now() < CACHE_EXPIRY[cache_key]:  
+                logger.info(f"Cache hit for {username}")  
+                return STORY_CACHE[cache_key]  
+            else:  
+                # Expired cache  
+                logger.info(f"Cache expired for {username}")  
+                STORY_CACHE.pop(cache_key, None)  
+                CACHE_EXPIRY.pop(cache_key, None)  
+    
+    return None  
+
+# Save stories to cache  
+async def cache_stories(username: str, stories_data: Dict, ttl_minutes: int = 15):  
+    cache_key = get_cache_key(username)  
+    
+    async with CACHE_LOCK:  
+        STORY_CACHE[cache_key] = stories_data  
+        CACHE_EXPIRY[cache_key] = datetime.now() + timedelta(minutes=ttl_minutes)  
+        
+    logger.info(f"Cached stories for {username} for {ttl_minutes} minutes")  
+    
+    # Save to disk for persistence  
+    cache_file = os.path.join(CACHE_DIR, f"{cache_key}.json")  
+    try:  
+        with open(cache_file, 'w') as f:  
+            json.dump({  
+                "data": stories_data,  
+                "expires": CACHE_EXPIRY[cache_key].isoformat()  
+            }, f)  
+    except Exception as e:  
+        logger.warning(f"Failed to save cache to disk: {str(e)}")  
+
 def get_instaloader(force_login=False) -> instaloader.Instaloader:  
     """Create and configure Instaloader instance with proper parameters"""  
     L = instaloader.Instaloader(  
         sleep=True,  
-        request_timeout=30,  # Reduced timeout  
+        request_timeout=30,  
         max_connection_attempts=3,  
         user_agent=random.choice(USER_AGENTS),  
         download_pictures=False,  
@@ -71,14 +125,8 @@ def get_instaloader(force_login=False) -> instaloader.Instaloader:
             try:  
                 L.load_session_from_file(INSTAGRAM_USERNAME, SESSION_FILE)  
                 logger.info("Session loaded successfully")  
-                # Test session without using is_verified  
-                try:  
-                    test_profile = instaloader.Profile.from_username(L.context, INSTAGRAM_USERNAME)  
-                    logger.info(f"Session test: Profile ID {test_profile.userid} verified")  
-                    return L  
-                except Exception as e:  
-                    logger.warning(f"Session test failed: {str(e)}")  
-                    raise Exception("Invalid session")  
+                # Test session with a lightweight call if possible  
+                return L  
             except Exception as e:  
                 logger.warning(f"Session load failed: {str(e)}, performing fresh login")  
                 
@@ -131,39 +179,46 @@ def get_instaloader(force_login=False) -> instaloader.Instaloader:
             detail="Instagram login service unavailable"  
         )  
 
-def is_rate_limited(username: str) -> bool:  
+async def is_rate_limited(username: str) -> bool:  
     """Check if a given username is currently rate limited"""  
-    # If in cooldown period, check if it's expired  
-    if username in COOLDOWN_PERIODS:  
-        if datetime.now() < COOLDOWN_PERIODS[username]:  
-            remaining = (COOLDOWN_PERIODS[username] - datetime.now()).seconds  
-            logger.warning(f"User {username} in cooldown for {remaining} more seconds")  
-            return True  
-        else:  
-            # Cooldown expired  
-            del COOLDOWN_PERIODS[username]  
-    
-    # Check time since last request  
-    if username in LAST_REQUEST_TIME:  
-        elapsed = (datetime.now() - LAST_REQUEST_TIME[username]).seconds  
-        min_interval = 60  # Minimum 60 seconds between requests  
-        if elapsed < min_interval:  
-            logger.warning(f"Rate limiting {username}: {elapsed}s elapsed, need {min_interval}s")  
-            return True  
-    
-    # Update last request time  
-    LAST_REQUEST_TIME[username] = datetime.now()  
-    return False  
+    async with RATE_LIMIT_LOCK:  
+        # If in cooldown period, check if it's expired  
+        if username in COOLDOWN_PERIODS:  
+            if datetime.now() < COOLDOWN_PERIODS[username]:  
+                remaining = (COOLDOWN_PERIODS[username] - datetime.now()).seconds  
+                logger.warning(f"User {username} in cooldown for {remaining} more seconds")  
+                return True  
+            else:  
+                # Cooldown expired  
+                del COOLDOWN_PERIODS[username]  
+        
+        # Check time since last request  
+        if username in LAST_REQUEST_TIME:  
+            elapsed = (datetime.now() - LAST_REQUEST_TIME[username]).seconds  
+            min_interval = 60  # Minimum 60 seconds between requests  
+            if elapsed < min_interval:  
+                logger.warning(f"Rate limiting {username}: {elapsed}s elapsed, need {min_interval}s")  
+                return True  
+        
+        # Update last request time  
+        LAST_REQUEST_TIME[username] = datetime.now()  
+        return False  
 
-def handle_rate_limit_error(username: str):  
+async def handle_rate_limit_error(username: str, retry_after: Optional[int] = None):  
     """Handle rate limit by setting cooldown period"""  
-    # Set cooldown period (increases with repeated issues)  
-    cooldown_minutes = 5  
-    if username in COOLDOWN_PERIODS:  
-        cooldown_minutes = min(cooldown_minutes * 2, 30)  # Exponential backoff up to 30 minutes  
-    
-    COOLDOWN_PERIODS[username] = datetime.now() + timedelta(minutes=cooldown_minutes)  
-    logger.warning(f"Setting cooldown for {username} for {cooldown_minutes} minutes")  
+    async with RATE_LIMIT_LOCK:  
+        # Set cooldown period based on retry_after or default logic  
+        if retry_after:  
+            # Use server-provided retry duration if available  
+            cooldown_minutes = max(retry_after / 60, 1)  # At least 1 minute  
+        else:  
+            # Default exponential backoff  
+            cooldown_minutes = 5  
+            if username in COOLDOWN_PERIODS:  
+                cooldown_minutes = min(cooldown_minutes * 2, 30)  # Up to 30 minutes  
+        
+        COOLDOWN_PERIODS[username] = datetime.now() + timedelta(minutes=cooldown_minutes)  
+        logger.warning(f"Setting cooldown for {username} for {cooldown_minutes} minutes")  
 
 def handle_instagram_errors(func):  
     @wraps(func)  
@@ -175,21 +230,66 @@ def handle_instagram_errors(func):
             
         try:  
             # Check for rate limiting  
-            if request_username and is_rate_limited(request_username):  
-                raise HTTPException(  
-                    status_code=status.HTTP_429_TOO_MANY_REQUESTS,  
-                    detail="Rate limit exceeded. Please try again later."  
-                )  
+            if request_username and await is_rate_limited(request_username):  
+                cached_response = await get_cached_stories(request_username)  
+                if cached_response:  
+                    # Return cached response with cache note  
+                    logger.info(f"Returning cached response for rate-limited user {request_username}")  
+                    return {**cached_response, "from_cache": True}  
+                else:  
+                    # No cache available  
+                    raise HTTPException(  
+                        status_code=status.HTTP_429_TOO_MANY_REQUESTS,  
+                        detail="Rate limit exceeded. Please try again later."  
+                    )  
                 
-            return await func(*args, **kwargs)  
+            response = await func(*args, **kwargs)  
+            
+            # Cache successful responses if they contain story data  
+            if request_username and isinstance(response, dict) and "data" in response:  
+                await cache_stories(request_username, response)  
+                
+            return response  
             
         except instaloader.exceptions.ConnectionException as e:  
             error_message = str(e)  
             logger.error("Connection error: %s", error_message)  
             
-            if "401 Unauthorized" in error_message and "Please wait a few minutes" in error_message:  
+            # Check for rate limit related messages  
+            retry_after = None  
+            if "429 Too Many Requests" in error_message:  
+                # Extract retry timer from error message if possible  
+                try:  
+                    retry_text = error_message.split("retried in")[1].split("minutes")[0].strip()  
+                    retry_after = int(retry_text) * 60  # Convert to seconds  
+                except (IndexError, ValueError):  
+                    retry_after = 1800  # Default 30 minutes  
+                
+                if request_username:  
+                    await handle_rate_limit_error(request_username, retry_after)  
+                
+                # Try to serve from cache if available  
                 if request_username:  
-                    handle_rate_limit_error(request_username)  
+                    cached_response = await get_cached_stories(request_username)  
+                    if cached_response:  
+                        logger.info(f"Returning cached response for rate-limited request")  
+                        return {**cached_response, "from_cache": True}  
+                
+                raise HTTPException(  
+                    status_code=status.HTTP_429_TOO_MANY_REQUESTS,  
+                    detail=f"Instagram rate limit exceeded. Please try again in {retry_after//60} minutes."  
+                )  
+            elif "401 Unauthorized" in error_message and "Please wait a few minutes" in error_message:  
+                if request_username:  
+                    await handle_rate_limit_error(request_username)  
+                    
+                # Try to serve from cache if available  
+                if request_username:  
+                    cached_response = await get_cached_stories(request_username)  
+                    if cached_response:  
+                        logger.info(f"Returning cached response for rate-limited request")  
+                        return {**cached_response, "from_cache": True}  
+                        
                 raise HTTPException(  
                     status_code=status.HTTP_429_TOO_MANY_REQUESTS,  
                     detail="Instagram rate limit exceeded. Please try again later."  
@@ -208,7 +308,15 @@ def handle_instagram_errors(func):
         except instaloader.exceptions.QueryReturnedBadRequestException as e:  
             logger.error("API error 400: %s", str(e))  
             if request_username:  
-                handle_rate_limit_error(request_username)  
+                await handle_rate_limit_error(request_username)  
+                
+            # Try to serve from cache if available  
+            if request_username:  
+                cached_response = await get_cached_stories(request_username)  
+                if cached_response:  
+                    logger.info(f"Returning cached response for rate-limited request")  
+                    return {**cached_response, "from_cache": True}  
+                    
             raise HTTPException(  
                 status_code=status.HTTP_429_TOO_MANY_REQUESTS,  
                 detail="Instagram rate limit exceeded"  
@@ -228,107 +336,123 @@ def handle_instagram_errors(func):
 
 @app.get("/stories/{username}")  
 @handle_instagram_errors  
-async def get_stories(username: str, background_tasks: BackgroundTasks):  
+async def get_stories(username: str, background_tasks: BackgroundTasks, cached: bool = False):  
     """Retrieve stories with enhanced anti-detection measures"""  
     logger.info(f"Processing request for @{username}")  
     
+    # Check for cache first if requested  
+    if cached:  
+        cached_response = await get_cached_stories(username)  
+        if cached_response:  
+            return {**cached_response, "from_cache": True}  
+    
     try:  
-        # Get loader with session  
-        L = get_instaloader()  
-        logger.info("Instaloader instance configured")  
-        
-        # Randomized delay before profile request (more natural)  
-        delay = random.uniform(1.5, 3.0)  
-        logger.debug(f"Applying initial delay of {delay:.2f}s")  
-        time.sleep(delay)  
-        
-        # Profile lookup with retry  
-        profile = None  
-        for attempt in range(3):  
-            try:  
-                logger.info(f"Fetching profile (attempt {attempt+1}/3)")  
-                profile = instaloader.Profile.from_username(L.context, username)  
-                break  
-            except instaloader.exceptions.QueryReturnedBadRequestException:  
-                wait_time = (attempt + 1) * random.uniform(3.0, 5.0)  
-                logger.warning(f"Rate limited, waiting {wait_time:.2f}s")  
-                time.sleep(wait_time)  
-            except instaloader.exceptions.ConnectionException as e:  
-                if "401 Unauthorized" in str(e) and "Please wait a few minutes" in str(e):  
-                    # Try with a fresh login if session might be expired  
-                    if attempt < 2:  # Only try this once  
-                        logger.warning("Session may be expired, trying with fresh login")  
-                        L = get_instaloader(force_login=True)  
-                        time.sleep(random.uniform(4.0, 6.0))  
-                        continue  
-                raise  
-        
-        if profile is None:  
-            raise HTTPException(  
-                status.HTTP_429_TOO_MANY_REQUESTS,  
-                "Too many attempts to access Instagram"  
-            )  
+        # Run the Instagram operations in a thread pool to not block the event loop  
+        async def fetch_stories():  
+            # Get loader with session  
+            L = get_instaloader()  
+            logger.info("Instaloader instance configured")  
+            
+            # Randomized delay before profile request (more natural)  
+            delay = random.uniform(1.5, 3.0)  
+            logger.debug(f"Applying initial delay of {delay:.2f}s")  
+            time.sleep(delay)  
+            
+            # Profile lookup with retry  
+            profile = None  
+            for attempt in range(3):  
+                try:  
+                    logger.info(f"Fetching profile (attempt {attempt+1}/3)")  
+                    profile = instaloader.Profile.from_username(L.context, username)  
+                    break  
+                except instaloader.exceptions.QueryReturnedBadRequestException:  
+                    wait_time = (attempt + 1) * random.uniform(3.0, 5.0)  
+                    logger.warning(f"Rate limited, waiting {wait_time:.2f}s")  
+                    time.sleep(wait_time)  
+                except instaloader.exceptions.ConnectionException as e:  
+                    if "401 Unauthorized" in str(e) and "Please wait a few minutes" in str(e):  
+                        # Try with a fresh login if session might be expired  
+                        if attempt < 2:  # Only try this once  
+                            logger.warning("Session may be expired, trying with fresh login")  
+                            L = get_instaloader(force_login=True)  
+                            time.sleep(random.uniform(4.0, 6.0))  
+                            continue  
+                    raise  
+            
+            if profile is None:  
+                raise HTTPException(  
+                    status.HTTP_429_TOO_MANY_REQUESTS,  
+                    "Too many attempts to access Instagram"  
+                )  
 
-        logger.info(f"Access check for @{username}")  
-        if not profile.has_viewable_story:  
-            logger.warning("No viewable story")  
-            raise HTTPException(  
-                status.HTTP_404_NOT_FOUND,  
-                "No accessible stories for this profile"  
-            )  
+            logger.info(f"Access check for @{username}")  
+            if not profile.has_viewable_story:  
+                logger.warning("No viewable story")  
+                raise HTTPException(  
+                    status.HTTP_404_NOT_FOUND,  
+                    "No accessible stories for this profile"  
+                )  
 
-        # Additional delay before story fetch (variable to look more natural)  
-        time.sleep(random.uniform(1.0, 2.5))  
-        
-        logger.info("Fetching stories")  
-        stories = []  
-        try:  
-            for story in L.get_stories(userids=[profile.userid]):  
-                for item in story.get_items():  
-                    # Create a story dict with safe attribute access  
-                    story_data = {  
-                        "id": str(item.mediaid),  
-                        "url": item.url,  
-                        "type": "video" if item.is_video else "image",  
-                        "timestamp": item.date_utc.isoformat(),  
-                    }  
-                    
-                    # Add any available metadata safely  
-                    if hasattr(item, 'owner_username'):  
-                        story_data["username"] = item.owner_username  
+            # Additional delay before story fetch  
+            time.sleep(random.uniform(1.0, 2.5))  
+            
+            logger.info("Fetching stories")  
+            stories = []  
+            try:  
+                for story in L.get_stories(userids=[profile.userid]):  
+                    for item in story.get_items():  
+                        # Create a story dict with safe attribute access  
+                        story_data = {  
+                            "id": str(item.mediaid),  
+                            "url": item.url,  
+                            "type": "video" if item.is_video else "image",  
+                            "timestamp": item.date_utc.isoformat(),  
+                        }  
                         
-                    # Only try to add view_count if it's a video  
-                    if item.is_video:  
-                        try:  
-                            if hasattr(item, 'view_count'):  
-                                story_data["views"] = item.view_count  
-                        except AttributeError:  
-                            pass  
-                    
-                    stories.append(story_data)  
-                    
-        except instaloader.exceptions.QueryReturnedNotFoundException:  
-            logger.error("Stories not found")  
-            raise HTTPException(  
-                status.HTTP_404_NOT_FOUND,  
-                "Stories not available"  
-            )  
+                        # Add any available metadata safely  
+                        if hasattr(item, 'owner_username'):  
+                            story_data["username"] = item.owner_username  
+                            
+                        # Only try to add view_count if it's a video  
+                        if item.is_video:  
+                            try:  
+                                if hasattr(item, 'view_count'):  
+                                    story_data["views"] = item.view_count  
+                            except AttributeError:  
+                                pass  
+                        
+                        stories.append(story_data)  
+                        
+            except instaloader.exceptions.QueryReturnedNotFoundException:  
+                logger.error("Stories not found")  
+                raise HTTPException(  
+                    status.HTTP_404_NOT_FOUND,  
+                    "Stories not available"  
+                )  
 
-        if not stories:  
-            logger.info("No active stories found")  
-            raise HTTPException(  
-                status.HTTP_404_NOT_FOUND,  
-                "No active stories available"  
-            )  
+            if not stories:  
+                logger.info("No active stories found")  
+                raise HTTPException(  
+                    status.HTTP_404_NOT_FOUND,  
+                    "No active stories available"  
+                )  
 
-        # Queue session save in background to not delay response  
-        background_tasks.add_task(lambda: L.save_session_to_file(SESSION_FILE) if SESSION_FILE else None)  
-        
-        # Final randomized delay before response (looks more natural)  
-        time.sleep(random.uniform(0.3, 0.7))  
+            # Queue session save in background  
+            background_tasks.add_task(lambda: L.save_session_to_file(SESSION_FILE) if SESSION_FILE else None)  
+            
+            # Final randomized delay  
+            time.sleep(random.uniform(0.3, 0.7))  
+            
+            logger.info(f"Returning {len(stories)} stories")  
+            return {  
+                "data": stories,   
+                "count": len(stories),   
+                "username": username,  
+                "fetched_at": datetime.now().isoformat()  
+            }  
         
-        logger.info(f"Returning {len(stories)} stories")  
-        return {"data": stories, "count": len(stories), "username": username}  
+        # Execute in thread pool to not block the event loop  
+        return await run_in_threadpool(fetch_stories)  
 
     except Exception as e:  
         logger.error(f"Critical failure: {str(e)}")  
@@ -405,4 +529,43 @@ async def download_media(url: str):
 # Add a health check endpoint  
 @app.get("/health")  
 async def health_check():  
-    return {"status": "ok", "timestamp": datetime.now().isoformat()}
+    return {"status": "ok", "timestamp": datetime.now().isoformat()}  
+
+# Add middleware to handle rate limit headers  
+@app.middleware("http")  
+async def add_rate_limit_headers(request: Request, call_next):  
+    response = await call_next(request)  
+    
+    # Add custom headers for rate limiting info  
+    username = request.path_params.get("username", None)  
+    if username and username in COOLDOWN_PERIODS:  
+        remaining = max(0, int((COOLDOWN_PERIODS[username] - datetime.now()).total_seconds()))  
+        response.headers["X-RateLimit-Reset"] = str(remaining)  
+        response.headers["X-RateLimit-Remaining"] = "0" if remaining > 0 else "1"  
+        
+    return response  
+
+# Startup event to load cache from disk  
+@app.on_event("startup")  
+async def load_cache_from_disk():  
+    try:  
+        for filename in os.listdir(CACHE_DIR):  
+            if filename.endswith('.json'):  
+                file_path = os.path.join(CACHE_DIR, filename)  
+                try:  
+                    with open(file_path, 'r') as f:  
+                        cache_data = json.load(f)  
+                        
+                    if "data" in cache_data and "expires" in cache_data:  
+                        expire_time = datetime.fromisoformat(cache_data["expires"])  
+                        if expire_time > datetime.now():  
+                            cache_key = filename[:-5]  # Remove .json  
+                            async with CACHE_LOCK:  
+                                STORY_CACHE[cache_key] = cache_data["data"]  
+                                CACHE_EXPIRY[cache_key] = expire_time  
+                except Exception as e:  
+                    logger.warning(f"Failed to load cache file {filename}: {str(e)}")  
+        
+        logger.info(f"Loaded {len(STORY_CACHE)} items from cache")  
+    except Exception as e:  
+        logger.error(f"Error loading cache: {str(e)}")