Add complete backend application with all dependencies

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+*$py.class
+.env
+.venv
+env/
+venv/
+ENV/
+*.log
+.DS_Store

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.9
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user and switch to it
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+ENV PYTHONPATH="/app:$PYTHONPATH"
+
+# Copy requirements first to leverage Docker cache
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy the application code
+COPY --chown=user app.py .
+COPY --chown=user apikeys.csv .
+COPY --chown=user MODELS.csv .
+COPY --chown=user utils.py .
+COPY --chown=user core ./core
+COPY --chown=user models ./models
+
+# Expose the port
+EXPOSE 7860
+
+# Start the application
+CMD ["python", "app.py"]

MODELS.csv

@@ -0,0 +1,14 @@
+GROQ MODELS,COHERE ,SambaNova,GEMINI
+deepseek-r1-distill-llama-70b,Command R,DeepSeek-R1-Distill-Llama-70B,gemini-2.0-flash
+llama-3.3-70b-versatile,Command R7B,Llama-3.1-Swallow-70B-Instruct-v0.3,gemini-2.0-flash-lite
+llama-3.3-70b-specdec,Command R+,Llama-3.1-Swallow-8B-Instruct-v0.3,gemini-2.0-flash-thinking-exp-01-21
+llama-3.2-1b-preview,,Llama-3.1-Tulu-3-405B,gemini-1.5-flash
+llama-3.2-3b-preview,,Meta-Llama-3.1-405B-Instruct,gemini-1.5-flash-8b
+llama-3.1-8b-instant,,Meta-Llama-3.1-70B-Instruct,
+llama3-70b-8192,,Meta-Llama-3.1-8B-Instruct,
+llama3-8b-8192,,Meta-Llama-3.2-1B-Instruct,
+llama-guard-3-8b,,Meta-Llama-3.2-3B-Instruct,
+mixtral-8x7b-32768,,Meta-Llama-3.3-70B-Instruct,
+gemma2-9b-it,,Qwen2.5-72B-Instruct,
+llama-3.2-11b-vision-preview,,Qwen2.5-Coder-32B-Instruct,
+llama-3.2-90b-vision-preview,,QwQ-32B-Preview,

apikeys.csv

@@ -0,0 +1,6 @@
+GROQ API ,GEMINI API,COHERE  AI,Samba api key
+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,AIzaSyA5XHRFKUIAKKmvPyZA50H9bmXGuio22_o,GEHqd8ojtfK7Q2qTSrH1RNbLPceIXgdtKtsaG1Io,d7277c97-5abf-4f00-afc4-a05f2283b1c9
+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,AIzaSyCFWSxJ8Lwd8eeSrLfEyepU00_TuoGr3YE,IAcJ37niDUV3AuIzwzRLYnSTkrgou4VGKhgAWud0,7952d283-92c9-42aa-8511-52729a5375b3
+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,AIzaSyDw2aCCLrBn2fQIkfjo3mHoNVaOMZEmQPg,EN5HVdaRlruyxpQkMhALF2ViwTb2ZOyfN5oBZnvf,68fa8ba4-5329-493a-a733-495d5fa72386
+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,AIzaSyBbHEMA7zPTnfw3VAmYGVsQQcdAxO1O__Q,FfBExN70G0PAun5MRvw29rbq3soCJgwBb4TsVrFe,52de8b5e-4a32-4c6f-983f-5af048d14d96
+,AIzaSyBSjaqgysbmd74zXax-mQMuR4YeHjUH124,w4EAnXvuW0G3j44qPwoGaMqSaMr03EWOugidAGMs,4cb02d31-2b3f-4e94-b12d-4d694b6f0c48

app.py

@@ -0,0 +1,122 @@
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+import json
+import logging
+import logging.config
+import os
+
+from core.config import API_HOST, API_PORT, CORS_SETTINGS, LOG_CONFIG
+from core.exceptions import APIError, handle_api_error
+from core.text_generation import text_generator
+
+# Configure logging
+logging.config.dictConfig(LOG_CONFIG)
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title="AI Text Generation API",
+             description="API for text generation using multiple AI providers",
+             version="1.0.0")
+
+# Enable CORS with specific headers for SSE
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Update this in production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["Content-Type", "Cache-Control"]
+)
+
+# Mount static files
+frontend_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'frontend')
+app.mount("/static", StaticFiles(directory=frontend_dir), name="static")
+
+class PromptRequest(BaseModel):
+    model: str
+    prompt: str
+
+@app.get("/")
+async def read_root():
+    """Serve the frontend HTML."""
+    return FileResponse(os.path.join(frontend_dir, 'index.html'))
+
+@app.get("/models")
+async def get_models():
+    """Get list of all available models."""
+    try:
+        # Return models as a JSON array
+        return JSONResponse(content=text_generator.get_available_models())
+    except APIError as e:
+        error_response = handle_api_error(e)
+        raise HTTPException(
+            status_code=error_response["status_code"],
+            detail=error_response["detail"]
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error in get_models: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+async def generate_stream(model: str, prompt: str):
+    """Stream generator for text generation."""
+    try:
+        async for chunk in text_generator.generate_stream(model, prompt):
+            # Add extra newline to ensure proper event separation
+            yield f"data: {json.dumps({'content': chunk})}\n\n"
+    except APIError as e:
+        error_response = handle_api_error(e)
+        yield f"data: {json.dumps({'error': error_response['detail']})}\n\n"
+    except Exception as e:
+        logger.error(f"Unexpected error in generate_stream: {str(e)}")
+        yield f"data: {json.dumps({'error': 'Internal server error'})}\n\n"
+    finally:
+        yield "data: [DONE]\n\n"
+
+@app.get("/generate")
+@app.post("/generate")
+async def generate_response(request: Request):
+    """Generate response using selected model (supports both GET and POST)."""
+    try:
+        # Handle both GET and POST methods
+        if request.method == "GET":
+            params = dict(request.query_params)
+            model = params.get("model")
+            prompt = params.get("prompt")
+        else:
+            body = await request.json()
+            model = body.get("model")
+            prompt = body.get("prompt")
+
+        if not model or not prompt:
+            raise HTTPException(status_code=400, detail="Missing model or prompt parameter")
+
+        logger.info(f"Received {request.method} request for model: {model}")
+        
+        headers = {
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no"  # Disable buffering for nginx
+        }
+        
+        return StreamingResponse(
+            generate_stream(model, prompt),
+            media_type="text/event-stream",
+            headers=headers
+        )
+        
+    except APIError as e:
+        error_response = handle_api_error(e)
+        raise HTTPException(
+            status_code=error_response["status_code"],
+            detail=error_response["detail"]
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error in generate_response: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host=API_HOST, port=API_PORT)

core/__init__.py

@@ -0,0 +1,16 @@
+"""
+Core package for the AI Text Generation API.
+
+This package contains the core functionality of the API:
+- Configuration management
+- Key management
+- Text generation
+- Error handling
+"""
+
+from core.config import *
+from core.exceptions import *
+from core.key_manager import key_manager
+from core.text_generation import text_generator
+
+__version__ = "1.0.0"

core/config.py

@@ -0,0 +1,49 @@
+from typing import Dict
+
+# API Provider mapping - Maps model provider names to their API key column names
+PROVIDER_MAP: Dict[str, str] = {
+    "GROQ MODELS": "GROQ API ",  # Note the trailing space
+    "COHERE": "COHERE  AI",
+    "SambaNova": "Samba api key",
+    "GEMINI": "GEMINI API",
+    # New providers will be automatically detected from MODELS.csv
+    # Their API key column names should match the format: "{PROVIDER} API" or "{PROVIDER} api key"
+}
+
+# API Configuration
+API_HOST = "0.0.0.0"
+API_PORT = 80
+
+# CORS Configuration
+CORS_SETTINGS = {
+    "allow_origins": ["*"],
+    "allow_credentials": True,
+    "allow_methods": ["*"],
+    "allow_headers": ["*"]
+}
+
+# Logging Configuration
+LOG_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+        }
+    },
+    "handlers": {
+        "default": {
+            "level": "INFO",
+            "formatter": "standard",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout"
+        }
+    },
+    "loggers": {
+        "": {
+            "handlers": ["default"],
+            "level": "INFO",
+            "propagate": True
+        }
+    }
+}

core/exceptions.py

@@ -0,0 +1,38 @@
+class APIError(Exception):
+    """Base exception for API related errors."""
+    pass
+
+class APIKeyError(APIError):
+    """Exception raised for API key related errors."""
+    pass
+
+class ModelError(APIError):
+    """Exception raised for model related errors."""
+    pass
+
+class ProviderError(APIError):
+    """Exception raised for provider related errors."""
+    pass
+
+class ValidationError(APIError):
+    """Exception raised for input validation errors."""
+    pass
+
+def handle_api_error(error: APIError) -> dict:
+    """Convert API errors to response dictionaries."""
+    error_types = {
+        APIKeyError: (400, "API Key Error"),
+        ModelError: (400, "Model Error"),
+        ProviderError: (500, "Provider Error"),
+        ValidationError: (400, "Validation Error"),
+        APIError: (500, "Internal Server Error")
+    }
+    
+    error_class = type(error)
+    status_code, error_type = error_types.get(error_class, (500, "Unknown Error"))
+    
+    return {
+        "status_code": status_code,
+        "error_type": error_type,
+        "detail": str(error)
+    }

core/key_manager.py

@@ -0,0 +1,131 @@
+import pandas as pd
+from typing import Dict, List
+import logging
+import os
+from core.config import PROVIDER_MAP
+from core.exceptions import APIKeyError
+
+logger = logging.getLogger(__name__)
+
+class KeyManager:
+    def __init__(self):
+        self.current_indices: Dict[str, int] = {}
+        self.api_keys: Dict[str, List[str]] = {}
+        self.models_map: Dict[str, str] = {}
+        self.provider_map = PROVIDER_MAP.copy()
+        self._load_models()  # Load models first to detect providers
+        self._load_api_keys()
+
+    def _detect_api_key_column(self, provider: str, columns: List[str]) -> str:
+        """
+        Detect the API key column name for a provider.
+        Tries different common formats like '{PROVIDER} API' or '{PROVIDER} api key'.
+        """
+        provider_variants = [
+            f"{provider} API",
+            f"{provider} api",
+            f"{provider} API KEY",
+            f"{provider} api key",
+            provider.upper() + " API",
+            provider.lower() + " api",
+        ]
+        
+        for variant in provider_variants:
+            for column in columns:
+                if column.strip().lower() == variant.lower():
+                    return column
+        
+        return None
+
+    def _load_api_keys(self) -> None:
+        """Load API keys from CSV file and initialize rotation indices."""
+        try:
+            backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            keys_path = os.path.join(backend_dir, 'apikeys.csv')
+            keys_df = pd.read_csv(keys_path)
+            
+            logger.info("Available API key columns: %s", keys_df.columns.tolist())
+            
+            # Process both predefined and auto-detected providers
+            for provider in set(self.models_map.values()):
+                if provider not in self.provider_map:
+                    # Try to detect API key column for new provider
+                    api_column = self._detect_api_key_column(provider, keys_df.columns)
+                    if api_column:
+                        self.provider_map[provider] = api_column
+                        logger.info(f"Auto-detected API column '{api_column}' for provider '{provider}'")
+                
+                api_provider = self.provider_map.get(provider)
+                if api_provider and api_provider in keys_df.columns:
+                    valid_keys = [key for key in keys_df[api_provider].dropna()]
+                    logger.info(f"Found {len(valid_keys)} valid keys for {provider}")
+                    if valid_keys:
+                        self.api_keys[provider] = valid_keys
+                        self.current_indices[provider] = 0
+                else:
+                    logger.warning(f"No API keys found for provider: {provider}")
+                    if api_provider:
+                        logger.warning(f"Column '{api_provider}' not found in CSV")
+                    logger.warning(f"Available columns: {keys_df.columns.tolist()}")
+                    
+        except Exception as e:
+            logger.error(f"Error loading API keys: {str(e)}")
+            raise APIKeyError(f"Error loading API keys: {str(e)}")
+
+    def _load_models(self) -> None:
+        """Load models from CSV file and create model-to-provider mapping."""
+        try:
+            backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            models_path = os.path.join(backend_dir, 'MODELS.csv')
+            models_df = pd.read_csv(models_path)
+            
+            logger.info("Available model providers: %s", models_df.columns.tolist())
+            for column in models_df.columns:
+                provider = column.strip()
+                models = [model for model in models_df[column].dropna()]
+                logger.info(f"Loading {len(models)} models for provider {provider}")
+                for model in models:
+                    self.models_map[model.strip()] = provider
+                    
+            logger.info(f"Detected providers: {set(self.models_map.values())}")
+            
+        except Exception as e:
+            logger.error(f"Error loading models: {str(e)}")
+            raise APIKeyError(f"Error loading models: {str(e)}")
+
+    def get_next_key(self, provider: str) -> str:
+        """Get the next API key for the specified provider using rotation."""
+        if provider not in self.api_keys:
+            logger.error(f"No API keys found for provider: {provider}")
+            logger.error(f"Available providers: {list(self.api_keys.keys())}")
+            raise APIKeyError(f"No API keys found for provider: {provider}")
+
+        keys = self.api_keys[provider]
+        current_idx = self.current_indices[provider]
+        total_keys = len(keys)
+        
+        key = keys[current_idx]
+        next_idx = (current_idx + 1) % total_keys
+        self.current_indices[provider] = next_idx
+        
+        key_preview = f"{key[:4]}...{key[-4:]}"
+        logger.info(f"### API Key Rotation for {provider} ###")
+        logger.info(f"Using key #{current_idx + 1}/{total_keys} ({key_preview})")
+        logger.info(f"Next request will use key #{next_idx + 1}/{total_keys}")
+        
+        return key
+
+    def get_provider_for_model(self, model: str) -> str:
+        """Get the provider name for a given model."""
+        model = model.strip()
+        if model not in self.models_map:
+            raise APIKeyError(f"Model not found: {model}")
+        return self.models_map[model]
+
+    def get_available_models(self) -> List[Dict[str, str]]:
+        """Get list of all available models with their providers."""
+        return [{"model": model, "provider": provider} 
+                for model, provider in self.models_map.items()]
+
+# Global instance
+key_manager = KeyManager()

core/text_generation.py

@@ -0,0 +1,87 @@
+from typing import AsyncGenerator, Dict, Any
+import importlib
+import os
+import logging
+from core.exceptions import ModelError, ProviderError
+from core.key_manager import key_manager
+
+logger = logging.getLogger(__name__)
+
+class TextGenerator:
+    """Handles text generation across different model providers."""
+    
+    @staticmethod
+    async def generate_stream(model: str, prompt: str) -> AsyncGenerator[str, None]:
+        """
+        Generate streaming text responses using the specified model.
+        
+        Args:
+            model (str): The name of the model to use
+            prompt (str): The input prompt for text generation
+            
+        Yields:
+            str: Generated text chunks
+            
+        Raises:
+            ModelError: If the model is not found or invalid
+            ProviderError: If there's an error with the provider
+        """
+        try:
+            # Get provider for the selected model
+            provider = key_manager.get_provider_for_model(model)
+            logger.info(f"Using provider {provider} for model {model}")
+            
+            # Get API key
+            api_key = key_manager.get_next_key(provider)
+            logger.info(f"Retrieved API key for provider {provider}")
+            
+            # Import provider module
+            try:
+                # Convert provider name to valid module name
+                module_name = provider.lower().split()[0]  # Get first word in lowercase
+                logger.info(f"Importing module: models.{module_name}")
+                
+                # Check if module exists
+                backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+                module_path = os.path.join(backend_dir, 'models', f'{module_name}.py')
+                
+                if not os.path.exists(module_path):
+                    raise ProviderError(f"Provider module not found at: {module_path}")
+                
+                provider_module = importlib.import_module(f"models.{module_name}")
+                
+                # Verify required functions exist
+                if not hasattr(provider_module, 'run_model_stream'):
+                    raise ProviderError(f"Provider module {module_name} missing required function: run_model_stream")
+                
+            except ImportError as e:
+                raise ProviderError(f"Failed to import provider module: {str(e)}")
+            except Exception as e:
+                raise ProviderError(f"Error loading provider module: {str(e)}")
+            
+            # Generate text stream
+            async for chunk in provider_module.run_model_stream(api_key, model, prompt):
+                yield chunk
+                
+        except Exception as e:
+            logger.error(f"Error in generate_stream: {str(e)}")
+            raise
+
+    @staticmethod
+    def get_available_models() -> Dict[str, Any]:
+        """
+        Get all available models and their providers.
+        
+        Returns:
+            Dict[str, Any]: Dictionary containing model information
+        """
+        try:
+            models = key_manager.get_available_models()
+            # Return just the models array directly since that's what frontend expects
+            return models
+        except Exception as e:
+            logger.error(f"Error getting available models: {str(e)}")
+            raise ModelError(f"Failed to get available models: {str(e)}")
+
+# Global instance
+text_generator = TextGenerator()

models/cohere-test.py

@@ -0,0 +1,12 @@
+import cohere
+
+co = cohere.ClientV2(api_key='NYEQhh6CwVF3dCo57n25032m4s0FurCYI73IYWb5')
+
+response = co.chat_stream(
+    model="command-r-plus-08-2024",
+    messages=[{"role": "user", "content": "write a story about a drogon"}],
+)
+
+for event in response:
+    if event.type == "content-delta":
+        print(event.delta.message.content.text, end="")

models/cohere.py

@@ -0,0 +1,57 @@
+import cohere
+import asyncio
+from typing import AsyncGenerator
+
+async def run_model_stream(api_key: str, model: str, prompt: str):
+    """
+    Run the Cohere model with streaming response.
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Yields:
+        str: Chunks of the generated response
+    """
+    try:
+        client = cohere.Client(api_key=api_key)
+        
+        # Create chat message with streaming
+        response = await asyncio.get_event_loop().run_in_executor(
+            None,
+            lambda: client.chat(
+                chat_history=[],
+                message=prompt,
+                model=model,  # Use model name directly from MODELS.csv
+                stream=True,
+                temperature=0.7
+            )
+        )
+        
+        # Process each chunk
+        for event in response:
+            if hasattr(event, 'text') and event.text:
+                # Use asyncio.sleep to prevent blocking
+                await asyncio.sleep(0)
+                yield event.text
+        
+    except Exception as e:
+        raise Exception(f"Error with Cohere API: {str(e)}")
+
+async def run_model(api_key: str, model: str, prompt: str) -> str:
+    """
+    Run the Cohere model with the provided API key and prompt (non-streaming).
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Returns:
+        str: The generated response
+    """
+    response = ""
+    async for chunk in run_model_stream(api_key, model, prompt):
+        response += chunk
+    return response

models/gemini.py

@@ -0,0 +1,58 @@
+import google.generativeai as genai
+import asyncio
+from typing import AsyncGenerator
+
+async def run_model_stream(api_key: str, model: str, prompt: str):
+    """
+    Run the Gemini model with streaming response.
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Yields:
+        str: Chunks of the generated response
+    """
+    try:
+        # Configure the Gemini API
+        genai.configure(api_key=api_key)
+        
+        # Initialize the model with name from MODELS.csv
+        model_instance = genai.GenerativeModel(model)
+        
+        # Start the streaming response using async executor
+        response = await asyncio.get_event_loop().run_in_executor(
+            None,
+            lambda: model_instance.generate_content(
+                prompt,
+                stream=True
+            )
+        )
+        
+        # Process chunks with async handling
+        for chunk in response:
+            if chunk.text:
+                # Use asyncio.sleep to prevent blocking
+                await asyncio.sleep(0)
+                yield chunk.text
+        
+    except Exception as e:
+        raise Exception(f"Error with Gemini API: {str(e)}")
+
+async def run_model(api_key: str, model: str, prompt: str) -> str:
+    """
+    Run the Gemini model with the provided API key and prompt (non-streaming).
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Returns:
+        str: The generated response
+    """
+    response = ""
+    async for chunk in run_model_stream(api_key, model, prompt):
+        response += chunk
+    return response

models/groq.py

@@ -0,0 +1,75 @@
+from openai import AsyncOpenAI
+import httpx
+
+async def run_model_stream(api_key: str, model: str, prompt: str):
+    """
+    Run the Groq model with streaming response.
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Yields:
+        str: Chunks of the generated response
+    """
+    try:
+        # Initialize AsyncOpenAI client with specific configuration
+        client = AsyncOpenAI(
+            api_key=api_key,
+            base_url="https://api.groq.com/openai/v1",
+            http_client=httpx.AsyncClient(verify=True)  # Async client
+        )
+        
+        completion = await client.chat.completions.create(
+            messages=[
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
+            model=model,
+            stream=True
+        )
+
+        async for chunk in completion:
+            if chunk.choices[0].delta.content is not None:
+                yield chunk.choices[0].delta.content
+        
+    except Exception as e:
+        raise Exception(f"Error with Groq API: {str(e)}")
+
+async def run_model(api_key: str, model: str, prompt: str) -> str:
+    """
+    Run the Groq model with the provided API key and prompt (non-streaming).
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Returns:
+        str: The generated response
+    """
+    try:
+        # Initialize AsyncOpenAI client with specific configuration
+        client = AsyncOpenAI(
+            api_key=api_key,
+            base_url="https://api.groq.com/openai/v1",
+            http_client=httpx.AsyncClient(verify=True)  # Async client
+        )
+        
+        chat_completion = await client.chat.completions.create(
+            messages=[
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
+            model=model
+        )
+        
+        return chat_completion.choices[0].message.content
+        
+    except Exception as e:
+        raise Exception(f"Error with Groq API: {str(e)}")

models/mistral.py

@@ -0,0 +1,20 @@
+import os
+from mistralai import Mistral
+
+api_key = "gZkpXZUIvz1ryxpyODGmsommNbryox2s"
+model = "mistral-large-latest"
+
+client = Mistral(api_key=api_key)
+
+stream_response = client.chat.stream(
+    model = model,
+    messages = [
+        {
+            "role": "user",
+            "content": "What is the best French cheese?",
+        },
+    ]
+)
+
+for chunk in stream_response:
+    print(chunk.data.choices[0].delta.content)

models/sambanova.py

@@ -0,0 +1,90 @@
+import httpx
+import json
+from typing import AsyncGenerator
+
+async def run_model_stream(api_key: str, model: str, prompt: str):
+    """
+    Run the SambaNova model with streaming response.
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Yields:
+        str: Chunks of the generated response
+    """
+    try:
+        # Configure HTTP client with appropriate headers and SSL settings
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream"
+        }
+        
+        async with httpx.AsyncClient(
+            base_url="https://api.sambanova.ai/v1",
+            headers=headers,
+            verify=True,
+            timeout=httpx.Timeout(60.0, read=300.0)
+        ) as client:
+            # Make streaming request
+            async with client.stream(
+                "POST",
+                "/chat/completions",
+                json={
+                    "model": model,  # Use model name directly from MODELS.csv
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": prompt
+                        }
+                    ],
+                    "stream": True,
+                    "temperature": 0.7,
+                    "max_tokens": 2048
+                }
+            ) as response:
+                response.raise_for_status()
+                
+                async for line in response.aiter_lines():
+                    line = line.strip()
+                    if not line:
+                        continue
+                        
+                    if line.startswith("data: "):
+                        data = line[6:].strip()
+                        if data == "[DONE]":
+                            break
+                            
+                        try:
+                            chunk_data = json.loads(data)
+                            if chunk_data.get("choices") and chunk_data["choices"][0].get("delta"):
+                                content = chunk_data["choices"][0]["delta"].get("content")
+                                if content:
+                                    yield content
+                        except Exception as e:
+                            print(f"Error parsing chunk: {e}")
+                            continue
+                            
+    except httpx.HTTPError as e:
+        raise Exception(f"HTTP error with SambaNova API: {str(e)}")
+    except Exception as e:
+        raise Exception(f"Error with SambaNova API: {str(e)}")
+
+async def run_model(api_key: str, model: str, prompt: str) -> str:
+    """
+    Run the SambaNova model with the provided API key and prompt (non-streaming).
+    
+    Args:
+        api_key: The API key to use for this request
+        model: The model name to use
+        prompt: The user's input prompt
+        
+    Returns:
+        str: The generated response
+    """
+    response_text = ""
+    async for chunk in run_model_stream(api_key, model, prompt):
+        response_text += chunk
+    return response_text

requirements.txt

@@ -0,0 +1,8 @@
+fastapi==0.110.0
+uvicorn==0.27.1
+python-multipart==0.0.9
+pandas==2.2.1
+openai==1.13.3
+google-generativeai==0.3.2
+cohere==4.51
+requests==2.31.0

utils.py

@@ -0,0 +1,100 @@
+import pandas as pd
+from typing import Dict, List, Optional
+import os
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class KeyRotator:
+    def __init__(self):
+        self.current_indices: Dict[str, int] = {}
+        self.api_keys: Dict[str, List[str]] = {}
+        self.models_map: Dict[str, str] = {}
+        self._load_api_keys()
+        self._load_models()
+
+    def _load_api_keys(self) -> None:
+        """Load API keys from CSV file and initialize rotation indices."""
+        try:
+            keys_df = pd.read_csv('apikeys.csv')
+            # Map model provider names to API key provider names
+            provider_map = {
+                "GROQ MODELS": "GROQ API ",  # Note the trailing space
+                "COHERE": "COHERE  AI",
+                "SambaNova": "Samba api key",
+                "GEMINI": "GEMINI API"
+            }
+            
+            logger.info("Available API key columns: %s", keys_df.columns.tolist())
+            
+            for model_provider, api_provider in provider_map.items():
+                logger.info(f"Processing provider mapping: {model_provider} -> {api_provider}")
+                if api_provider in keys_df.columns:
+                    valid_keys = [key for key in keys_df[api_provider].dropna()]
+                    logger.info(f"Found {len(valid_keys)} valid keys for {model_provider}")
+                    if valid_keys:
+                        self.api_keys[model_provider] = valid_keys
+                        self.current_indices[model_provider] = 0
+                else:
+                    logger.warning(f"API provider column {api_provider} not found in CSV")
+                    logger.warning(f"Available columns: {keys_df.columns.tolist()}")
+        except Exception as e:
+            logger.error(f"Error loading API keys: {str(e)}")
+            raise Exception(f"Error loading API keys: {str(e)}")
+
+    def _load_models(self) -> None:
+        """Load models from CSV file and create model-to-provider mapping."""
+        try:
+            models_df = pd.read_csv('MODELS.csv')
+            logger.info("Available model providers: %s", models_df.columns.tolist())
+            for column in models_df.columns:
+                provider = column.strip()
+                models = [model for model in models_df[column].dropna()]
+                logger.info(f"Loading {len(models)} models for provider {provider}")
+                for model in models:
+                    self.models_map[model.strip()] = provider
+        except Exception as e:
+            logger.error(f"Error loading models: {str(e)}")
+            raise Exception(f"Error loading models: {str(e)}")
+
+    def get_next_key(self, provider: str) -> str:
+        """Get the next API key for the specified provider using rotation."""
+        if provider not in self.api_keys:
+            logger.error(f"No API keys found for provider: {provider}")
+            logger.error(f"Available providers: {list(self.api_keys.keys())}")
+            raise ValueError(f"No API keys found for provider: {provider}")
+
+        keys = self.api_keys[provider]
+        current_idx = self.current_indices[provider]
+        total_keys = len(keys)
+        
+        # Get current key and update index for next time
+        key = keys[current_idx]
+        next_idx = (current_idx + 1) % total_keys
+        self.current_indices[provider] = next_idx
+        
+        # Log key rotation info with more details
+        key_preview = f"{key[:4]}...{key[-4:]}"
+        logger.info(f"### API Key Rotation for {provider} ###")
+        logger.info(f"Using key #{current_idx + 1}/{total_keys} ({key_preview})")
+        logger.info(f"Next request will use key #{next_idx + 1}/{total_keys}")
+        logger.info(f"#################################")
+        
+        return key
+
+    def get_provider_for_model(self, model: str) -> str:
+        """Get the provider name for a given model."""
+        model = model.strip()
+        if model not in self.models_map:
+            raise ValueError(f"Model not found: {model}")
+        return self.models_map[model]
+
+    def get_available_models(self) -> List[Dict[str, str]]:
+        """Get list of all available models with their providers."""
+        return [{"model": model, "provider": provider} 
+                for model, provider in self.models_map.items()]
+
+# Global instance
+key_rotator = KeyRotator()