Added resoruces folder for configs

main/main.py

@@ -2,26 +2,19 @@
 LLM Inference Server main application using LitServe framework.
 """
 import litserve as ls
-import yaml
 import logging
 import os
-from pathlib import Path
 from fastapi.middleware.cors import CORSMiddleware
 from huggingface_hub import login
 from .routes import router, init_router
 from .api import InferenceApi
+from .utils import load_config
 
 # Store process list globally so it doesn't get garbage collected
 _WORKER_PROCESSES = []
 _MANAGER = None
 
-
-def load_config():
-    """Load configuration from config.yaml"""
-    config_path = Path(__file__).parent / "config.yaml"
-    with open(config_path) as f:
-        return yaml.safe_load(f)
-
+# Load configuration
 config = load_config()
 
 

main/resources/local_config.yaml

@@ -0,0 +1,24 @@
+server:
+  host: "0.0.0.0"
+  port: 8002
+  timeout: 60
+  max_batch_size: 1
+
+llm_server:
+  host: "0.0.0.0"
+  port: 8002  # Will be ignored for hf.space URLs
+  timeout: 60.0
+  api_prefix: "/api/v1"  # This will be used for route prefixing
+  endpoints:
+    generate: "/generate"
+    generate_stream: "/generate/stream"
+    embedding: "/embedding"
+    system_status: "/system/status"
+    system_validate: "/system/validate"
+    model_initialize: "/model/initialize"
+    model_initialize_embedding: "/model/initialize/embedding"
+    model_download: "/model/download"
+
+model:
+  defaults:
+    model_name: "microsoft/Phi-3.5-mini-instruct"

main/utils.py

@@ -1,8 +1,14 @@
 """Utility functions for the inference API."""
 import json
+import logging
+import os
 import re
+from pathlib import Path
 from typing import Dict, Any
 
+import yaml
+
+
 def extract_json(text: str) -> Dict[str, Any]:
     """Extract JSON from text that might contain other content.
 
@@ -25,4 +31,36 @@ def extract_json(text: str) -> Dict[str, Any]:
             continue
 
     # If we couldn't find any valid JSON, raise an error
-    raise ValueError("No valid JSON found in response")
+    raise ValueError("No valid JSON found in response")
+
+def load_config():
+    """
+    Load configuration from config files in the resources directory.
+    Uses CONFIG_ENV environment variable to determine which config to load.
+    Defaults to 'local' if no environment is specified.
+    """
+    # Get environment name from env var, default to 'local'
+    env_name = os.environ.get("CONFIG_ENV", "local")
+
+    # Construct path to resources directory and config file
+    resources_dir = Path(__file__).parent / "resources"
+    config_path = resources_dir / f"{env_name}_config.yaml"
+
+    # Create resources directory if it doesn't exist
+    resources_dir.mkdir(exist_ok=True)
+
+    # Check if config file exists
+    if not config_path.exists():
+        logging.warning(f"Config file {config_path} not found, falling back to local_config.yaml")
+        config_path = resources_dir / "local_config.yaml"
+
+        # If even local config doesn't exist, raise error
+        if not config_path.exists():
+            raise FileNotFoundError(
+                f"No configuration file found at {config_path}. "
+                "Please ensure at least local_config.yaml exists in the resources directory."
+            )
+
+    # Load and return config
+    with open(config_path) as f:
+        return yaml.safe_load(f)