Update app.py

app.py

@@ -1,13 +1,24 @@
 import json
+import logging
 import requests
 import streamlit as st
 import pdfplumber
 import pandas as pd
 import sqlalchemy
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Callable
 from functools import lru_cache
 
-# Provider clients with import guards
+# --- Logging Configuration ---
+logger = logging.getLogger("SyntheticDataGenerator")
+logger.setLevel(logging.DEBUG)
+if not logger.handlers:
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+# --- Provider Clients with Import Guards ---
 try:
     from openai import OpenAI
 except ImportError:
@@ -19,6 +30,13 @@ except ImportError:
     groq = None
 
 
+# --- Custom Exceptions ---
+class ProviderClientError(Exception):
+    """Custom exception for provider client issues."""
+    pass
+
+
+# --- Core Synthetic Data Generator ---
 class SyntheticDataGenerator:
     """World's Most Advanced Synthetic Data Generation System"""
 
@@ -46,42 +64,36 @@ class SyntheticDataGenerator:
     }
 
     def __init__(self) -> None:
-        """Initialize session state, input handlers, and providers."""
         self._init_session_state()
-        self._setup_input_handlers()
         self._setup_providers()
+        self._setup_input_handlers()
 
     def _init_session_state(self) -> None:
-        """Initialize enterprise-grade session management with default values."""
+        """Initialize session state with default values."""
         defaults = {
             "active_provider": "OpenAI",
             "api_keys": {},
-            "input_sources": [],
-            "generation_results": [],
-            "system_metrics": {
-                "api_calls": 0,
-                "tokens_used": 0,
-                "error_count": 0,
-            },
+            "system_metrics": {"api_calls": 0, "tokens_used": 0, "error_count": 0},
             "error_logs": [],
             "debug_mode": False,
         }
-        for key, val in defaults.items():
+        for key, value in defaults.items():
             if key not in st.session_state:
-                st.session_state[key] = val
+                st.session_state[key] = value
 
     def _setup_providers(self) -> None:
-        """Configure available providers with health checks."""
+        """Configure available providers based on dependency availability."""
         self.available_providers: List[str] = []
         for provider, config in self.PROVIDER_CONFIG.items():
             required_lib = config.get("requires_library")
             if required_lib and not globals().get(required_lib.title()):
-                continue  # Skip providers with missing dependencies
+                logger.warning(f"Skipping provider {provider} due to missing dependency: {required_lib}")
+                continue
             self.available_providers.append(provider)
 
     def _setup_input_handlers(self) -> None:
-        """Register enterprise input processors."""
-        self.input_processors = {
+        """Register input processors."""
+        self.input_processors: Dict[str, Callable[[Any], str]] = {
             "text": self._process_text,
             "pdf": self._process_pdf,
             "csv": self._process_csv,
@@ -93,65 +105,53 @@ class SyntheticDataGenerator:
     @lru_cache(maxsize=100)
     def generate(self, provider: str, model: str, prompt: str) -> Dict[str, Any]:
         """
-        Unified generation endpoint with failover support.
-        Caches results to improve performance.
+        Unified generation endpoint with caching and failover support.
         """
         try:
             if provider not in self.available_providers:
-                raise ValueError(f"Provider {provider} is not available.")
-
+                raise ProviderClientError(f"Provider {provider} is not available.")
             client = self._get_client(provider)
             if not client:
-                raise ConnectionError("Client initialization failed.")
-
+                raise ProviderClientError(f"Client initialization failed for provider {provider}.")
             return self._execute_generation(client, provider, model, prompt)
-
         except Exception as e:
-            self._log_error(f"Generation Error with provider '{provider}': {str(e)}")
+            self._log_error(f"Generation error using provider '{provider}': {e}")
             return self._failover_generation(provider, model, prompt)
 
     def _get_client(self, provider: str) -> Any:
         """
-        Secure client initialization with connection pooling.
-        Raises ValueError if API key is missing.
+        Initialize and return a client for the specified provider.
+        Raises ProviderClientError if API key or dependency issues occur.
         """
         config = self.PROVIDER_CONFIG[provider]
         api_key = st.session_state["api_keys"].get(provider, "")
-
         if not api_key:
-            raise ValueError(f"API key required for provider {provider}.")
-
+            raise ProviderClientError(f"Missing API key for {provider}.")
         try:
             if provider == "Groq":
                 return groq.Groq(api_key=api_key)
             elif provider == "HuggingFace":
                 return {"headers": {"Authorization": f"Bearer {api_key}"}}
             else:
-                # Initialize OpenAI client for OpenAI and Deepseek
                 return OpenAI(
                     base_url=config["base_url"],
                     api_key=api_key,
-                    timeout=30
+                    timeout=30,
                 )
         except Exception as e:
-            self._log_error(f"Client Initialization Failed for {provider}: {str(e)}")
-            return None
+            self._log_error(f"Error initializing client for {provider}: {e}")
+            raise ProviderClientError(f"Client init error for {provider}")
 
     def _execute_generation(self, client: Any, provider: str, model: str, prompt: str) -> Dict[str, Any]:
         """
-        Execute provider-specific generation with circuit breaker.
-        Increments API call and token usage metrics.
+        Execute the generation request for the given provider.
+        Updates system metrics and returns the result.
         """
         st.session_state["system_metrics"]["api_calls"] += 1
 
         if provider == "HuggingFace":
             url = self.PROVIDER_CONFIG[provider]["base_url"] + model
-            response = requests.post(
-                url,
-                headers=client["headers"],
-                json={"inputs": prompt},
-                timeout=30
-            )
+            response = requests.post(url, headers=client["headers"], json={"inputs": prompt}, timeout=30)
             response.raise_for_status()
             return response.json()
         else:
@@ -159,77 +159,70 @@ class SyntheticDataGenerator:
                 model=model,
                 messages=[{"role": "user", "content": prompt}],
                 temperature=0.1,
-                max_tokens=2000
+                max_tokens=2000,
             )
-            # Update token usage if available
             if hasattr(completion.usage, "total_tokens"):
                 st.session_state["system_metrics"]["tokens_used"] += completion.usage.total_tokens
             try:
-                result = json.loads(completion.choices[0].message.content)
-            except json.JSONDecodeError:
-                result = {"response": completion.choices[0].message.content}
-            return result
+                return json.loads(completion.choices[0].message.content)
+            except Exception:
+                return {"response": completion.choices[0].message.content}
 
     def _failover_generation(self, original_provider: str, model: str, prompt: str) -> Dict[str, Any]:
         """
-        Enterprise failover to secondary providers.
-        Tries available providers (excluding the original) until one succeeds.
+        Attempt to generate synthetic data using alternative providers.
         """
         for backup_provider in self.available_providers:
             if backup_provider == original_provider:
                 continue
-            # Determine model to use: if the desired model is available, use it; otherwise use the first available model.
             backup_models = self.PROVIDER_CONFIG[backup_provider]["models"]
             backup_model = model if model in backup_models else backup_models[0]
             try:
                 st.session_state["active_provider"] = backup_provider
                 result = self.generate(backup_provider, backup_model, prompt)
-                # Optionally, log the failover event
-                self._log_error(f"Failover succeeded with provider '{backup_provider}' using model '{backup_model}'.")
+                self._log_error(f"Failover succeeded: provider '{backup_provider}' with model '{backup_model}'")
                 return result
             except Exception as e:
-                self._log_error(f"Failover attempt with provider '{backup_provider}' failed: {str(e)}")
-                continue
-        raise RuntimeError("All generation providers are unavailable.")
+                self._log_error(f"Failover attempt with {backup_provider} failed: {e}")
+        raise ProviderClientError("All generation providers failed.")
 
     # --- Input Processors ---
     def _process_text(self, text: str) -> str:
-        """Process plain text input by stripping unnecessary whitespace."""
+        """Strip and return plain text input."""
         return text.strip()
 
     def _process_pdf(self, file) -> str:
-        """Advanced PDF processing with OCR fallback."""
+        """Extract and return text from a PDF file."""
         try:
             with pdfplumber.open(file) as pdf:
-                return "\n".join(page.extract_text() or "" for page in pdf.pages)
+                return "\n".join((page.extract_text() or "") for page in pdf.pages)
         except Exception as e:
-            self._log_error(f"PDF Processing Error: {str(e)}")
+            self._log_error(f"PDF processing error: {e}")
             return ""
 
     def _process_csv(self, file) -> str:
-        """Process CSV input by reading it into a DataFrame and converting to CSV string."""
+        """Convert CSV file to string via DataFrame conversion."""
         try:
             df = pd.read_csv(file)
             return df.to_csv(index=False)
         except Exception as e:
-            self._log_error(f"CSV Processing Error: {str(e)}")
+            self._log_error(f"CSV processing error: {e}")
             return ""
 
     def _process_api(self, api_url: str) -> str:
-        """Process API input by fetching JSON data from an endpoint."""
+        """Fetch and return JSON data from the provided API URL."""
         try:
             response = requests.get(api_url, timeout=10)
             response.raise_for_status()
-            data = response.json()
-            return json.dumps(data, indent=2)
+            return json.dumps(response.json(), indent=2)
         except Exception as e:
-            self._log_error(f"API Processing Error: {str(e)}")
+            self._log_error(f"API processing error: {e}")
             return ""
 
     def _process_database(self, config: Dict[str, str]) -> str:
         """
-        Process database input by executing a query.
-        Expects a configuration dictionary with 'connection_string' and 'query' keys.
+        Execute a database query using a provided configuration.
+        Expects a dict with 'connection_string' and 'query' keys.
         """
         try:
             connection_string = config.get("connection_string", "")
@@ -238,45 +231,43 @@ class SyntheticDataGenerator:
                 raise ValueError("Missing connection string or query.")
             engine = sqlalchemy.create_engine(connection_string)
             with engine.connect() as connection:
-                result = pd.read_sql(query, connection)
-            return result.to_csv(index=False)
+                df = pd.read_sql(query, connection)
+            return df.to_csv(index=False)
         except Exception as e:
-            self._log_error(f"Database Processing Error: {str(e)}")
+            self._log_error(f"Database processing error: {e}")
             return ""
 
     def _process_web(self, url: str) -> str:
-        """Web content extraction with anti-bot measures."""
+        """Fetch and return webpage content using anti-bot headers."""
         try:
-            response = requests.get(url, headers={
-                "User-Agent": "Mozilla/5.0 (compatible; SyntheticBot/1.0)"
-            }, timeout=10)
+            response = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (SyntheticBot/1.0)"}, timeout=10)
             response.raise_for_status()
             return response.text
         except Exception as e:
-            self._log_error(f"Web Extraction Error: {str(e)}")
+            self._log_error(f"Web extraction error: {e}")
             return ""
 
-    # --- Enterprise Features ---
+    # --- Logging & Diagnostics ---
     def _log_error(self, message: str) -> None:
-        """Centralized error logging with telemetry."""
+        """Log errors centrally and update system metrics."""
         st.session_state["system_metrics"]["error_count"] += 1
         st.session_state["error_logs"].append(message)
+        logger.error(message)
         if st.session_state.get("debug_mode"):
             st.error(f"[DEBUG] {message}")
 
     def health_check(self) -> Dict[str, Any]:
-        """Comprehensive system diagnostics."""
+        """Return diagnostics including provider connectivity and system metrics."""
+        connectivity = {provider: self._test_provider_connectivity(provider)
+                        for provider in self.available_providers}
         return {
             "providers_available": self.available_providers,
-            "api_connectivity": {
-                provider: self._test_provider_connectivity(provider)
-                for provider in self.available_providers
-            },
+            "api_connectivity": connectivity,
             "system_metrics": st.session_state["system_metrics"],
         }
 
     def _test_provider_connectivity(self, provider: str) -> bool:
-        """Provider-specific connectivity test."""
+        """Test connectivity for a given provider."""
         try:
             client = self._get_client(provider)
             if provider == "HuggingFace":
@@ -284,23 +275,23 @@ class SyntheticDataGenerator:
                 response = requests.get(url, headers=client["headers"], timeout=5)
                 return response.status_code == 200
             else:
-                # Assuming client has a models.list() method for other providers.
                 client.models.list()
                 return True
-        except Exception:
+        except Exception as e:
+            self._log_error(f"Connectivity test failed for {provider}: {e}")
             return False
 
 
-# --- Enterprise UI Components ---
-def provider_config_ui(gen: SyntheticDataGenerator) -> None:
-    """Advanced provider configuration interface."""
+# --- Streamlit UI Components ---
+def provider_config_ui(generator: SyntheticDataGenerator) -> None:
+    """Provider configuration and health check UI."""
     with st.sidebar:
         st.header("⚙️ AI Engine Configuration")
         provider = st.selectbox(
             "AI Provider",
-            gen.available_providers,
-            help="Available providers based on system configuration",
-            index=gen.available_providers.index(st.session_state.get("active_provider", "OpenAI"))
+            generator.available_providers,
+            index=generator.available_providers.index(st.session_state.get("active_provider", "OpenAI")),
+            help="Select your preferred AI provider."
         )
         st.session_state["active_provider"] = provider
 
@@ -308,30 +299,25 @@ def provider_config_ui(gen: SyntheticDataGenerator) -> None:
             f"{provider} API Key",
             type="password",
             value=st.session_state["api_keys"].get(provider, ""),
-            help=f"Obtain API key from {provider}'s portal"
+            help=f"Enter your API key for {provider}."
         )
         st.session_state["api_keys"][provider] = api_key
 
         model = st.selectbox(
             "Model",
-            gen.PROVIDER_CONFIG[provider]["models"],
-            help="Select model version based on your API plan"
+            generator.PROVIDER_CONFIG[provider]["models"],
+            help="Select the model to use."
         )
 
         if st.button("Run Health Check"):
-            report = gen.health_check()
+            report = generator.health_check()
             st.json(report)
 
 
 def main() -> None:
-    """Enterprise-grade user interface."""
-    st.set_page_config(
-        page_title="Synthetic Data Factory Pro",
-        page_icon="🏭",
-        layout="wide"
-    )
-
-    gen = SyntheticDataGenerator()
+    """Main Streamlit UI entry point."""
+    st.set_page_config(page_title="Synthetic Data Factory Pro", page_icon="🏭", layout="wide")
+    generator = SyntheticDataGenerator()
 
     st.title("🏭 Synthetic Data Factory Pro")
     st.markdown(
@@ -341,13 +327,13 @@ def main() -> None:
         """
     )
 
-    provider_config_ui(gen)
+    provider_config_ui(generator)
 
-    # Example: Input management and processing
+    # --- Input Data Section ---
     st.subheader("Input Data")
-    input_type = st.selectbox("Select Input Type", list(gen.input_processors.keys()))
+    input_type = st.selectbox("Select Input Type", list(generator.input_processors.keys()))
     if input_type == "text":
-        user_input = st.text_area("Enter your text here:")
+        user_input = st.text_area("Enter your text:")
     elif input_type == "pdf":
         user_input = st.file_uploader("Upload a PDF file", type=["pdf"])
     elif input_type == "csv":
@@ -361,29 +347,32 @@ def main() -> None:
 
     processed_input = ""
     if st.button("Process Input"):
-        processor = gen.input_processors.get(input_type)
+        processor = generator.input_processors.get(input_type)
         if processor:
-            if input_type in ["pdf", "csv"]:
+            if input_type in ("pdf", "csv"):
                 processed_input = processor(user_input)
             elif input_type == "database":
                 try:
-                    db_config = json.loads(user_input)
-                    processed_input = processor(db_config)
-                except json.JSONDecodeError:
-                    st.error("Invalid JSON for database configuration.")
+                    config = json.loads(user_input)
+                    processed_input = processor(config)
+                except Exception as e:
+                    st.error("Invalid JSON configuration for database.")
+                    processed_input = ""
             else:
                 processed_input = processor(user_input)
             st.text_area("Processed Input", value=processed_input, height=200)
 
-    # Generation section
+    # --- Data Generation Section ---
     st.subheader("Generate Synthetic Data")
-    prompt = st.text_area("Enter your prompt for data generation:")
+    prompt = st.text_area("Enter your generation prompt:")
     if st.button("Generate"):
         active_provider = st.session_state.get("active_provider", "OpenAI")
-        # Allow model selection for the generation step
-        model = st.selectbox("Select Generation Model", gen.PROVIDER_CONFIG[active_provider]["models"])
-        result = gen.generate(active_provider, model, prompt)
-        st.json(result)
+        model = st.selectbox("Select Generation Model", generator.PROVIDER_CONFIG[active_provider]["models"])
+        try:
+            result = generator.generate(active_provider, model, prompt)
+            st.json(result)
+        except Exception as e:
+            st.error(f"Data generation failed: {e}")
 
 
 if __name__ == "__main__":