Update app.py

app.py

@@ -1,11 +1,10 @@
 import json
-import logging
 import requests
 import streamlit as st
 import pdfplumber
 import pandas as pd
 import sqlalchemy
-from typing import Any, Dict, List, Callable
+from typing import Any, Dict, List
 
 # Provider clients – ensure these libraries are installed
 try:
@@ -18,20 +17,10 @@ try:
 except ImportError:
     groq = None
 
-# Hugging Face inference endpoint and defaults
-HF_API_URL: str = "https://api-inference.huggingface.co/models/"
-DEFAULT_TEMPERATURE: float = 0.1
-GROQ_MODEL: str = "mixtral-8x7b-32768"
-
-# Optional: Configure a basic logger for debugging purposes
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-if not logger.handlers:
-    ch = logging.StreamHandler()
-    ch.setLevel(logging.DEBUG)
-    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
-    ch.setFormatter(formatter)
-    logger.addHandler(ch)
+# Hugging Face inference endpoint
+HF_API_URL = "https://api-inference.huggingface.co/models/"
+DEFAULT_TEMPERATURE = 0.1
+GROQ_MODEL = "mixtral-8x7b-32768"
 
 
 class QADataGenerator:
@@ -39,29 +28,26 @@ class QADataGenerator:
     A Q&A Synthetic Generator that extracts and generates question-answer pairs
     from various input sources using an LLM provider.
     """
-
     def __init__(self) -> None:
         self._setup_providers()
         self._setup_input_handlers()
         self._initialize_session_state()
-        # Prompt template for generating 3 Q&A pairs.
-        self.custom_prompt_template: str = (
+        # Updated prompt template with escaped curly braces
+        self.custom_prompt_template = (
             "You are an expert in extracting question and answer pairs from documents. "
             "Generate 3 Q&A pairs from the following data, formatted as a JSON list of dictionaries. "
             "Each dictionary must have keys 'question' and 'answer'. "
             "The questions should be clear and concise, and the answers must be based solely on the provided data with no external information. "
             "Do not hallucinate. \n\n"
             "Example JSON Output:\n"
-            "[{'question': 'What is the capital of France?', 'answer': 'Paris'}, "
-            "{'question': 'What is the highest mountain in the world?', 'answer': 'Mount Everest'}, "
-            "{'question': 'What is the chemical symbol for gold?', 'answer': 'Au'}]\n\n"
+            "[{{'question': 'What is the capital of France?', 'answer': 'Paris'}}, "
+            "{{'question': 'What is the highest mountain in the world?', 'answer': 'Mount Everest'}}, "
+            "{{'question': 'What is the chemical symbol for gold?', 'answer': 'Au'}}]\n\n"
             "Now, generate 3 Q&A pairs from this data:\n{data}"
         )
-
+    
     def _setup_providers(self) -> None:
-        """
-        Configure available LLM providers and their client initialization routines.
-        """
+        """Configure available LLM providers and their client initialization routines."""
         self.providers: Dict[str, Dict[str, Any]] = {
             "Deepseek": {
                 "client": lambda key: OpenAI(base_url="https://api.deepseek.com/v1", api_key=key) if OpenAI else None,
@@ -80,24 +66,20 @@ class QADataGenerator:
                 "models": ["gpt2", "llama-2"],
             },
         }
-
+    
     def _setup_input_handlers(self) -> None:
-        """
-        Register handlers for different input data types.
-        """
-        self.input_handlers: Dict[str, Callable[[Any], Dict[str, Any]]] = {
+        """Register handlers for different input data types."""
+        self.input_handlers: Dict[str, Any] = {
             "text": self.handle_text,
             "pdf": self.handle_pdf,
             "csv": self.handle_csv,
             "api": self.handle_api,
             "db": self.handle_db,
         }
-
+    
     def _initialize_session_state(self) -> None:
-        """
-        Initialize Streamlit session state with default configuration.
-        """
-        defaults: Dict[str, Any] = {
+        """Initialize Streamlit session state with default configuration."""
+        defaults = {
             "config": {
                 "provider": "OpenAI",
                 "model": "gpt-4-turbo",
@@ -111,50 +93,38 @@ class QADataGenerator:
         for key, value in defaults.items():
             if key not in st.session_state:
                 st.session_state[key] = value
-
+    
     def log_error(self, message: str) -> None:
-        """
-        Log an error message to session state, display it, and send it to the logger.
-        """
+        """Log an error message to session state and display it."""
         st.session_state.error_logs.append(message)
         st.error(message)
-        logger.error(message)
-
+    
     # ----- Input Handlers -----
     def handle_text(self, text: str) -> Dict[str, Any]:
-        """
-        Process plain text input.
-        """
         return {"data": text, "source": "text"}
-
+    
     def handle_pdf(self, file) -> Dict[str, Any]:
-        """
-        Extract text from a PDF file.
-        """
         try:
             with pdfplumber.open(file) as pdf:
-                full_text = "\n".join(page.extract_text() or "" for page in pdf.pages)
-            return {"data": full_text, "source": "pdf"}
+                full_text = ""
+                for page in pdf.pages:
+                    page_text = page.extract_text() or ""
+                    full_text += page_text + "\n"
+                return {"data": full_text, "source": "pdf"}
         except Exception as e:
             self.log_error(f"PDF Processing Error: {e}")
             return {"data": "", "source": "pdf"}
-
+    
     def handle_csv(self, file) -> Dict[str, Any]:
-        """
-        Process a CSV file by reading it into a DataFrame and converting it to JSON.
-        """
         try:
             df = pd.read_csv(file)
-            json_data = df.to_json(orient="records")
-            return {"data": json_data, "source": "csv"}
+            # Convert the DataFrame to a JSON string
+            return {"data": df.to_json(orient="records"), "source": "csv"}
         except Exception as e:
             self.log_error(f"CSV Processing Error: {e}")
             return {"data": "", "source": "csv"}
-
+    
     def handle_api(self, config: Dict[str, str]) -> Dict[str, Any]:
-        """
-        Fetch data from an API endpoint.
-        """
         try:
             response = requests.get(config["url"], headers=config.get("headers", {}), timeout=10)
             response.raise_for_status()
@@ -162,11 +132,8 @@ class QADataGenerator:
         except Exception as e:
             self.log_error(f"API Processing Error: {e}")
             return {"data": "", "source": "api"}
-
+    
     def handle_db(self, config: Dict[str, str]) -> Dict[str, Any]:
-        """
-        Query a database using the provided connection string and SQL query.
-        """
         try:
             engine = sqlalchemy.create_engine(config["connection"])
             with engine.connect() as conn:
@@ -176,17 +143,15 @@ class QADataGenerator:
         except Exception as e:
             self.log_error(f"Database Processing Error: {e}")
             return {"data": "", "source": "db"}
-
+    
     def aggregate_inputs(self) -> str:
-        """
-        Combine all input sources into a single aggregated string.
-        """
+        """Combine all input sources into a single aggregated string."""
         aggregated_data = ""
         for item in st.session_state.inputs:
             aggregated_data += f"Source: {item.get('source', 'unknown')}\n"
             aggregated_data += item.get("data", "") + "\n\n"
         return aggregated_data.strip()
-
+    
     def build_prompt(self) -> str:
         """
         Build the complete prompt using the custom template and aggregated inputs.
@@ -196,52 +161,50 @@ class QADataGenerator:
         st.write("### Built Prompt")
         st.write(prompt)
         return prompt
-
+    
     def generate_qa_pairs(self) -> bool:
         """
         Generate Q&A pairs by sending the built prompt to the selected LLM provider.
         """
-        api_key: str = st.session_state.api_key
+        api_key = st.session_state.api_key
         if not api_key:
             self.log_error("API key is missing!")
             return False
-
-        provider_name: str = st.session_state.config["provider"]
-        provider_cfg: Dict[str, Any] = self.providers.get(provider_name, {})
+        
+        provider_name = st.session_state.config["provider"]
+        provider_cfg = self.providers.get(provider_name)
         if not provider_cfg:
             self.log_error(f"Provider {provider_name} is not configured.")
             return False
-
-        client_initializer: Callable[[str], Any] = provider_cfg["client"]
+        
+        client_initializer = provider_cfg["client"]
         client = client_initializer(api_key)
-        model: str = st.session_state.config["model"]
-        temperature: float = st.session_state.config["temperature"]
-        prompt: str = self.build_prompt()
-
+        model = st.session_state.config["model"]
+        temperature = st.session_state.config["temperature"]
+        prompt = self.build_prompt()
+        
         st.info(f"Using **{provider_name}** with model **{model}** at temperature **{temperature:.2f}**")
         try:
             if provider_name == "HuggingFace":
                 response = self._huggingface_inference(client, prompt, model)
             else:
                 response = self._standard_inference(client, prompt, model, temperature)
-
+            
             st.write("### Raw API Response")
             st.write(response)
-
+            
             qa_pairs = self._parse_response(response, provider_name)
             st.write("### Parsed Q&A Pairs")
             st.write(qa_pairs)
-
+            
             st.session_state.qa_pairs = qa_pairs
             return True
         except Exception as e:
             self.log_error(f"Generation failed: {e}")
             return False
-
+    
     def _standard_inference(self, client: Any, prompt: str, model: str, temperature: float) -> Any:
-        """
-        Inference method for providers using an OpenAI-compatible API.
-        """
+        """Inference method for providers using an OpenAI-compatible API."""
         try:
             st.write("Sending prompt via standard inference...")
             result = client.chat.completions.create(
@@ -254,11 +217,9 @@ class QADataGenerator:
         except Exception as e:
             self.log_error(f"Standard Inference Error: {e}")
             return None
-
+    
     def _huggingface_inference(self, client: Dict[str, Any], prompt: str, model: str) -> Any:
-        """
-        Inference method for the Hugging Face Inference API.
-        """
+        """Inference method for the Hugging Face Inference API."""
         try:
             st.write("Sending prompt to HuggingFace API...")
             response = requests.post(
@@ -273,7 +234,7 @@ class QADataGenerator:
         except Exception as e:
             self.log_error(f"HuggingFace Inference Error: {e}")
             return None
-
+    
     def _parse_response(self, response: Any, provider: str) -> List[Dict[str, str]]:
         """
         Parse the LLM response and return a list of Q&A pairs.
@@ -289,14 +250,15 @@ class QADataGenerator:
                     self.log_error("Unexpected HuggingFace response format.")
                     return []
             else:
-                # For OpenAI and similar providers, expect response.choices[0].message.content
+                # For OpenAI (and similar providers) assume the response is similar to:
+                # response.choices[0].message.content
                 if response and hasattr(response, "choices") and response.choices:
                     raw_text = response.choices[0].message.content
                 else:
                     self.log_error("Unexpected response format from provider.")
                     return []
-
-            # Parse the raw text as JSON
+            
+            # Try parsing the raw text as JSON
             try:
                 qa_list = json.loads(raw_text)
                 if isinstance(qa_list, list):
@@ -314,33 +276,28 @@ class QADataGenerator:
 
 # ============ UI Components ============
 
-def config_ui(generator: QADataGenerator) -> None:
-    """
-    Display configuration options in the sidebar.
-    """
+def config_ui(generator: QADataGenerator):
+    """Display configuration options in the sidebar."""
     with st.sidebar:
         st.header("Configuration")
         provider = st.selectbox("Select Provider", list(generator.providers.keys()))
         st.session_state.config["provider"] = provider
         provider_cfg = generator.providers[provider]
-
+        
         model = st.selectbox("Select Model", provider_cfg["models"])
         st.session_state.config["model"] = model
-
+        
         temperature = st.slider("Temperature", 0.0, 1.0, DEFAULT_TEMPERATURE)
         st.session_state.config["temperature"] = temperature
-
+        
         api_key = st.text_input(f"{provider} API Key", type="password")
         st.session_state.api_key = api_key
 
-
-def input_ui(generator: QADataGenerator) -> None:
-    """
-    Display input data source options using tabs.
-    """
+def input_ui(generator: QADataGenerator):
+    """Display input data source options using tabs."""
     st.subheader("Input Data Sources")
     tabs = st.tabs(["Text", "PDF", "CSV", "API", "Database"])
-
+    
     with tabs[0]:
         text_input = st.text_area("Enter text input", height=150)
         if st.button("Add Text Input", key="text_input"):
@@ -349,19 +306,19 @@ def input_ui(generator: QADataGenerator) -> None:
                 st.success("Text input added!")
             else:
                 st.warning("Empty text input.")
-
+    
     with tabs[1]:
         pdf_file = st.file_uploader("Upload PDF", type=["pdf"])
         if pdf_file is not None:
             st.session_state.inputs.append(generator.handle_pdf(pdf_file))
             st.success("PDF input added!")
-
+    
     with tabs[2]:
         csv_file = st.file_uploader("Upload CSV", type=["csv"])
         if csv_file is not None:
             st.session_state.inputs.append(generator.handle_csv(csv_file))
             st.success("CSV input added!")
-
+    
     with tabs[3]:
         api_url = st.text_input("API Endpoint URL")
         api_headers = st.text_area("API Headers (JSON format, optional)", height=100)
@@ -374,7 +331,7 @@ def input_ui(generator: QADataGenerator) -> None:
                 generator.log_error(f"Invalid JSON for API Headers: {e}")
             st.session_state.inputs.append(generator.handle_api({"url": api_url, "headers": headers}))
             st.success("API input added!")
-
+    
     with tabs[4]:
         db_conn = st.text_input("Database Connection String")
         db_query = st.text_area("Database Query", height=100)
@@ -382,11 +339,8 @@ def input_ui(generator: QADataGenerator) -> None:
             st.session_state.inputs.append(generator.handle_db({"connection": db_conn, "query": db_query}))
             st.success("Database input added!")
 
-
-def output_ui(generator: QADataGenerator) -> None:
-    """
-    Display the generated Q&A pairs and provide a download option.
-    """
+def output_ui(generator: QADataGenerator):
+    """Display the generated Q&A pairs and provide a download option."""
     st.subheader("Q&A Pairs Output")
     if st.session_state.qa_pairs:
         st.write("### Generated Q&A Pairs")
@@ -395,16 +349,13 @@ def output_ui(generator: QADataGenerator) -> None:
             "Download Output",
             json.dumps(st.session_state.qa_pairs, indent=2),
             file_name="qa_pairs.json",
-            mime="application/json",
+            mime="application/json"
         )
     else:
         st.info("No Q&A pairs generated yet.")
 
-
-def logs_ui() -> None:
-    """
-    Display error logs and debugging information in an expandable section.
-    """
+def logs_ui():
+    """Display error logs and debugging information in an expandable section."""
     with st.expander("Error Logs & Debug Info", expanded=False):
         if st.session_state.error_logs:
             for log in st.session_state.error_logs:
@@ -412,11 +363,7 @@ def logs_ui() -> None:
         else:
             st.write("No logs yet.")
 
-
-def main() -> None:
-    """
-    Main Streamlit application entry point.
-    """
+def main():
     st.set_page_config(page_title="Advanced Q&A Synthetic Generator", layout="wide")
     st.title("Advanced Q&A Synthetic Generator")
     st.markdown(
@@ -425,17 +372,17 @@ def main() -> None:
         from various input sources. Configure your provider in the sidebar, add input data, and click the button below to generate Q&A pairs.
         """
     )
-
+    
     # Initialize generator and display configuration UI
     generator = QADataGenerator()
     config_ui(generator)
-
+    
     st.header("1. Input Data")
     input_ui(generator)
     if st.button("Clear All Inputs"):
         st.session_state.inputs = []
         st.success("All inputs have been cleared!")
-
+    
     st.header("2. Generate Q&A Pairs")
     if st.button("Generate Q&A Pairs", key="generate_qa"):
         with st.spinner("Generating Q&A pairs..."):
@@ -443,13 +390,12 @@ def main() -> None:
                 st.success("Q&A pairs generated successfully!")
             else:
                 st.error("Q&A generation failed. Check logs for details.")
-
+    
     st.header("3. Output")
     output_ui(generator)
-
+    
     st.header("4. Logs & Debug Information")
     logs_ui()
 
-
 if __name__ == "__main__":
     main()