Update app.py

app.py

@@ -16,8 +16,6 @@ from scipy.stats import ttest_ind, f_oneway
 from sklearn.model_selection import train_test_split
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import accuracy_score
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
 
 from statsmodels.tsa.seasonal import seasonal_decompose
 from statsmodels.tsa.stattools import adfuller
@@ -27,29 +25,36 @@ from Bio import Entrez  # Ensure BioPython is installed
 
 from dotenv import load_dotenv
 import requests
+import openai  # Added for OpenAI GPT-4 integration
 import ast
+import logging
 
 # ---------------------- Load Environment Variables ---------------------------
 load_dotenv()
 
+# ---------------------- Logging Configuration ---------------------------
+logging.basicConfig(
+    filename='app.log',
+    filemode='a',
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+logger = logging.getLogger()
+
 # ---------------------- Streamlit Page Configuration ---------------------------
 # This must be the first Streamlit command in the script
 st.set_page_config(page_title="AI Clinical Intelligence Hub", layout="wide")
 
 # ---------------------- Initialize External Clients ---------------------------
-# Initialize Groq Client with API Key from environment variables
-GROQ_API_ENDPOINT = os.getenv("GROQ_API_ENDPOINT")
-GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+# Initialize OpenAI API details from environment variables
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 PUB_EMAIL = os.getenv("PUB_EMAIL", "")
 
-if not GROQ_API_ENDPOINT or not GROQ_API_KEY:
-    st.error("Groq API endpoint and key must be set as environment variables.")
+if not OPENAI_API_KEY:
+    st.error("OpenAI API key must be set as an environment variable (OPENAI_API_KEY).")
     st.stop()
 
-headers = {
-    "Authorization": f"Bearer {GROQ_API_KEY}",
-    "Content-Type": "application/json"
-}
+openai.api_key = OPENAI_API_KEY
 
 # Load spaCy model with error handling
 try:
@@ -119,6 +124,7 @@ class AdvancedEDA(DataAnalyzer):
             }
             return analysis
         except Exception as e:
+            logger.error(f"EDA Failed: {str(e)}")
             return {"error": f"EDA Failed: {str(e)}"}
 
 class DistributionVisualizer(DataAnalyzer):
@@ -139,6 +145,7 @@ class DistributionVisualizer(DataAnalyzer):
             plt.close()
             return base64.b64encode(buf.getvalue()).decode()
         except Exception as e:
+            logger.error(f"Visualization Error: {str(e)}")
             return f"Visualization Error: {str(e)}"
 
 class TemporalAnalyzer(DataAnalyzer):
@@ -167,6 +174,7 @@ class TemporalAnalyzer(DataAnalyzer):
                 "visualization": plot_data
             }
         except Exception as e:
+            logger.error(f"Temporal Analysis Failed: {str(e)}")
             return {"error": f"Temporal Analysis Failed: {str(e)}"}
 
 class HypothesisTester(DataAnalyzer):
@@ -197,6 +205,7 @@ class HypothesisTester(DataAnalyzer):
                 "interpretation": self.interpret_p_value(p)
             }
         except Exception as e:
+            logger.error(f"Hypothesis Testing Failed: {str(e)}")
             return {"error": f"Hypothesis Testing Failed: {str(e)}"}
 
     @staticmethod
@@ -206,7 +215,8 @@ class HypothesisTester(DataAnalyzer):
             mean_diff = abs(x.mean() - y.mean())
             pooled_std = np.sqrt((x.var() + y.var()) / 2)
             return mean_diff / pooled_std
-        except Exception:
+        except Exception as e:
+            logger.error(f"Error calculating Cohen's d: {str(e)}")
             return None
 
     @staticmethod
@@ -242,6 +252,7 @@ class LogisticRegressionTrainer(DataAnalyzer):
                 "model_params": model.get_params()
             }
         except Exception as e:
+            logger.error(f"Logistic Regression Model Error: {str(e)}")
             return {"error": f"Logistic Regression Model Error: {str(e)}"}
 
 # ---------------------- Business Logic Layer ---------------------------
@@ -273,6 +284,7 @@ class ClinicalRulesEngine:
                     "severity": rule.severity if rule_matched else None
                 }
             except Exception as e:
+                logger.error(f"Error executing rule '{rule_name}': {str(e)}")
                 results[rule_name] = {
                     "rule_matched": False,
                     "error": str(e),
@@ -298,6 +310,7 @@ class ClinicalRulesEngine:
                     raise ValueError(f"Unsupported expression: {expr}")
             return eval(compile(node, '<string>', mode='eval'), {"__builtins__": None}, variables)
         except Exception as e:
+            logger.error(f"safe_eval error: {str(e)}")
             raise ValueError(f"Invalid expression: {e}")
 
 class ClinicalKPI(BaseModel):
@@ -327,6 +340,7 @@ class ClinicalKPIMonitoring:
                     "status": status
                 }
             except Exception as e:
+                logger.error(f"Error calculating KPI '{kpi_name}': {str(e)}")
                 results[kpi_name] = {"error": str(e)}
         return results
 
@@ -357,6 +371,7 @@ class ClinicalKPIMonitoring:
                     raise ValueError(f"Unsupported expression: {expr}")
             return eval(compile(node, '<string>', mode='eval'), {"__builtins__": None}, variables)
         except Exception as e:
+            logger.error(f"safe_eval error: {str(e)}")
             raise ValueError(f"Invalid expression: {e}")
 
 class DiagnosisSupport(ABC):
@@ -397,6 +412,7 @@ class SimpleDiagnosis(DiagnosisSupport):
                     diagnosis_key: [f"Diagnosis failed: {result.get('error', 'Unknown error')}"]
                 })
         except Exception as e:
+            logger.error(f"Error during diagnosis: {str(e)}")
             return pd.DataFrame({
                 diagnosis_key: [f"Error during diagnosis: {e}"]
             })
@@ -425,6 +441,7 @@ class BasicTreatmentRecommendation(TreatmentRecommendation):
         **kwargs
     ) -> pd.DataFrame:
         if condition_col not in data.columns or treatment_col not in data.columns:
+            logger.warning(f"Condition or Treatment columns not found: {condition_col}, {treatment_col}")
             return pd.DataFrame({
                 recommendation_key: ["Condition or Treatment columns not found!"]
             })
@@ -448,61 +465,61 @@ class MedicalKnowledgeBase(ABC):
         pass
 
 class SimpleMedicalKnowledge(MedicalKnowledgeBase):
-    """Enhanced Medical Knowledge Class using Groq API."""
+    """Enhanced Medical Knowledge Class using OpenAI GPT-4."""
     def __init__(self):
-        self.api_endpoint = GROQ_API_ENDPOINT
-        self.api_key = GROQ_API_KEY
-        self.pub_email = PUB_EMAIL
-
-        self.headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
-        }
-        
-        # Initialize spaCy model for entity recognition if needed
-        try:
-            self.nlp = spacy.load("en_core_web_sm")
-        except OSError:
-            import subprocess
-            import sys
-            subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_sm"])
-            self.nlp = spacy.load("en_core_web_sm")
+        self.nlp = nlp  # Using the loaded spaCy model
 
     def search_medical_info(self, query: str, pub_email: str = "") -> str:
         """
-        Uses the Groq API to fetch medical information based on the user's query.
+        Uses OpenAI's GPT-4 to fetch medical information based on the user's query.
         """
+        logger.info(f"Received medical query: {query}")
         try:
-            # Preprocess the query if necessary (e.g., entity recognition)
+            # Preprocess the query (e.g., entity recognition)
             doc = self.nlp(query.lower())
             entities = [ent.text for ent in doc.ents]
-            if entities:
-                processed_query = " ".join(entities)
-            else:
-                processed_query = query.lower()
+            processed_query = " ".join(entities) if entities else query.lower()
+
+            logger.info(f"Processed query: {processed_query}")
+
+            # Create a prompt for GPT-4
+            prompt = f"""
+            You are a medical assistant. Provide a comprehensive and accurate response to the following medical query:
+
+            Query: {processed_query}
+
+            Please ensure the information is clear, concise, and evidence-based.
+            """
+
+            # Make the API request to OpenAI GPT-4
+            response = openai.ChatCompletion.create(
+                model="gpt-4",
+                messages=[
+                    {"role": "system", "content": "You are a helpful medical assistant."},
+                    {"role": "user", "content": prompt}
+                ],
+                max_tokens=500,
+                n=1,
+                stop=None,
+                temperature=0.7,
+            )
 
-            # Prepare the payload for the Groq API
-            payload = {
-                "query": processed_query,
-                "context": "medical"  # Assuming the API can handle context specification
-            }
+            # Extract the answer from the response
+            answer = response.choices[0].message['content'].strip()
 
-            # Make the API request
-            response = requests.post(
-                self.api_endpoint,
-                headers=self.headers,
-                data=json.dumps(payload)
-            )
+            logger.info("Successfully retrieved data from OpenAI GPT-4.")
 
-            if response.status_code == 200:
-                data = response.json()
-                answer = data.get("answer", "I'm sorry, I couldn't find relevant information.")
-                pubmed_abstract = self.fetch_pubmed_abstract(processed_query, pub_email)
-                return f"**Based on your query:** {answer}\n\n**PubMed Abstract:**\n\n{pubmed_abstract}"
-            else:
-                return f"Error: Received status code {response.status_code} from Groq API."
+            # Fetch PubMed abstract related to the query
+            pubmed_abstract = self.fetch_pubmed_abstract(processed_query, pub_email)
 
+            # Format the response
+            return f"**Based on your query:** {answer}\n\n**PubMed Abstract:**\n\n{pubmed_abstract}"
+
+        except openai.error.OpenAIError as e:
+            logger.error(f"OpenAI API Error: {str(e)}")
+            return f"OpenAI API Error: {str(e)}"
         except Exception as e:
+            logger.error(f"Medical Knowledge Search Failed: {str(e)}")
             return f"Medical Knowledge Search Failed: {str(e)}"
 
     def fetch_pubmed_abstract(self, query: str, email: str) -> str:
@@ -510,18 +527,27 @@ class SimpleMedicalKnowledge(MedicalKnowledgeBase):
         Searches PubMed for abstracts related to the query.
         """
         try:
+            if not email:
+                logger.warning("PubMed abstract retrieval skipped: Email not provided.")
+                return "No PubMed abstract available: Email not provided."
+
             Entrez.email = email
             handle = Entrez.esearch(db="pubmed", term=query, retmax=1, sort='relevance')
             record = Entrez.read(handle)
             handle.close()
+            logger.info(f"PubMed search for query '{query}' returned IDs: {record['IdList']}")
+
             if record["IdList"]:
                 handle = Entrez.efetch(db="pubmed", id=record["IdList"][0], rettype="abstract", retmode="text")
                 abstract = handle.read()
                 handle.close()
+                logger.info(f"Fetched PubMed abstract for ID {record['IdList'][0]}")
                 return abstract
             else:
+                logger.info(f"No PubMed abstracts found for query '{query}'.")
                 return "No abstracts found for this query on PubMed."
         except Exception as e:
+            logger.error(f"Error searching PubMed: {e}")
             return f"Error searching PubMed: {e}"
 
 # ---------------------- Forecasting Engine ---------------------------
@@ -559,8 +585,10 @@ class AutomatedInsights:
                 try:
                     results[name] = analyzer.invoke(data=data, **kwargs)
                 except Exception as e:
+                    logger.error(f"Error in analysis '{name}': {str(e)}")
                     results[name] = {"error": str(e)}
             else:
+                logger.warning(f"Analysis '{name}' not found.")
                 results[name] = {"error": "Analysis not found"}
         return results