initial commit

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+vectors_data/med_data_vec/index.faiss filter=lfs diff=lfs merge=lfs -text
+vectors_data/msd_data_vec/index.faiss filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,9 @@
+# Python
+__pycache__/
+.env
+*.env
+chatbot/
+processed_data/
+src/
+vectors_data/
+logs/

app.py

@@ -0,0 +1,202 @@
+import logging
+import asyncio
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Optional
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, constr, Field, validator, constr
+from rag import generate_answer  # Importing the function to generate answers
+
+# Create logs directory if it doesn't exist
+Path("logs").mkdir(exist_ok=True)  # Ensure logs directory is available for logging
+
+# Initialize FastAPI app with metadata
+app = FastAPI(
+    title="Question Answering API",  # Title of the API
+    description="API for generating answers using RAG",  # Description of the API
+    version="1.0.0",  # Version of the API
+    docs_url="/docs",  # URL for API documentation
+    redoc_url="/redoc"  # URL for ReDoc documentation
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allow all origins; modify in production for security
+    allow_credentials=True,  # Allow credentials to be included in requests
+    allow_methods=["*"],  # Allow all HTTP methods
+    allow_headers=["*"],  # Allow all headers
+)
+
+# Configure detailed logging
+logging.basicConfig(
+    level=logging.INFO,  # Set logging level to INFO
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',  # Log format
+    handlers=[  # Handlers for logging output
+        logging.StreamHandler(),  # Log to console
+        logging.FileHandler(f'logs/app_{datetime.now().strftime("%Y%m%d")}.log')  # Log to file
+    ]
+)
+logger = logging.getLogger(__name__)  # Create a logger for this module
+
+def generate_request_id() -> str:
+    """Generate a unique request ID."""  # Docstring for function
+    return str(uuid.uuid4())  # Return a new UUID as a string
+
+class QuestionRequest(BaseModel):
+    """Request model for question answering endpoint."""  # Docstring for model
+    question: constr(min_length=1) = Field(  # Field for the question with validation
+        ...,
+        description="The question to be answered",  # Description of the field
+        example="What is the capital of France?"  # Example question
+    )
+    
+    @validator('question')  # Validator for the question field
+    def validate_question(cls, v):
+        if v.strip() == "":  # Check if the question is empty
+            raise HTTPException(
+                status_code=500,  # Raise HTTP 500 if empty
+                detail="Error processing request"  # Error message
+            )
+        return v.strip()  # Return the trimmed question
+
+class ErrorResponse(BaseModel):
+    """Standard error response model."""  # Docstring for error response model
+    detail: str  # Detail of the error
+    request_id: Optional[str] = None  # Optional request ID for tracking
+
+@app.middleware("http")
+async def add_request_id(request: Request, call_next):
+    """Middleware to add request ID to all requests."""  # Docstring for middleware
+    request_id = generate_request_id()  # Generate a new request ID
+    request.state.request_id = request_id  # Store request ID in request state
+    response = await call_next(request)  # Process the request
+    response.headers["X-Request-ID"] = request_id  # Add request ID to response headers
+    return response  # Return the response
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException):
+    """Custom exception handler to include request ID."""  # Docstring for exception handler
+    return JSONResponse(
+        status_code=exc.status_code,  # Return the status code from the exception
+        content={"detail": exc.detail, "request_id": request.state.request_id}  # Include error details and request ID
+    )
+
+@app.post("/answer",
+    response_model=Dict[str, str],  # Response model for the endpoint
+    summary="Generate an answer for the given question",  # Summary of the endpoint
+    response_description="Returns the generated answer",  # Description of the response
+    responses={  # Possible responses
+        400: {"model": ErrorResponse},  # Bad request response
+        500: {"model": ErrorResponse},  # Internal server error response
+        504: {"model": ErrorResponse}   # Gateway timeout response
+    }
+)
+async def get_answer(request: Request, question_request: QuestionRequest) -> Dict[str, str]:
+    """
+    Generate an answer for the given question.
+    
+    Args:
+        request: FastAPI request object  # Description of request argument
+        question_request: The question request model  # Description of question_request argument
+        
+    Returns:
+        Dict containing the generated answer  # Description of return value
+        
+    Raises:
+        HTTPException: For various error conditions  # Description of possible exceptions
+    """
+    request_id = request.state.request_id  # Retrieve request ID from state
+    logger.info("Processing request %s - Question: %s", request_id, question_request.question)  # Log the processing request
+    
+    try:
+        # Additional validation
+        if len(question_request.question) > 1000:  # Check if question exceeds max length
+            logger.warning("Request %s: Question exceeds maximum length", request_id)  # Log warning
+            raise HTTPException(
+                status_code=400,  # Raise HTTP 400 for bad request
+                detail="Question length exceeds maximum allowed characters (1000)"  # Error message
+            )
+
+        # Convert generate_answer to async if it's not already
+        async def async_generate_answer(question: str):
+            loop = asyncio.get_event_loop()  # Get the current event loop
+            return await loop.run_in_executor(None, generate_answer, question)  # Run the generate_answer function in executor
+
+        # Generate answer with timeout handling
+        answer = await asyncio.wait_for(
+            async_generate_answer(question_request.question),  # Await the answer generation
+            timeout=120.0  # 120 second timeout
+        )
+        
+        logger.info("Request %s: Successfully generated answer", request_id)  # Log successful answer generation
+        return {"answer": answer}  # Return the generated answer
+
+    except asyncio.TimeoutError:  # Handle timeout errors
+        logger.error("Request %s: Generation timeout", request_id)  # Log timeout error
+        raise HTTPException(
+            status_code=504,  # Raise HTTP 504 for timeout
+            detail="Answer generation timed out"  # Error message
+        )
+        
+    except ValueError as ve:  # Handle value errors
+        logger.error("Request %s: Validation error - %s", request_id, str(ve))  # Log validation error
+        raise HTTPException(
+            status_code=400,  # Raise HTTP 400 for bad request
+            detail=str(ve)  # Return the validation error message
+        )
+    
+    except Exception as e:  # Handle all other exceptions
+        logger.error(
+            "Request %s: Unexpected error - %s",
+            request_id,
+            str(e),
+            exc_info=True  # Include stack trace in logs
+        )
+        raise HTTPException(
+            status_code=500,  # Raise HTTP 500 for internal server error
+            detail="Internal server error occurred. Please try again later."  # Error message
+        )
+
+@app.get("/health",
+    summary="Health check endpoint",  # Summary of the health check endpoint
+    response_description="Returns the API health status"  # Description of the response
+)
+async def health_check():
+    """Health check endpoint to verify API is running."""  # Docstring for health check
+    return {"status": "healthy"}  # Return health status
+
+@app.on_event("startup")
+async def startup_event():
+    """Initialize any resources on startup."""  # Docstring for startup event
+    logger.info("Starting up Question Answering API")  # Log startup event
+    # Add any initialization code here (e.g., loading models)
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    """Cleanup any resources on shutdown."""  # Docstring for shutdown event
+    logger.info("Shutting down Question Answering API")  # Log shutdown event
+    # Add cleanup code here
+
+if __name__ == "__main__":  # Entry point for the application
+    import uvicorn  # Import Uvicorn for running the app
+    
+    # Load configuration from environment variables or use defaults
+    host = "0.0.0.0"  # Host address
+    port = 8000  # Port number
+    
+    logger.info(f"Starting server on {host}:{port}")  # Log server start
+    
+    uvicorn.run(
+        "app:app",  # Application to run
+        host=host,  # Host address
+        port=port,  # Port number
+        reload=False,  # Disable auto-reload in production
+        log_level="info",  # Set log level
+        access_log=True,  # Enable access logging
+        workers=1  # Number of worker processes
+    )

gradio_app.py

@@ -0,0 +1,38 @@
+import os
+import gradio as gr
+from rag import generate_answer  # Import the generate_answer function from rag.py
+
+def chat_with_bot(query: str) -> tuple:
+    """
+    Chat with the bot using the provided query.
+    
+    Args:
+        query: User's question
+        
+    Returns:
+        tuple: Generated answer and extracted contents
+    """
+    response = generate_answer(query)  # Call the existing generate_answer function
+    
+    return response
+
+def main():
+    """
+    Main function to create and launch the Gradio interface.
+    """
+    try:
+        # Create Gradio interface
+        interface = gr.Interface(
+            fn=chat_with_bot,  # Use the chat function
+            inputs="text",  # Input type
+            outputs="text",  # Output type
+            title="RAG-LLM based Medical Chatbot",  # Title of the app
+            description="Ask your medical questions and get answers from the chatbot."  # Description
+        )
+        interface.launch(share=True)  # Launch the Gradio app
+        
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+
+if __name__ == "__main__":
+    main()

log.txt

@@ -0,0 +1,68 @@
+Query: who is john cena?
+Response: I'm sorry, but based on the provided context, there is no information related to "John Cena" or any medical relevance to that name. If you have any other medical questions or concerns, please feel free to ask. Remember, for personalized medical advice, it is always best to consult with a healthcare professional.
+
+Query: i have headache, suggest me some medicine
+Response: Based on the context provided, if you have a headache, you can consider taking Ibuprofen EG syrup suspension. However, it is important to consult with a healthcare professional before taking any medication, especially if you have other underlying health conditions or are taking other medications. Ibuprofen may cause side effects such as injection site reaction, tenderness, fatigue, myalgia, arthralgia, fever, and nausea. If your headache persists or worsens, it is advisable to seek medical advice for proper evaluation and treatment.
+
+Query: what is the best medicine when i have a headache?
+Response: The best medicine for a headache can vary depending on the individual and the underlying cause of the headache. In general, over-the-counter pain relievers such as acetaminophen (oral or IV), ibuprofen, or naproxen can be effective for mild to moderate headaches. However, it is important to consult with a healthcare professional before taking any medication, especially if you are immunosuppressed or have papilledema. Additionally, it is important to avoid overusing medications, as this can lead to medication overuse headaches. In some cases, cognitive therapy, biofeedback, and education can also be helpful in managing headaches. If you have frequent headaches, it is recommended to seek medical advice for proper diagnosis and treatment.
+
+Query: I have Diarrhea, suggest some medicine
+Response: Based on the provided context, it is important to first consult a healthcare professional for a proper diagnosis and treatment plan. However, if you are experiencing diarrhea, you may consider over-the-counter medications such as loperamide (Imodium) to help alleviate symptoms. It is important to follow the recommended dosage and consult a healthcare provider if symptoms persist or worsen. Additionally, it is important to stay hydrated by drinking plenty of fluids.
+
+Query: What are the clinical signs and symptoms of mitral stenosis?
+Response: The clinical signs and symptoms of mitral stenosis include those of heart failure, such as fatigue, cold skin, and right upper quadrant abdominal discomfort. Additionally, signs may include an opening snap and a diastolic murmur. It is important to consult a healthcare professional for proper diagnosis and treatment. Medications such as diuretics and beta-blockers may be prescribed, but potential side effects should be discussed with a healthcare provider.
+
+Query: How does dehydration affect blood pressure in someone with autonomic dysfunction?
+Response: Dehydration can lead to hypotension (low blood pressure) in someone with autonomic dysfunction. This is because autonomic dysfunction can already cause orthostatic hypotension, and dehydration further exacerbates this by reducing blood volume and affecting the body's ability to regulate blood pressure. It is important for individuals with autonomic dysfunction to stay well-hydrated to help maintain their blood pressure within a healthy range. However, for personalized advice and treatment options, it is recommended to consult a healthcare professional.
+
+Query: Can an ear infection cause a heart problem? If yes, how?
+Response: Yes, in heavy infections, the inflammation from an ear infection may cause complications such as cardiac issues like myocarditis, heart failure, arrhythmia, and even death. The inflammation can spread to the heart and lead to serious complications. It is important to seek medical attention if you suspect an ear infection to prevent any potential complications. Consulting a healthcare professional for proper diagnosis and treatment is recommended.
+
+Query: When do VPBs require treatment?
+Response: VPBs, or ventricular premature beats, may require treatment if they are causing symptoms such as palpitations, dizziness, or fainting. Treatment options may include medications such as antiarrhythmics. It is important to consult with a healthcare professional for proper evaluation and management of VPBs.
+
+Query: When do VPBs require treatment?
+Response: VPBs, or ventricular premature beats, may require treatment if they are causing symptoms such as palpitations, dizziness, or fainting. Treatment options may include medications such as quetiapine or olanzapine, but it is important to consult a healthcare professional for proper evaluation and management. In some cases, preventive treatment may be indicated for patients with certain medical conditions that require bed rest. If you are experiencing symptoms of VPBs, it is important to seek medical advice for appropriate care.
+
+Query: How does microvascular angina differ from typical angina?
+Response: Microvascular angina differs from typical angina in that it is caused by impaired coronary blood flow due to issues in the smaller blood vessels, rather than blockages in the larger epicardial coronary arteries. This can lead to symptoms such as angina pectoris, syncope, or arrhythmias even in the absence of coronary artery disease. To diagnose microvascular angina, guidewires with pressure or flow sensors can be used to estimate blood flow across stenoses, with a fractional flow reserve (FFR) of < 0.75 to 0.8 indicating myocardial ischemia. Treatment for microvascular angina may involve medications like Lipiodol Ultra Fluide or Solu-Medrol, but it is important to consult a healthcare professional for proper diagnosis and management, as well as to discuss potential side effects of these medications.
+
+Query: What are the common causes of pericarditis?
+Response: Common causes of pericarditis include systemic rheumatic diseases such as rheumatoid arthritis, systemic lupus erythematosus (SLE), and systemic sclerosis. Bacterial infections from gram-negative bacilli, Haemophilus influenzae, Lyme disease, rheumatic fever, staphylococci, and streptococci can also lead to pericarditis. Additionally, tuberculosis, certain cancers like breast cancer, Kaposi sarcoma (in people with HIV infection), leukemia, and lung cancer, as well as certain medications like anticoagulants, can be common causes of pericarditis. It is important to consult a healthcare professional for proper diagnosis and treatment.
+
+Query: suggest some medicine for pericarditis
+Response: For pericarditis, medications such as colchicine or interleukin-1 antagonists may be prescribed based on the underlying cause and severity of the condition. It is important to consult a healthcare professional for a proper diagnosis and treatment plan. Additionally, for patients with newly diagnosed constrictive pericarditis, anti-inflammatory medications may be considered before pericardiectomy. It is crucial to follow the guidance of a healthcare provider and discuss any potential side effects of the prescribed medication.
+
+Query: What are the common causes of orthostatic hypotension?
+Response: Common causes of orthostatic hypotension include dehydration, electrolyte disturbances, use of certain drugs (such as antihypertensives and nitrates), aging, heart failure, chronic kidney disease, respiratory disorders, liver failure, renal insufficiency, and sedatives or opioids used for intubation and ventilation. It is important to consult a healthcare professional for proper diagnosis and treatment, as the underlying cause may vary from person to person.
+
+Query: How is long QT syndrome diagnosed and treated?'
+Response: Long QT syndrome is diagnosed based on clinical criteria. Treatment typically involves supportive care, including medications as needed for associated symptoms. It is important to consult with a healthcare professional for proper diagnosis and treatment. If medications are prescribed, it is essential to be aware of potential side effects.
+
+Query: What are the classes of antiarrhythmic drugs and their mechanisms of action?
+Response: The classes of antiarrhythmic drugs are grouped into four main classes based on their dominant cellular electrophysiologic effect. Class I drugs are sodium channel blockers that slow conduction in fast-channel tissues. They are subdivided into subclasses A, B, and C based on their kinetics of the sodium channel effects. Class Ib drugs have fast kinetics. The effects of these drugs may be reflected in the electrocardiogram as widening of the P wave, widening of the QRS complex, prolongation of the PR interval, or a combination. It is important to consult a healthcare professional for specific information on antiarrhythmic drugs and their mechanisms of action.
+
+Query: What are the key causes of pitting and non-pitting edema?
+Response: The key causes of pitting and non-pitting edema can include injury (such as burns, chemicals, toxins, or blunt trauma), severe sepsis causing vascular endothelial leakage, soft-tissue infections like cellulitis or necrotizing myofasciitis, and other inflammatory conditions. Pitting edema may present as painless, symmetrical swelling, while non-pitting edema may be focal and erythematous. It is important to seek clinical evaluation, cultures, and imaging studies as needed to determine the underlying cause of edema. For specific treatment options and management, it is advisable to consult a healthcare professional.
+
+Query: tell me about this medicine: Rhinathiol Antirhinitis 10/4 compr.
+Response: Rhinathiol Antirhinitis 10/4 compr. is a medication in the form of tablets. It is indicated for the treatment of rhinitis, which is inflammation of the mucous membrane of the nose. This medication may help reduce symptoms such as nasal congestion, runny nose, and sneezing associated with rhinitis. 
+
+It is important to note that this information is general and for educational purposes only. It is always recommended to consult with a healthcare professional or pharmacist before starting any new medication. Additionally, common side effects of Rhinathiol Antirhinitis 10/4 compr. may include drowsiness, dizziness, dry mouth, and blurred vision. If you experience any severe or persistent side effects, it is important to seek medical attention.
+
+Query: What are the first-line treatments for type 2 diabetes, and how do they differ in mechanism?
+Response: The first-line treatments for type 2 diabetes include oral antihyperglycemics, non-insulin injectable medications such as glucagon-like peptide-1 (GLP-1) receptor agonists, insulin, or a combination of these. These medications work by helping to lower blood sugar levels in different ways. Oral antihyperglycemics help improve insulin sensitivity or reduce glucose production in the liver. GLP-1 receptor agonists stimulate insulin secretion and reduce glucagon secretion. Insulin helps regulate blood sugar levels by replacing or supplementing the body's natural insulin production. It is important to consult healthcare professionals for personalized treatment recommendations and to discuss potential side effects of these medications.
+
+Query: What are the first-line treatments for type 2 diabetes, and how do they differ in mechanism?
+Response: The first-line treatments for type 2 diabetes include oral antihyperglycemics, non-insulin injectable medications such as glucagon-like peptide-1 (GLP-1) receptor agonists, insulin, or a combination of these. These medications work by helping to lower blood sugar levels in different ways. Oral antihyperglycemics help improve insulin sensitivity or reduce glucose production in the liver. GLP-1 receptor agonists help increase insulin secretion and decrease glucagon secretion. Insulin therapy helps regulate blood sugar levels by replacing or supplementing the body's insulin production. It is important to consult healthcare professionals for personalized treatment recommendations and to discuss potential side effects of these medications.
+
+Query: What are the first-line treatments for type 2 diabetes, and how do they differ in mechanism?
+Response: The first-line treatments for type 2 diabetes include oral antihyperglycemics, non-insulin injectable medications such as glucagon-like peptide-1 (GLP-1) receptor agonists, insulin, or a combination of these. These medications work by helping to lower blood sugar levels in different ways. Oral antihyperglycemics help improve insulin sensitivity or reduce glucose production in the liver. GLP-1 receptor agonists help increase insulin secretion and decrease glucagon secretion. Insulin therapy helps regulate blood sugar levels by replacing or supplementing the body's insulin production. It is important to consult healthcare professionals for personalized treatment plans and to discuss potential side effects of these medications.
+
+Query: who is John Cena?
+Response: I'm sorry, but based on the provided context, there is no information related to John Cena. If you have any other medical questions or concerns, please feel free to ask. It is always recommended to consult with a healthcare professional for personalized medical advice.
+
+Query: who is John Cena?
+Response: I'm sorry, but based on the provided context, there is no information related to a person named John Cena. If you are looking for information on a specific individual, it is recommended to consult reliable sources or conduct a separate search. If you have any medical questions or concerns, it is always best to consult a healthcare professional for personalized advice and guidance.
+

preprocess.py

@@ -0,0 +1,193 @@
+import logging
+from pathlib import Path
+from typing import Union, Optional
+import pandas as pd
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class DataPreprocessor:
+    """A class to handle data preprocessing operations for different file formats."""
+    
+    @staticmethod
+    def _preprocess_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Applies standard preprocessing steps to a DataFrame.
+        
+        Args:
+            df (pd.DataFrame): Input DataFrame to preprocess
+            
+        Returns:
+            pd.DataFrame: Preprocessed DataFrame
+        """
+        try:
+            # Convert text columns to lowercase for standardization
+            df = df.map(lambda x: x.lower() if isinstance(x, str) else x)
+            
+            # Drop columns that are fully null
+            df = df.dropna(axis=1, how='all')
+            
+            # Fill remaining NaN values with empty strings
+            df = df.fillna('')
+            
+            # Remove duplicate rows
+            df = df.drop_duplicates()
+            
+            return df
+            
+        except Exception as e:
+            logger.error(f"Error during DataFrame preprocessing: {str(e)}")
+            raise
+
+    @classmethod
+    def preprocess_msd(cls, 
+                      file_path: Union[str, Path], 
+                      output_path: Union[str, Path],
+                      sheet_name: Optional[Union[str, int]] = 0) -> pd.DataFrame:
+        """
+        Preprocesses an MSD Excel file and saves the result.
+        
+        Args:
+            file_path: Path to the Excel file
+            output_path: Directory path for the output file
+            sheet_name: Sheet name or index to load (default: 0)
+            
+        Returns:
+            pd.DataFrame: Preprocessed DataFrame
+            
+        Raises:
+            FileNotFoundError: If input file doesn't exist
+            PermissionError: If output directory is not writable
+        """
+        try:
+            # Convert to Path objects
+            file_path = Path(file_path)
+            output_path = Path(output_path)
+            
+            # Validate input file
+            if not file_path.exists():
+                raise FileNotFoundError(f"Input file not found: {file_path}")
+            
+            # Ensure output directory exists
+            output_path.mkdir(parents=True, exist_ok=True)
+            
+            logger.info(f"Processing MSD file: {file_path}")
+            df = pd.read_excel(file_path, sheet_name=sheet_name)
+            
+            # Apply preprocessing
+            df = cls._preprocess_dataframe(df)
+            
+            # Save processed file
+            output_file = output_path / "msd_processed.csv"
+            df.to_csv(output_file, index=False)
+            logger.info(f"Saved processed file to: {output_file}")
+            
+            return df
+            
+        except Exception as e:
+            logger.error(f"Error processing MSD file: {str(e)}")
+            raise
+
+    @classmethod
+    def preprocess_cbip(cls,
+                       input_dir: Union[str, Path],
+                       output_dir: Union[str, Path]) -> None:
+        """
+        Preprocesses all CSV files in the CBIP directory.
+        
+        Args:
+            input_dir: Directory containing input CSV files
+            output_dir: Directory for output files
+            
+        Raises:
+            FileNotFoundError: If input directory doesn't exist
+            PermissionError: If output directory is not writable
+        """
+        try:
+            # Convert to Path objects
+            input_dir = Path(input_dir)
+            output_dir = Path(output_dir)
+            
+            # Validate input directory
+            if not input_dir.exists():
+                raise FileNotFoundError(f"Input directory not found: {input_dir}")
+            
+            # Ensure output directory exists
+            output_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Process all CSV files
+            csv_files = list(input_dir.rglob("*.csv"))
+            if not csv_files:
+                logger.warning(f"No CSV files found in: {input_dir}")
+                return
+            
+            for file_path in csv_files:
+                try:
+                    logger.info(f"Processing CBIP file: {file_path}")
+                    
+                    # Read CSV file
+                    df = pd.read_csv(
+                        file_path,
+                        delimiter=';',
+                        quotechar='"',
+                        skip_blank_lines=True
+                    )
+                    
+                    # Apply preprocessing
+                    df = cls._preprocess_dataframe(df)
+                    
+                    # Save processed file
+                    output_file = output_dir / file_path.name
+                    df.to_csv(output_file, index=False)
+                    logger.info(f"Saved processed file to: {output_file}")
+                    
+                except Exception as e:
+                    logger.error(f"Error processing {file_path}: {str(e)}")
+                    continue
+                    
+        except Exception as e:
+            logger.error(f"Error processing CBIP directory: {str(e)}")
+            raise
+
+def main():
+    """Main execution function."""
+    try:
+        import os
+        import argparse
+        from pathlib import Path
+        
+        # Create processed_data directory in current working directory
+        output_base = Path.cwd() / "processed_data"
+        msd_output = output_base / "msd"
+        cbip_output = output_base / "cbip"
+        
+        parser = argparse.ArgumentParser(description='Process MSD and CBIP data files.')
+        parser.add_argument('--msd-input', required=True, help='Path to MSD Excel file')
+        parser.add_argument('--cbip-input', required=True, help='Input directory containing CBIP CSV files')
+        
+        args = parser.parse_args()
+        
+        preprocessor = DataPreprocessor()
+        
+        # Process MSD file
+        preprocessor.preprocess_msd(
+            args.msd_input,
+            msd_output
+        )
+        
+        # Process CBIP directory
+        preprocessor.preprocess_cbip(
+            args.cbip_input,
+            cbip_output
+        )
+        
+    except Exception as e:
+        logger.error(f"Main execution failed: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    main()

rag.py

@@ -0,0 +1,171 @@
+import os
+import logging
+from pathlib import Path
+from typing import Optional
+from dotenv import load_dotenv
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_openai import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain.retrievers import MergerRetriever
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+def load_faiss_index(folder_path: str, model_name: str) -> Optional[FAISS]:
+    """
+    Load a FAISS index with a specific embedding model.
+    
+    Args:
+        folder_path: Path to the FAISS index folder
+        model_name: Name of the HuggingFace embedding model
+    
+    Returns:
+        FAISS: Loaded FAISS index object
+        
+    Raises:
+        ValueError: If the folder path doesn't exist
+    """
+    try:
+        if not os.path.exists(folder_path):
+            raise ValueError(f"FAISS index folder not found: {folder_path}")
+            
+        logger.info(f"Loading FAISS index from {folder_path}")
+        embeddings = HuggingFaceEmbeddings(model_name=model_name)
+        return FAISS.load_local(
+            folder_path=folder_path,
+            embeddings=embeddings,
+            allow_dangerous_deserialization=True
+        )
+    except Exception as e:
+        logger.error(f"Error loading FAISS index: {str(e)}")
+        raise
+
+def generate_answer(query: str) -> str:
+    """
+    Generate an answer for the given query using RAG.
+    
+    Args:
+        query: User's question
+        
+    Returns:
+        str: Generated answer
+        
+    Raises:
+        ValueError: If query is empty or required files are missing
+    """
+    try:
+        if not query or not query.strip():
+            raise ValueError("Query cannot be empty")
+            
+        # Get the current directory and construct paths
+        current_dir = Path(__file__).parent
+        vectors_dir = current_dir / "vectors_data"
+        
+        # Validate vectors directory exists
+        if not vectors_dir.exists():
+            raise ValueError(f"Vectors directory not found at {vectors_dir}")
+            
+        # Load FAISS indices
+        logger.info("Loading FAISS indices...")
+        data_vec = load_faiss_index(
+            str(vectors_dir / "msd_data_vec"),
+            "sentence-transformers/all-MiniLM-L12-v2"
+        )
+        med_vec = load_faiss_index(
+            str(vectors_dir / "med_data_vec"),
+            "sentence-transformers/all-MiniLM-L12-v2"
+        )
+
+        # Create the LLM instance
+        llm = ChatOpenAI(
+            model="gpt-3.5-turbo",
+            temperature=0,
+            openai_api_key="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"
+        )
+
+        # Define the prompt template
+        template = """You are a helpful medical information assistant. Use the following pieces of context to answer the medical question at the end.
+        
+        Important notes:
+        - Base your answer strictly on the provided context and understandable for all readers
+        - If you don't know the answer, just say that you don't know
+        - Include relevant disclaimers about consulting healthcare professionals
+        - If suggesting medications (upon question), mention potential side effects if provided in the context
+        - Highlight if the information is general knowledge or requires professional medical advice
+        
+        Context: {context}
+        
+        Question: {question}
+        
+        Medical Information Assistant:"""
+
+        QA_CHAIN_PROMPT = PromptTemplate(
+            input_variables=["context", "question"], 
+            template=template
+        )
+
+        # Initialize and combine retrievers
+        logger.info("Setting up retrieval chain...")
+        data_retriever = data_vec.as_retriever()
+        med_retriever = med_vec.as_retriever()
+        combined_retriever = MergerRetriever(
+            retrievers=[data_retriever, med_retriever]
+        )
+
+        # Initialize the RetrievalQA chain
+        qa_chain = RetrievalQA.from_chain_type(
+            llm=llm,
+            retriever=combined_retriever,
+            return_source_documents=True,
+            chain_type_kwargs={"prompt": QA_CHAIN_PROMPT}
+        )
+
+        # Run the chain
+        logger.info("Generating answer...")
+        result = qa_chain.invoke({"query": query})
+        logger.info("Answer generated successfully")
+
+        # Extracting the relevant documents from the result
+        extracted_docs = result.get("source_documents", [])
+        logger.info(f"Extracted documents: {extracted_docs}")  # Log the extracted documents
+        
+        # New organized printing of extracted documents
+        print("\nExtracted Documents:")
+        for doc in extracted_docs:
+            print(f"Source: {doc.metadata['source']}, Row: {doc.metadata['row']}")
+            print(f"Content: {doc.page_content}\n")
+        
+        return result["result"]
+        
+    except Exception as e:
+        logger.error(f"Error generating answer: {str(e)}")
+        raise
+
+def main():
+    """
+    Main function to demonstrate the usage of the RAG system.
+    """
+    try:
+        # Example usage
+        query = "suggest me some medicine for bronchitis"
+        logger.info(f"Processing query: {query}")
+        
+        response = generate_answer(query)
+        print("\nQuery:", query)
+        print("\nResponse:", response)
+        
+    except Exception as e:
+        logger.error(f"Error in main function: {str(e)}")
+        print(f"An error occurred: {str(e)}")
+
+if __name__ == "__main__":
+    main()

readme.md

@@ -0,0 +1,235 @@
+# Building a Retrieval-Augmented Question-Answering System with FastAPI and LangChain
+Develop a simple question-answering web service that leverages Retrieval-Augmented Generation (RAG) to provide answers based on a set of provided documents. The service will be built using Python, FastAPI, and LangChain.
+
+## Installation
+
+1. Clone or download this repository
+```
+git clone https://github.com/shamim237/artech_med_bot.git
+```
+2. Create a virtual environment (optional but recommended):
+
+```
+python -m venv venv
+source venv/bin/activate  # On Windows use `venv\Scripts\activate`
+```
+3. Install the required packages:    
+```
+pip install -r requirements.txt
+```
+
+# Run the system
+
+To run the system on your machine, follow the steps shown below:
+
+## **preprocess.py**:
+This script for preprocessing MSD (Excel) and CBIP (CSV) data files. This script provides standardized data cleaning and transformation operations for both file formats.
+
+### Features
+- Processes MSD Excel files and CBIP CSV files
+- Standardizes text to lowercase
+- Removes empty columns
+- Handles missing values
+- Eliminates duplicate rows
+- Preserves original file structure
+- Comprehensive logging
+
+### Usage
+```
+python preprocess.py --msd-input "path/to/msd.xlsx" --cbip-input "path/to/cbip/directory"
+```
+#### Arguments
+- `--msd-input`: Path to the MSD Excel file
+- `--cbip-input`: Path to the directory containing CBIP CSV files
+
+
+#### Output
+
+The script creates a `processed_data` directory in your current working directory with the following structure:
+processed_data/
+├── msd/
+│ └── msd_processed.csv
+└── cbip/
+└── [processed_csv_files]
+
+### Data Processing Steps
+
+1. **Text Standardization**: Converts all text to lowercase
+2. **Column Cleaning**: Removes columns that are completely empty
+3. **Missing Value Handling**: Fills NaN values with empty strings
+4. **Duplicate Removal**: Removes duplicate rows from the dataset
+
+### Error Handling
+
+- The script includes comprehensive error handling and logging
+- Errors are logged with timestamps and detailed messages
+- Processing continues even if individual files fail
+
+
+## "vectorize.py":
+This script processes CSV documents and creates FAISS vector stores using LangChain and Hugging Face embeddings. It's designed to handle both MSD (Master Service Data) and medical data sources, converting them into efficient searchable vector representations.
+
+### Features
+- CSV document loading with support for multiple files
+- Text chunking with configurable size and overlap
+- FAISS vector store creation and persistence
+- Comprehensive error handling and logging
+- Support for Hugging Face embedding models
+
+### Configuration
+
+The script uses the following default configuration:
+
+- MSD Data Path: `./processed_data/msd/msd_processed.csv`
+- Medical CSV Path: `./processed_data/cbip/*.csv`
+- MSD Vector Output: `./vectors_data/msd_data_vec`
+- Medical Vector Output: `./vectors_data/med_data_vec`
+- Embedding Model: `sentence-transformers/all-MiniLM-L12-v2`
+
+### Usage
+- Just run the script to get default output
+```
+python -m vectorize.py
+```
+- or change paths of the dataset
+
+## "rag.py": 
+This script implements a Retrieval-Augmented Generation (RAG) system using LangChain, FAISS vector store, and OpenAI's GPT-3.5 model. The system combines medical and general data sources to provide informed answers to user queries.
+
+### Features
+
+- Dual vector store integration (medical and general data)
+- HuggingFace embeddings using `all-MiniLM-L12-v2` model
+- OpenAI GPT-3.5 for answer generation
+- Comprehensive error handling and logging
+- Environment variable support for API keys
+
+### Prerequisites
+
+- OpenAI API key
+    - Create a `.env` file in the project root and add your OpenAI API key: OPENAI_API_KEY=your_api_key_here
+- Required vector stores in the `vectors_data` directory:
+  - `msd_data_vec/` - General data vector store
+  - `med_data_vec/` - Medical data vector store
+
+### Usage
+```
+python rag.py
+```
+
+## "app.py":
+This script is a FastAPI-based REST API that generates answers to questions using RAG (Retrieval-Augmented Generation) technology.
+
+### Features
+
+- Question answering endpoint with RAG integration
+- Request ID tracking for all API calls
+- Comprehensive error handling and logging
+- Health check endpoint
+- CORS support
+- API documentation (Swagger UI and ReDoc)
+
+### Usage
+```
+uvicorn app:app --reload
+```
+
+The server will start on `http://localhost:8000`
+
+### API Endpoints
+
+#### 1. Question Answering
+- **Endpoint**: `/answer`
+- **Method**: POST
+- **Request Body**:
+```
+{
+"question": "What is an overactive bladder?"
+}
+```
+- **Response**:
+```
+{
+"answer": "The generated answer..."
+}
+```
+
+## test_rag.py:
+
+The test suite validates the functionality of:
+- Individual data retrievers (medicine and general data)
+- Combined retriever functionality
+- Answer generation system
+- Error handling for edge cases
+
+### Test Cases
+
+The test suite includes the following test cases:
+
+1. `test_data_retriever`: Tests retrieval from general data store
+2. `test_med_retriever`: Tests retrieval from medical data store
+3. `test_combined_retriever`: Tests the merged retriever functionality
+4. `test_generate_answer`: Validates answer generation
+5. `test_empty_query`: Tests error handling for invalid inputs
+
+### Usage
+```
+python -m unittest test_rag.py
+```
+
+### Vector Store Setup
+
+The system expects two FAISS indices in the `vectors/` directory:
+- `msd_data_vec`: General knowledge vector store
+- `med_data_vec`: Medical knowledge vector store
+
+Both indices use the `sentence-transformers/all-MiniLM-L12-v2` embedding model.
+
+#### Notes
+
+- Ensure all vector stores are properly initialized before running tests
+- The system uses the MiniLM-L12-v2 model for embeddings
+- Empty or whitespace-only queries will raise ValueError exceptions
+
+
+## test_app.py:
+
+The test suite (`test_app.py`) validates the `/answer` endpoint's response to different types of requests, ensuring proper handling of both valid and invalid inputs.
+
+### Test Cases
+
+The test suite includes the following test cases:
+
+1. **Valid Question Test**
+   - Verifies that the endpoint correctly processes a valid question
+   - Expects a 200 status code and an answer in the response
+
+2. **Empty Question Test**
+   - Validates handling of empty string inputs
+   - Expects a 422 status code (Pydantic validation error)
+
+3. **Whitespace Question Test**
+   - Checks handling of whitespace-only inputs
+   - Expects a 500 status code with an error message
+
+4. **Missing Question Field Test**
+   - Verifies behavior when the question field is omitted
+   - Expects a 422 status code (FastAPI validation error)
+
+5. **Invalid JSON Test**
+   - Tests handling of malformed JSON requests
+   - Expects a 422 status code (FastAPI validation error)
+
+### Usage
+```
+python -m unittest test_app.py
+```
+
+## Assumptions and Trade-offs:
+I generated and stored vector embeddings separately for disease/MSD data and medicine/CBIP data, believing that this separation would enhance the LLM's performance.
+
+## Comments:
+The quality of responses from this RAG-based LLM can be further strengthened through the following steps:
+- Organizing the disease-related dataset more systematically.
+- Structuring the medicine-related dataset more effectively.
+- Enhancing disease-treatment and drug recommendations through better-organized mappings.

requirements.txt

@@ -0,0 +1,13 @@
+pandas
+openpyxl 
+langchain
+langchain-community
+langchain-huggingface
+faiss-cpu
+sentence-transformers
+python-dotenv
+langchain-openai
+openai
+fastapi
+uvicorn
+pydantic

test_app.py

@@ -0,0 +1,35 @@
+import unittest
+from fastapi.testclient import TestClient
+from app import app
+
+class TestAnswerEndpoint(unittest.TestCase):
+    def setUp(self):
+        self.client = TestClient(app)
+
+    def test_valid_question(self):
+        response = self.client.post("/answer", json={"question": "What is an overactive bladder?"})
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("answer", response.json())
+
+    def test_empty_question(self):
+        # Empty string fails Pydantic validation (constr(min_length=1))
+        response = self.client.post("/answer", json={"question": ""})
+        self.assertEqual(response.status_code, 422)  # Pydantic validation error
+        self.assertIn("should have at least 1 character", response.json()["detail"][0]["msg"])
+
+    def test_whitespace_question(self):
+        # Whitespace-only string passes Pydantic but fails our custom validation
+        response = self.client.post("/answer", json={"question": "   "})
+        self.assertEqual(response.status_code, 500)  # Internal server error from our validation
+        self.assertEqual(response.json()["detail"], "Error processing request")
+
+    def test_missing_question_field(self):
+        response = self.client.post("/answer", json={})
+        self.assertEqual(response.status_code, 422)  # FastAPI validation error
+
+    def test_invalid_json(self):
+        response = self.client.post("/answer", data="invalid json")
+        self.assertEqual(response.status_code, 422)  # FastAPI validation error
+
+if __name__ == '__main__':
+    unittest.main()

test_rag.py

@@ -0,0 +1,69 @@
+import unittest
+from rag import load_faiss_index, generate_answer
+from langchain.retrievers import MergerRetriever
+
+class TestRetrieval(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Initialize FAISS indices 
+        cls.data_vec = load_faiss_index("./vectors_data/msd_data_vec", "sentence-transformers/all-MiniLM-L12-v2")
+        cls.med_vec = load_faiss_index("./vectors_data/med_data_vec", "sentence-transformers/all-MiniLM-L12-v2")
+        
+        # Initialize retrievers 
+        cls.data_retriever = cls.data_vec.as_retriever()
+        cls.med_retriever = cls.med_vec.as_retriever()
+        # Combine both retrievers into a single retriever
+        cls.combined_retriever = MergerRetriever(retrievers=[cls.data_retriever, cls.med_retriever])
+
+    def test_data_retriever(self):
+        # Test the data retriever with a specific query
+        query = "what are the symptoms of diabetes?"
+        docs = self.data_retriever.get_relevant_documents(query)
+        
+        # Assert that documents are returned and are not empty
+        self.assertIsNotNone(docs)
+        self.assertTrue(len(docs) > 0)
+        # Check if documents have content
+        self.assertTrue(all(doc.page_content.strip() != "" for doc in docs))
+
+    def test_med_retriever(self):
+        # Test the medical retriever with a specific query
+        query = "what are common antibiotics?"
+        docs = self.med_retriever.get_relevant_documents(query)
+        
+        # Assert that documents are returned and are not empty
+        self.assertIsNotNone(docs)
+        self.assertTrue(len(docs) > 0)
+        self.assertTrue(all(doc.page_content.strip() != "" for doc in docs))
+
+    def test_combined_retriever(self):
+        # Test the combined retriever with a specific query
+        query = "what is the treatment for high blood pressure?"
+        docs = self.combined_retriever.get_relevant_documents(query)
+        
+        # Assert that documents are returned and are not empty
+        self.assertIsNotNone(docs)
+        self.assertTrue(len(docs) > 0)
+        self.assertTrue(all(doc.page_content.strip() != "" for doc in docs))
+
+    def test_generate_answer(self):
+        # Test the answer generation function with a specific query
+        query = "what are the side effects of aspirin?"
+        response = generate_answer(query)
+        
+        # Assert that a valid response is returned
+        self.assertIsNotNone(response)
+        self.assertIsInstance(response, str)
+        self.assertTrue(len(response) > 0)
+
+def test_empty_query(self):
+    # Test the answer generation function with an empty query
+    with self.assertRaises(ValueError):  # More specific exception
+        generate_answer("")
+    
+    # Test the answer generation function with a whitespace-only query
+    with self.assertRaises(ValueError):
+        generate_answer("   ")  # Test whitespace-only query
+
+if __name__ == '__main__':
+    unittest.main()

vectorize.py

@@ -0,0 +1,149 @@
+import glob
+import logging
+from pathlib import Path
+from typing import List, Optional
+from langchain_community.vectorstores import FAISS
+from langchain_community.document_loaders import CSVLoader
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('vectorize.log'),
+        logging.StreamHandler()
+    ]
+)
+
+class VectorizationError(Exception):
+    """Custom exception for vectorization-related errors"""
+    pass
+
+def load_csv_documents(csv_file_path: str) -> List[Document]:
+    """
+    Load CSV documents from the specified path.
+
+    Args:
+        csv_file_path (str): Path pattern to search for CSV files.
+
+    Returns:
+        List[Document]: A list of documents loaded from the CSV files.
+
+    Raises:
+        VectorizationError: If no CSV files are found or if there's an error loading them.
+    """
+    try:
+        documents = []
+        csv_files = list(glob.glob(csv_file_path))
+        
+        if not csv_files:
+            raise VectorizationError(f"No CSV files found at path: {csv_file_path}")
+        
+        for csv_file in csv_files:
+            logging.info(f"Loading CSV file: {csv_file}")
+            loader = CSVLoader(csv_file, encoding="utf-8")
+            documents.extend(loader.load())
+            
+        logging.info(f"Successfully loaded {len(documents)} documents from {len(csv_files)} CSV files")
+        return documents
+    
+    except Exception as e:
+        raise VectorizationError(f"Error loading CSV documents: {str(e)}")
+
+def create_vector_store(
+    documents: List[Document],
+    embeddings_model: HuggingFaceEmbeddings,
+    output_path: str,
+    chunk_size: int = 500,
+    chunk_overlap: int = 50
+) -> Optional[FAISS]:
+    """
+    Create and save a FAISS vector store from documents.
+
+    Args:
+        documents (List[Document]): List of documents to vectorize
+        embeddings_model (HuggingFaceEmbeddings): The embeddings model to use
+        output_path (str): Path to save the FAISS index
+        chunk_size (int, optional): Size of text chunks. Defaults to 500.
+        chunk_overlap (int, optional): Overlap between chunks. Defaults to 50.
+
+    Returns:
+        Optional[FAISS]: The created FAISS index if successful, None otherwise
+    """
+    try:
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+        
+        chunked_documents = text_splitter.split_documents(documents)
+        logging.info(f"Created {len(chunked_documents)} chunks from {len(documents)} documents")
+        
+        faiss_index = FAISS.from_documents(chunked_documents, embeddings_model)
+        
+       
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        
+        faiss_index.save_local(output_path)
+        logging.info(f"Successfully saved FAISS index to {output_path}")
+        
+        return faiss_index
+    
+    except Exception as e:
+        logging.error(f"Error creating vector store: {str(e)}")
+        return None
+
+def main():
+    try:
+        # Configuration with relative paths
+        config = {
+            'msd_data_path': "./processed_data/msd/msd_processed.csv",
+            'medical_csv_path': "./processed_data/cbip/*.csv",
+            'msd_vector_path': "./vectors_data/msd_data_vec",
+            'medical_vector_path': "./vectors_data/med_data_vec",
+            'model_name': "sentence-transformers/all-MiniLM-L12-v2"
+        }
+        
+        # Create vectors_data directory if it doesn't exist
+        Path("./vectors_data").mkdir(exist_ok=True)
+        
+        logging.info("Starting vectorization process")
+        
+        # Load documents
+        msd_data_documents = load_csv_documents(config['msd_data_path'])
+        medical_documents = load_csv_documents(config['medical_csv_path'])
+        
+        # Initialize embeddings model
+        logging.info(f"Initializing embeddings model: {config['model_name']}")
+        embeddings_model = HuggingFaceEmbeddings(model_name=config['model_name'])
+        
+        # Create vector stores
+        msd_index = create_vector_store(
+            msd_data_documents,
+            embeddings_model,
+            config['msd_vector_path']
+        )
+        
+        medical_index = create_vector_store(
+            medical_documents,
+            embeddings_model,
+            config['medical_vector_path']
+        )
+        
+        if msd_index and medical_index:
+            logging.info("Vectorization process completed successfully")
+        else:
+            logging.error("Vectorization process completed with errors")
+            
+    except VectorizationError as ve:
+        logging.error(f"Vectorization error: {str(ve)}")
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    main()

vectors_data/med_data_vec/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178a3e2e7fea1a9ca64ca39dd32443d3c90f7fcd7602328470f65a896c43557d
+size 13323309

vectors_data/med_data_vec/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f1a9804e83879fc36cc98e68dc88564bd16ce3d552eb0b0b54da145f8d92e14
+size 2648619

vectors_data/msd_data_vec/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504d3ef09a0197a1f77de7865f23a801596ece8722e024cdcf07859c9ae53a52
+size 370983981

vectors_data/msd_data_vec/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57f6ad56b1c1462bf030ab66e7f37bf2616c6c8b0cfcd9841df39606a8157eba
+size 135114811