Linted code with Black

backend/app/main.py

@@ -7,6 +7,9 @@ from backend.app.problem_generator import ProblemGenerationPipeline
 from backend.app.problem_grader import ProblemGradingPipeline
 from typing import Dict, List
 import asyncio
+import logging
+import os
+from crawler import DomainCrawler
 
 app = FastAPI()
 
@@ -18,38 +21,48 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
 class UrlInput(BaseModel):
     url: str
 
+
 class UserQuery(BaseModel):
     user_query: str
 
+
 # TODO: Make this a list of {problem: str, answer: str}. Would be cleaner for data validation
 class FeedbackRequest(BaseModel):
     user_query: str
     problems: list[str]
     user_answers: list[str]
 
+
 class FeedbackResponse(BaseModel):
     feedback: List[str]
 
+
 @app.post("/api/crawl/")
 async def crawl_documentation(input_data: UrlInput):
     print(f"Received url {input_data.url}")
     return {"status": "received"}
 
+
 @app.post("/api/problems/")
 async def generate_problems(query: UserQuery):
     problems = ProblemGenerationPipeline().generate_problems(query.user_query)
     return {"Problems": problems}
 
+
 @app.post("/api/feedback", response_model=FeedbackResponse)
 async def get_feedback(request: FeedbackRequest):
     if len(request.problems) != len(request.user_answers):
-        raise HTTPException(status_code=400, detail="Problems and user answers must have the same length")
+        raise HTTPException(
+            status_code=400,
+            detail="Problems and user answers must have the same length",
+        )
     try:
         grader = ProblemGradingPipeline()
-        
+
         grading_tasks = [
             grader.grade(
                 query=request.user_query,
@@ -58,32 +71,59 @@ async def get_feedback(request: FeedbackRequest):
             )
             for problem, user_answer in zip(request.problems, request.user_answers)
         ]
-        
+
         feedback_list = await asyncio.gather(*grading_tasks)
-        
+
         return FeedbackResponse(feedback=feedback_list)
-        
+
     except Exception as e:
         # log exception and stack trace
         import traceback
+
         print(f"Exception: {e}")
         print(f"Stack trace: {traceback.format_exc()}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 # Serve static files
 app.mount("/static", StaticFiles(directory="/app/static/static"), name="static")
 
+
 # Root path handler
 @app.get("/")
 async def serve_root():
     return FileResponse("/app/static/index.html")
 
+
 # Catch-all route for serving index.html
 @app.get("/{full_path:path}")
 async def serve_react(full_path: str):
     # Skip API routes
     if full_path.startswith("api/"):
         raise HTTPException(status_code=404, detail="Not found")
-    
+
     # For all other routes, serve the React index.html
-    return FileResponse("/app/static/index.html") 
+    return FileResponse("/app/static/index.html")
+
+
+def setup_logging():
+    """Configure logging for the entire application"""
+    # Create logs directory if it doesn't exist
+    logs_dir = "logs"
+    if not os.path.exists(logs_dir):
+        os.makedirs(logs_dir)
+
+    # Configure logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[
+            # Console handler
+            logging.StreamHandler(),
+            # File handler
+            logging.FileHandler(os.path.join(logs_dir, "crawler.log")),
+        ],
+    )
+
+
+setup_logging()

backend/app/problem_generator.py

@@ -32,14 +32,15 @@ USER_ROLE_PROMPT = """
 
 class ProblemGenerationPipeline:
     def __init__(self, return_context: bool = False, embedding_model_id: str = None):
-        self.chat_prompt = ChatPromptTemplate.from_messages([
-            ("system", SYSTEM_ROLE_PROMPT),
-            ("user", USER_ROLE_PROMPT)
-        ])
-        
+        self.chat_prompt = ChatPromptTemplate.from_messages(
+            [("system", SYSTEM_ROLE_PROMPT), ("user", USER_ROLE_PROMPT)]
+        )
+
         self.llm = ChatOpenAI(model=MODEL, temperature=0.7)
-        self.retriever = get_vector_db(embedding_model_id).as_retriever(search_kwargs={"k": 2})
-        
+        self.retriever = get_vector_db(embedding_model_id).as_retriever(
+            search_kwargs={"k": 2}
+        )
+
         # TODO: This is a hack to get the context for the questions. Very messy interface.
         self.return_context = return_context
         if not return_context:
@@ -52,18 +53,24 @@ class ProblemGenerationPipeline:
         else:
             # response looks like: {response: str, context: List[Document]}
             self.rag_chain = (
-                {"context": itemgetter("query") | self.retriever, "query": itemgetter("query")}
+                {
+                    "context": itemgetter("query") | self.retriever,
+                    "query": itemgetter("query"),
+                }
                 | RunnablePassthrough.assign(context=itemgetter("context"))
-                | {"response": self.chat_prompt | self.llm | StrOutputParser(), "context": itemgetter("context")}
+                | {
+                    "response": self.chat_prompt | self.llm | StrOutputParser(),
+                    "context": itemgetter("context"),
+                }
             )
 
     def generate_problems(self, query: str, debug: bool = False) -> List[str]:
         """
         Generate problems based on the user's query using RAG.
-        
+
         Args:
             query (str): The topic to generate questions about
-            
+
         Returns:
             List[str]: A list of generated questions
         """
@@ -75,4 +82,4 @@ class ProblemGenerationPipeline:
             return raw_result
         # raw_result is a string when return_context is False
         else:
-            return json.loads(raw_result)["questions"]
+            return json.loads(raw_result)["questions"]

backend/app/problem_grader.py

@@ -7,6 +7,7 @@ from langchain_core.runnables import RunnablePassthrough
 from langchain_core.output_parsers import StrOutputParser
 from backend.app.vectorstore import get_vector_db
 from operator import itemgetter
+
 MODEL = "gpt-3.5-turbo"
 
 SYSTEM_ROLE_PROMPT = """
@@ -35,23 +36,24 @@ USER_ROLE_PROMPT = """
 
 class ProblemGradingPipeline:
     def __init__(self):
-        self.chat_prompt = ChatPromptTemplate.from_messages([
-            ("system", SYSTEM_ROLE_PROMPT),
-            ("user", USER_ROLE_PROMPT)
-        ])
-        
+        self.chat_prompt = ChatPromptTemplate.from_messages(
+            [("system", SYSTEM_ROLE_PROMPT), ("user", USER_ROLE_PROMPT)]
+        )
+
         self.llm = ChatOpenAI(model=MODEL, temperature=0.3)
         self.retriever = get_vector_db().as_retriever(search_kwargs={"k": 2})
-        
-        self.rag_chain = (   
+
+        self.rag_chain = (
             {
                 # Use the query to retrieve documents from the vectorstore
-                "context": itemgetter("query") | self.retriever | (lambda docs: "\n\n".join([doc.page_content for doc in docs])),
+                "context": itemgetter("query")
+                | self.retriever
+                | (lambda docs: "\n\n".join([doc.page_content for doc in docs])),
                 # Pass through all other inputs directly
                 "query": itemgetter("query"),
                 "problem": itemgetter("problem"),
-                "answer": itemgetter("answer")
-            } 
+                "answer": itemgetter("answer"),
+            }
             | self.chat_prompt
             | self.llm
             | StrOutputParser()
@@ -60,18 +62,16 @@ class ProblemGradingPipeline:
     async def grade(self, query: str, problem: str, answer: str) -> str:
         """
         Asynchronously grade a student's answer to a problem using RAG for context-aware evaluation.
-        
+
         Args:
             query (str): The topic/context to use for grading
             problem (str): The question being answered
             answer (str): The student's answer to evaluate
-            
+
         Returns:
             str: Grading response indicating if the answer is correct and providing feedback
         """
         print(f"Grading problem: {problem} with answer: {answer} for query: {query}")
-        return await self.rag_chain.ainvoke({
-            "query": query,
-            "problem": problem,
-            "answer": answer
-        }) 
+        return await self.rag_chain.ainvoke(
+            {"query": query, "problem": problem, "answer": answer}
+        )

backend/app/vectorstore.py

@@ -3,20 +3,26 @@ Super early version of a vector store. Just want to make something available for
 
 Vector store implementation with singleton pattern to ensure only one instance exists.
 """
+
 import os
 import requests
 import nltk
+import logging
 from typing import Optional
 from langchain_community.vectorstores import Qdrant
 from langchain_openai.embeddings import OpenAIEmbeddings
 from langchain_community.document_loaders import DirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_huggingface import HuggingFaceEmbeddings
+from qdrant_client import QdrantClient
 
-nltk.download('punkt_tab')
-nltk.download('averaged_perceptron_tagger_eng')
+nltk.download("punkt_tab")
+nltk.download("averaged_perceptron_tagger_eng")
 
 DEFAULT_EMBEDDING_MODEL_ID = "text-embedding-3-small"
+LOCAL_QDRANT_PATH = "/data/qdrant_db"
+
+logger = logging.getLogger(__name__)
 
 # Global variable to store the singleton instance
 _vector_db_instance: Optional[Qdrant] = None
@@ -24,13 +30,29 @@ _vector_db_instance: Optional[Qdrant] = None
 #   to match the new embedding model.
 _embedding_model_id: str = None
 
+
+def get_qdrant_client():
+    if os.environ.get("QDRANT_URL") is None or os.environ.get("QDRANT_API_KEY") is None:
+        logger.error(
+            "QDRANT_URL or QDRANT_API_KEY is not set. Defaulting to local memory vector store."
+        )
+
+        os.makedirs(LOCAL_QDRANT_PATH, exist_ok=True)
+        return QdrantClient(path=LOCAL_QDRANT_PATH)
+
+    QDRANT_URL = os.environ.get("QDRANT_URL")
+    QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
+
+    return QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+
+
 def get_vector_db(embedding_model_id: str = None) -> Qdrant:
     """
     Factory function that returns a singleton instance of the vector database.
     Creates the instance if it doesn't exist.
     """
     global _vector_db_instance
-    
+
     if _vector_db_instance is None:
         # Create static/data directory if it doesn't exist
         os.makedirs("static/data", exist_ok=True)
@@ -45,7 +67,7 @@ def get_vector_db(embedding_model_id: str = None) -> Qdrant:
 
         embedding_model = None
         if embedding_model_id is None:
-            embedding_model = OpenAIEmbeddings(model=DEFAULT_EMBEDDING_MODEL_ID)
+            embedding_model = OpenAIEmbeddings(modzŻel=DEFAULT_EMBEDDING_MODEL_ID)
         else:
             embedding_model = HuggingFaceEmbeddings(model_name=embedding_model_id)
 
@@ -55,16 +77,16 @@ def get_vector_db(embedding_model_id: str = None) -> Qdrant:
 
         # Split documents into chunks
         text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1000,
-            chunk_overlap=200
+            chunk_size=1000, chunk_overlap=200
         )
         split_chunks = text_splitter.split_documents(documents)
 
         # Create vector store instance
+        client = get_qdrant_client()
         _vector_db_instance = Qdrant.from_documents(
             split_chunks,
             embedding_model,
-            location=":memory:",
+            client=client,
             collection_name="extending_context_window_llama_3",
         )
 

backend/tests/test_api.py

@@ -4,22 +4,19 @@ import pytest
 
 client = TestClient(app)
 
+
 def test_crawl_endpoint():
-    response = client.post(
-        "/api/crawl/",
-        json={"url": "https://example.com"}
-    )
+    response = client.post("/api/crawl/", json={"url": "https://example.com"})
     assert response.status_code == 200
     assert response.json() == {"status": "received"}
 
+
 def test_problems_endpoint():
-    response = client.post(
-        "/api/problems/",
-        json={"user_query": "RAG"}
-    )
+    response = client.post("/api/problems/", json={"user_query": "RAG"})
     assert response.status_code == 200
     assert "Problems" in response.json()
-    assert len(response.json()["Problems"]) == 5 
+    assert len(response.json()["Problems"]) == 5
+
 
 def test_feedback_validation_error():
     """Test that mismatched problems and answers lengths return 400"""
@@ -28,13 +25,16 @@ def test_feedback_validation_error():
         json={
             "user_query": "Python lists",
             "problems": ["What is a list?", "How do you append?"],
-            "user_answers": ["A sequence",]  # Only one answer
-        }
+            "user_answers": [
+                "A sequence",
+            ],  # Only one answer
+        },
     )
-    
+
     assert response.status_code == 400
     assert "same length" in response.json()["detail"]
 
+
 @pytest.mark.asyncio
 async def test_successful_feedback():
     """Test successful grading of multiple problems"""
@@ -44,24 +44,22 @@ async def test_successful_feedback():
             "user_query": "RAG",
             "problems": [
                 "What are the two main components of a typical RAG application?",
-                "What is the purpose of the indexing component in a RAG application?"
+                "What is the purpose of the indexing component in a RAG application?",
             ],
             "user_answers": [
                 "A list is a mutable sequence type that can store multiple items in Python",
-                "You use the append() method to add an element to the end of a list"
-            ]
-        }
+                "You use the append() method to add an element to the end of a list",
+            ],
+        },
     )
-    
+
     assert response.status_code == 200
     result = response.json()
     assert "feedback" in result
     assert len(result["feedback"]) == 2
-    
+
     # Check that responses start with either "Correct" or "Incorrect"
     for feedback in result["feedback"]:
         assert feedback.startswith(("Correct", "Incorrect"))
         # Check that there's an explanation after the classification
         assert len(feedback.split(". ")) >= 2
-
-    

backend/tests/test_vectorstore.py

@@ -2,43 +2,47 @@ import os
 from langchain.schema import Document
 from backend.app.vectorstore import get_vector_db
 
+
 def test_directory_creation():
     get_vector_db()
     assert os.path.exists("static/data")
     assert os.path.exists("static/data/langchain_rag_tutorial.html")
 
+
 # TODO remove this test when data ingrestion layer is implemented
 def test_html_content():
     with open("static/data/langchain_rag_tutorial.html", "r", encoding="utf-8") as f:
         content = f.read()
-    
+
     # Check for some expected content from the LangChain RAG tutorial
     assert "RAG" in content
     assert "LangChain" in content
 
+
 def test_vector_store_similarity_search():
     """Test that the vector store can perform similarity search"""
     # Test query
     query = "What is RAG?"
-    
+
     # Get vector db instance and perform similarity search
     vector_db = get_vector_db()
     results = vector_db.similarity_search(query, k=2)
-    
+
     # Verify we get results
     assert len(results) == 2
     assert isinstance(results[0], Document)
-    
+
     # Verify the results contain relevant content
     combined_content = " ".join([doc.page_content for doc in results]).lower()
     assert "rag" in combined_content
     assert "retrieval" in combined_content
 
+
 def test_vector_db_singleton():
     """Test that get_vector_db returns the same instance each time"""
     # Get two instances
     instance1 = get_vector_db()
     instance2 = get_vector_db()
-    
+
     # Verify they are the same object
-    assert instance1 is instance2
+    assert instance1 is instance2

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "wandb>=0.19.6",
     "datasets>=3.2.0",
     "ragas==0.2.10",
+    "black>=25.1.0",
 ]
 
 [tool.setuptools]