hybrid search doc string added

rag_app/get_db_retriever.py

@@ -1,4 +1,4 @@
-# retriever and qa_chain function
+gi# retriever and qa_chain function
 
 # HF libraries
 from langchain.llms import HuggingFaceHub

rag_app/hybrid_search.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+from langchain_community.vectorstores import FAISS
+from dotenv import load_dotenv
+import os
+from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
+from langchain.retrievers import EnsembleRetriever
+from langchain_community.retrievers import BM25Retriever
+
+
+def get_hybrid_search_results(query:str,
+                              path_to_db:str,
+                              embedding_model:str,
+                              hf_api_key:str,
+                              num_docs:int=5) -> list:
+    """ Uses an ensemble retriever of BM25 and FAISS to return k num documents
+
+        Args:
+            query (str): The search query
+            path_to_db (str): Path to the vectorstore database
+            embedding_model (str): Embedding model used in the vector store
+            num_docs (int): Number of documents to return
+        
+        Returns
+            List of documents
+    
+    """
+    
+    embeddings = HuggingFaceInferenceAPIEmbeddings(api_key=hf_api_key,
+                                                   model_name=embedding_model)
+    # Load the vectorstore database
+    db = FAISS.load_local(folder_path=path_to_db,
+                          embeddings=embeddings,
+                          allow_dangerous_deserialization=True)
+
+    all_docs = db.similarity_search("", k=db.index.ntotal)
+
+    bm25_retriever = BM25Retriever.from_documents(all_docs)
+    bm25_retriever.k = num_docs  # How many results you want
+
+    faiss_retriever = db.as_retriever(search_kwargs={'k': num_docs})
+
+    ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, faiss_retriever],
+                                           weights=[0.5,0.5])
+    
+    results = ensemble_retriever.invoke(input=query) 
+    return results
+
+
+if __name__ == "__main__":
+    query = "Haustierversicherung"
+    HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+    EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
+    
+    path_to_vector_db = Path("..")/'vectorstore/faiss-insurance-agent-500'
+
+    results = get_hybrid_search_results(query=query, 
+                                    path_to_db=path_to_vector_db, 
+                                    embedding_model=EMBEDDING_MODEL, 
+                                    hf_api_key=HUGGINGFACEHUB_API_TOKEN)
+    
+    for doc in results:
+        print(doc)
+        print()

rag_app/metadata.ipynb

@@ -1,170 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pathlib import Path\n",
-    "from langchain_community.vectorstores import FAISS\n",
-    "from dotenv import load_dotenv\n",
-    "import os\n",
-    "from langchain_huggingface import HuggingFaceEmbeddings"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "load_dotenv()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFCEHUB_API_TOKEN')\n",
-    "EMBEDDING_MODEL = os.getenv(\"EMBEDDING_MODEL\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "folder_path = Path('..') / \"vectorstore/faiss-insurance-agent-500\"\n",
-    "faissdb = FAISS.load_local(folder_path=str(folder_path.resolve()),\n",
-    "                                embeddings=embeddings,\n",
-    "                                allow_dangerous_deserialization=True)    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Content: Die private Haftpflichtversicherung...\n",
-      "Metadata: {'source': 'https://www.wuerttembergische.de/versicherungen/stadt/wuppertal/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Versicherung in Wuppertal', 'description': 'Ihre Versicherungsagentur in Wuppertal: Kommen Sie zur Württembergischen Versicherung und profitieren Sie von einer persönlichen Beratung und ausgezeichnetem Service. ', 'language': 'de'}\n",
-      "---\n",
-      "Content: Haftpflichtversicherung...\n",
-      "Metadata: {'source': 'https://www.wuerttembergische.de/wohnen/hausratversicherung/sengschaden/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Sengschäden: So schützt Sie Ihre Hausrat- und Wohngebäudeversicherung', 'description': 'Deckt Ihre Hausratversicherung Sengschäden ab? Finden Sie heraus, wie Sie bei Schäden durch Glut oder Hitze ohne direktes Feuer geschützt sind.\\n', 'language': 'de'}\n",
-      "---\n",
-      "Content: Die Leistungen unserer privaten Haftpflichtversich...\n",
-      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/drohnen-versichern/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Drohnen über die private Haftpflicht versichern', 'description': 'Müssen Drohnen versichert sein? Welcher Tarif ist der beste? Erfahren Sie hier die wichtigsten Informationen rund ums Thema Drohne versichern.', 'language': 'de'}\n",
-      "---\n",
-      "Content: Das kann ohne private Haftpflichtversicherung pass...\n",
-      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/pflicht/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Ist die private Haftpflichtversicherung Pflicht oder freiwillig?', 'description': 'Ist eine Privathaftpflichtversicherung gesetzlich vorgeschrieben? Welche Haftpflichtversicherung Pflicht sind und welche freiwillig - das erfahren Sie hier.', 'language': 'de'}\n",
-      "---\n",
-      "Content: Private Haftpflicht: keine Pflichtversicherung\n",
-      "Fre...\n",
-      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/pflicht/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Ist die private Haftpflichtversicherung Pflicht oder freiwillig?', 'description': 'Ist eine Privathaftpflichtversicherung gesetzlich vorgeschrieben? Welche Haftpflichtversicherung Pflicht sind und welche freiwillig - das erfahren Sie hier.', 'language': 'de'}\n",
-      "---\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Perform a similarity search with an empty query to get random documents\n",
-    "documents = faissdb.similarity_search(\"Private Haftpflicht­versicherung\", k=5)\n",
-    "\n",
-    "for doc in documents:\n",
-    "    print(f\"Content: {doc.page_content[:50]}...\")  # Print first 50 chars of content\n",
-    "    print(f\"Metadata: {doc.metadata}\")\n",
-    "    print(\"---\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Number of entries in the database: 62496\n"
-     ]
-    }
-   ],
-   "source": [
-    "num_entries = len(faissdb.index_to_docstore_id)\n",
-    "print(f\"Number of entries in the database: {num_entries}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Number of entries in the database: 62496\n"
-     ]
-    }
-   ],
-   "source": [
-    "num_entries = faissdb.index.ntotal\n",
-    "print(f\"Number of entries in the database: {num_entries}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.4"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

rag_app/metadata_filtering.py

@@ -1,29 +0,0 @@
-from pathlib import Path
-from langchain_community.vectorstores import FAISS
-from dotenv import load_dotenv
-import os
-from langchain_huggingface import HuggingFaceEmbeddings
-
-
-load_dotenv(".env")
-
-HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
-EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
-
-
-if __name__ == "__main__":
-
-    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
-
-    folder_path = Path('..') / "vectorstore/faiss-insurance-agent-500"
-
-    print(f'{Path(folder_path).exists() = }')        
-
-    faissdb = FAISS.load_local(folder_path=str(folder_path.resolve()),
-                                embeddings=embeddings,
-                                allow_dangerous_deserialization=True)
-    
-    documents = faissdb.get(list(range(5)))
-
-    for doc in documents:
-        print(f"Metadata: {doc.metadata}")