Medivocate : An AI-powered platform exploring African history, culture, and traditional medicine, fostering understanding and appreciation of the continent's rich heritage.

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/chroma_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,178 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+.vscode/
+
+*.out
+.python-version
+
+.venv
+*.sh

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 KameniAlexNea
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,14 +1,65 @@
----
-title: Medivocate
-emoji: 📈
-colorFrom: purple
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.16.0
-app_file: app.py
-pinned: false
-license: mit
-short_description: Exploring African history, culture, and traditional medicine
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Medivocate
+emoji: 🐢
+colorFrom: purple
+colorTo: red
+sdk: gradio
+sdk_version: 5.12.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: Medivocate is an AI-driven platform leveraging Retrieval-Aug
+---
+
+# Medivocate
+
+An AI-driven platform empowering users with trustworthy, personalized history guidance to combat misinformation and promote equitable history.
+
+## Follows us [here](https://github.com/KameniAlexNea/medivocate)
+
+* [**Alex Kameni**](https://www.linkedin.com/in/elie-alex-kameni-ngangue/)
+* [**Esdras Fandio**](https://www.linkedin.com/in/esdras-fandio/)
+* [**Patric Zeufack**](https://www.linkedin.com/in/zeufack-patric-hermann-7a9256143/)
+
+## Project Overview
+
+**Medivocate** is structured for modular development and ease of scalability, as seen in its directory layout:
+
+```
+📦 ./
+├── 📁 docs/
+├── 📁 src/
+│   ├── 📁 ocr/
+│   ├── 📁 preprocessing/
+│   ├── 📁 chunking/
+│   ├── 📁 vector_store/
+│   ├── 📁 rag_pipeline/
+│   ├── 📁 llm_integration/
+│   └── 📁 prompt_engineering/
+├── 📁 tests/
+│   ├── 📁 unit/
+│   └── 📁 integration/
+├── 📁 examples/
+├── 📁 notebooks/
+├── 📁 config/
+├── 📄 README.md
+├── 📄 CONTRIBUTING.md
+├── 📄 requirements.txt
+├── 📄 .gitignore
+└── 📄 LICENSE
+```
+
+### Key Features
+
+1. **Trustworthy Information Access** : Using RAG (Retrieval-Augmented Generation) pipelines to deliver fact-based responses.
+2. **Advanced Document Handling** : Leveraging OCR, preprocessing, and chunking for scalable document ingestion.
+3. **Integrated Tools** : Supports integration with vector databases (e.g., Chroma), LLMs, and advanced prompt engineering techniques.
+
+### Recommendations for Integration
+
+* **Groq** : Utilize Groq APIs for free-tier LLM support, perfect for prototyping RAG applications.
+* **LangChain + LangSmith** : Build and monitor intelligent agents with LangChain and enhance debugging and evaluation using LangSmith.
+* **Hugging Face Datasets** : For one-liner dataset loading and preprocessing, supporting efficient ML training pipelines.
+* **Search Index** : Include Chroma for robust semantic search capabilities in RAG.
+
+This modular design and extensive integration make Medivocate a powerful tool for historical education and research.

app.py

@@ -0,0 +1,57 @@
+import os
+
+os.environ["IS_APP"] = "1"
+from typing import List
+
+import gradio as gr
+
+from src.rag_pipeline.rag_system import RAGSystem
+from load_data import download_and_prepare_data
+
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+
+
+class ChatInterface:
+    def __init__(self, rag_system: RAGSystem):
+        self.rag_system = rag_system
+        self.history_depth = int(os.getenv("MAX_MESSAGES") or 5) * 2
+
+    def respond(self, message: str, history: List[List[str]]):
+        result = ""
+        history = [(turn["role"], turn["content"]) for turn in history[-self.history_depth:]]
+        for text in self.rag_system.query(message, history):
+            result += text
+            yield result
+        return result
+
+    def create_interface(self) -> gr.ChatInterface:
+        description = (
+            "Medivocate is an application that offers clear and structured information "
+            "about African history and traditional medicine. The knowledge is exclusively "
+            "based on historical documentaries about the African continent.\n\n"
+            "🌟 **Code Repository**: [Medivocate GitHub](https://github.com/KameniAlexNea/medivocate)"
+        )
+        return gr.ChatInterface(
+            fn=self.respond,
+            type="messages",
+            title="Medivocate",
+            description=description,
+        )
+
+
+# Usage example:
+if __name__ == "__main__":
+    # Example usage
+    zip_filename = "chroma_db.zip"
+    extract_to = "chroma_db"
+    target_folder = "data/chroma_db"
+    gdrive_url = os.getenv("GDRIVE_URL")
+    download_and_prepare_data(gdrive_url, zip_filename, extract_to, target_folder)
+
+    top_k_docs = int(os.getenv("N_CONTEXT") or 4)
+    rag_system = RAGSystem(top_k_documents=top_k_docs)
+    rag_system.initialize_vector_store()
+
+    chat_interface = ChatInterface(rag_system)
+    demo = chat_interface.create_interface()
+    demo.launch(share=False)

data/chroma_db/ad04fd29-b3fe-456e-a525-757a6756c29e/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fea540d03a37ecce67de43d3bce99a5ca61a0fcec19cbfe67928ed19064e72
+size 16296000

data/chroma_db/ad04fd29-b3fe-456e-a525-757a6756c29e/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f49783cbcb025a93b8ac35a9c337d4cfdc94f741d1bd1c4b944127b212554a6
+size 100

data/chroma_db/ad04fd29-b3fe-456e-a525-757a6756c29e/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d834576f8575c2eb62318da8a2e7fa1ec58b849d9bc02ae88f09c50c6cf60dad
+size 755153

data/chroma_db/ad04fd29-b3fe-456e-a525-757a6756c29e/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f6284c00d0f3b2567bb2d48cf91e813e827d7d3045017992afd4f904428f64
+size 56000

data/chroma_db/ad04fd29-b3fe-456e-a525-757a6756c29e/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:610ed567f1bd6179e7df87fe10f2d8c0b0d29cab23628d900a99ce93e3688922
+size 118696

data/chroma_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:399ec17ca61e724b6f5fe0818842f32c046f5f2e2014b9dda310f967b68faeb1
+size 199651328

load_data.py

@@ -0,0 +1,47 @@
+import os
+import subprocess
+import zipfile
+import shutil
+
+import logging
+
+def download_and_prepare_data(gdrive_url, zip_filename, extract_to, target_folder):
+    """
+    Download, extract, and organize data from a Google Drive link.
+    
+    :param gdrive_url: Google Drive URL to download the zip file.
+    :param zip_filename: Name for the downloaded zip file.
+    :param extract_to: Directory to extract the zip file.
+    :param target_folder: Final directory to move extracted content.
+    """
+    try:
+        if os.path.exists(os.path.join(target_folder, "chroma.sqlite3")):
+            logging.info(f"Data already exists in {target_folder}")
+            return
+        # Step 1: Download the file using gdown
+        logging.info("Downloading file...")
+        subprocess.run(["gdown", gdrive_url, "-O", zip_filename], check=True)
+        
+        # Step 2: Unzip the downloaded file
+        logging.info("Unzipping file...")
+        with zipfile.ZipFile(zip_filename, 'r') as zip_ref:
+            zip_ref.extractall(extract_to)
+        
+        # Step 3: Remove old data folder if it exists
+        if os.path.exists(target_folder):
+            logging.info(f"Removing existing folder: {target_folder}")
+            shutil.rmtree(target_folder)
+        
+        # Step 4: Move the extracted folder to the target location
+        logging.info(f"Moving extracted data to {target_folder}")
+        extracted_folder = os.path.join(extract_to, os.path.basename(target_folder))
+        shutil.move(extracted_folder, target_folder)
+        
+        # Step 5: Remove the downloaded zip file
+        logging.info(f"Cleaning up, removing zip file: {zip_filename}")
+        os.remove(zip_filename)
+        
+        logging.info("Data preparation completed successfully!")
+    except Exception as e:
+        logging.info(f"An error occurred: {e}")
+

requirements.txt

@@ -0,0 +1,15 @@
+langchain-ollama==0.2.2
+langchain-groq==0.2.3
+langchain-community==0.3.14
+langchain-chroma==0.1.4
+langchain-huggingface
+langchain==0.3.14
+ollama==0.4.5
+chromadb==0.5.23
+# OCR
+tqdm==4.67.1
+gradio==5.11.0
+rank_bm25==0.2.2
+groq==0.15.0
+gdown==5.2.0
+einops==0.8.1

src/rag_pipeline/prompts.py

@@ -0,0 +1,62 @@
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    SystemMessagePromptTemplate,
+)
+
+system_prompt = """
+Vous êtes **Dikoka**, un assistant IA expert en histoire de l'Afrique et en médecine traditionnelle africaine, basé sur des recherches et documents historiques validés.
+
+**Instructions :**
+- **Répondez strictement en utilisant uniquement le contexte fourni.**
+- **Résumez les points clés lorsque cela est demandé.**
+- **Maintenez une grande rigueur dans l'exactitude et la neutralité ; évitez toute spéculation ou ajout d'informations externes.**
+
+**Directives de réponse :**
+1. **Réponses fondées uniquement sur le contexte :** Appuyez-vous exclusivement sur le contexte fourni.
+2. **Informations insuffisantes :** Si les détails manquent, répondez :
+   > "Je n'ai pas suffisamment d'informations pour répondre à cette question en fonction du contexte fourni."
+3. **Demandes concernant la langue :** Si une question est posée dans une langue africaine ou demande une traduction, répondez :
+   > "Je ne peux fournir les informations que dans la langue du contexte original. Pourriez-vous reformuler votre question dans cette langue ?"
+4. **Sujets non pertinents :** Pour les questions qui ne concernent pas :
+   - L'histoire de l'Afrique
+   - La médecine traditionnelle africaine
+   
+   répondez :
+   > "Je n'ai pas d'informations sur ce sujet en fonction du contexte fourni. Pourriez-vous poser une question relative à l'histoire de l'Afrique ou à la médecine traditionnelle africaine ?"
+5. **Résumés :** Fournissez des résumés concis et structurés (à l'aide de points ou de paragraphes) basés uniquement sur le contexte.
+6. **Mise en forme :** Organisez vos réponses avec des listes à puces, des listes numérotées, ainsi que des titres et sous-titres lorsque cela est approprié.
+
+Contexte :
+{context}
+"""
+
+# Define the messages for the main chat prompt
+chat_messages = [
+    MessagesPlaceholder(variable_name="chat_history"),
+    SystemMessagePromptTemplate.from_template(system_prompt),
+    HumanMessagePromptTemplate.from_template(
+        "Repondre dans la même langue que l'utilisateur:\n{input}"
+    ),
+]
+CHAT_PROMPT = ChatPromptTemplate.from_messages(chat_messages)
+
+
+contextualize_q_system_prompt = (
+    "Votre tâche consiste à formuler une question autonome, claire et compréhensible sans recourir à l'historique de conversation. Veuillez suivre ces instructions :\n"
+    "1. Analysez l'historique de conversation ainsi que la dernière question posée par l'utilisateur.\n"
+    "2. Reformulez la question en intégrant tout contexte nécessaire pour qu'elle soit compréhensible sans l'historique.\n"
+    "3. Si la question initiale est déjà autonome, renvoyez-la telle quelle.\n"
+    "4. Conservez l'intention et la langue d'origine de la question.\n"
+    "5. Fournissez uniquement la question autonome, sans explications ou texte additionnel.\n"
+    "NE répondez PAS à la question."
+)
+
+CONTEXTUEL_QUERY_PROMPT = ChatPromptTemplate.from_messages(
+    [
+        # SystemMessagePromptTemplate.from_template(contextualize_q_system_prompt),
+        MessagesPlaceholder("chat_history"),
+        HumanMessagePromptTemplate.from_template("{input}"),
+    ]
+)

src/rag_pipeline/rag_system.py

@@ -0,0 +1,115 @@
+import logging
+import os
+from typing import List, Optional
+
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains.conversational_retrieval.base import (
+    BaseConversationalRetrievalChain,
+)
+from langchain.chains.history_aware_retriever import (
+    create_history_aware_retriever,
+)
+from langchain.chains.retrieval import create_retrieval_chain
+
+from ..utilities.llm_models import get_llm_model_chat
+from ..vector_store.vector_store import VectorStoreManager
+from .prompts import CHAT_PROMPT, CONTEXTUEL_QUERY_PROMPT
+
+
+class RAGSystem:
+    def __init__(
+        self,
+        docs_dir: str = "data/chunks",
+        persist_directory_dir="data/chroma_db",
+        batch_size: int = 64,
+        top_k_documents=5,
+    ):
+        self.top_k_documents = top_k_documents
+        self.llm = self._get_llm()
+        self.chain: Optional[BaseConversationalRetrievalChain] = None
+        self.vector_store_management = VectorStoreManager(
+            persist_directory_dir, batch_size
+        )
+        self.docs_dir = docs_dir
+
+    def _get_llm(
+        self,
+    ):
+        return get_llm_model_chat(temperature=0.1, max_tokens=1000)
+
+    def load_documents(self) -> List:
+        """Load and split documents from the specified directory"""
+        return self.vector_store_management.load_and_process_documents(self.docs_dir)
+
+    def initialize_vector_store(self, documents: List = None):
+        """Initialize or load the vector store"""
+        self.vector_store_management.initialize_vector_store(documents)
+
+    def setup_rag_chain(self):
+        if self.chain is not None:
+            return
+        retriever = self.vector_store_management.create_retriever(
+            self.llm, self.top_k_documents, bm25_portion=0.03
+        )
+
+        # Contextualize question
+        self.history_aware_retriever = create_history_aware_retriever(
+            self.llm, retriever, CONTEXTUEL_QUERY_PROMPT
+        )
+        self.question_answer_chain = create_stuff_documents_chain(self.llm, CHAT_PROMPT)
+        self.chain = create_retrieval_chain(
+            self.history_aware_retriever, self.question_answer_chain
+        )
+        logging.info("RAG chain setup complete" + str(self.chain))
+
+        return self.chain
+
+    def query(self, question: str, history: list = []):
+        """Query the RAG system"""
+        if not self.vector_store_management.vs_initialized:
+            self.initialize_vector_store()
+
+        self.setup_rag_chain()
+
+        for token in self.chain.stream({"input": question, "chat_history": history}):
+            if "answer" in token:
+                yield token["answer"]
+
+
+if __name__ == "__main__":
+    from glob import glob
+
+    from dotenv import load_dotenv
+
+    # loading variables from .env file
+    load_dotenv()
+
+    docs_dir = "data/docs"
+    persist_directory_dir = "data/chroma_db"
+    batch_size = 64
+
+    # Initialize RAG system
+    rag = RAGSystem(docs_dir, persist_directory_dir, batch_size)
+
+    if len(glob(os.path.join(persist_directory_dir, "*/*.bin"))):
+        rag.initialize_vector_store()  # vector store initialized
+    else:
+        # Load and index documents
+        documents = rag.load_documents()
+        rag.initialize_vector_store(documents)  # documents
+
+    queries = [
+        "Quand a eu lieu la traite négrière ?",
+        "Explique moi comment soigner la tiphoide puis le paludisme",
+        "Quels étaient les premiers peuples d'afrique centrale et quelles ont été leurs migrations?",
+    ]
+
+    print("Comparaison méthodes de query")
+
+    for query in queries:
+        print("Query: ", query, "\n\n")
+        print("1. Méthode simple:--------------------\n")
+        rag.query(question=query)
+
+        print("\n\n2. Méthode par décomposition:-----------------------\n\n")
+        rag.query_complex(question=query, verbose=True)

src/utilities/embedding.py

@@ -0,0 +1,128 @@
+import logging
+import os
+from typing import Any, List
+
+import torch
+from langchain_core.embeddings import Embeddings
+from langchain_huggingface import (
+    HuggingFaceEmbeddings,
+    HuggingFaceEndpointEmbeddings,
+)
+from pydantic import BaseModel, Field
+
+
+class CustomEmbedding(BaseModel, Embeddings):
+    """
+    Custom embedding class that supports both hosted and CPU embeddings.
+    """
+
+    hosted_embedding: HuggingFaceEndpointEmbeddings = Field(
+        default_factory=lambda: None
+    )
+    cpu_embedding: HuggingFaceEmbeddings = Field(default_factory=lambda: None)
+    matryoshka_dim: int = Field(default=256)
+
+    def get_instruction(self) -> str:
+        """
+        Generates the instruction for the embedding model based on environment variables.
+
+        Returns:
+            str: The instruction string.
+        """
+        if "nomic" in os.getenv("HF_MODEL"):
+            return (
+                "query"
+                if (os.getenv("IS_APP", "0") == "1")
+                else "search_document: "
+            )
+        return (
+            "Represent this sentence for searching relevant passages"
+            if (os.getenv("IS_APP", "0") == "1")
+            else ""
+        )
+
+    def get_hf_embedd(self) -> HuggingFaceEmbeddings:
+        """
+        Initializes the HuggingFaceEmbeddings with the appropriate settings.
+
+        Returns:
+            HuggingFaceEmbeddings: The initialized HuggingFaceEmbeddings object.
+        """
+        return HuggingFaceEmbeddings(
+            model_name=os.getenv("HF_MODEL"),  # You can replace with any HF model
+            model_kwargs={
+                "device": "cpu" if not torch.cuda.is_available() else "cuda",
+                "trust_remote_code": True,
+            },
+            encode_kwargs={
+                "normalize_embeddings": True,
+                "prompt": self.get_instruction(),
+            },
+        )
+
+    def __init__(self, matryoshka_dim=256, **kwargs: Any):
+        """
+        Initializes the CustomEmbedding with the given parameters.
+
+        Args:
+            matryoshka_dim (int): Dimension of the embeddings.
+            **kwargs: Additional keyword arguments.
+        """
+        super().__init__(**kwargs)
+        query_instruction = self.get_instruction()
+        self.matryoshka_dim = matryoshka_dim
+        if torch.cuda.is_available():
+            logging.info("CUDA is available")
+            self.hosted_embedding = self.get_hf_embedd()
+            self.cpu_embedding = self.hosted_embedding
+        else:
+            logging.info("CUDA is not available")
+            self.hosted_embedding = HuggingFaceEndpointEmbeddings(
+                model=os.getenv("HF_MODEL"),
+                model_kwargs={
+                    "encode_kwargs": {
+                        "normalize_embeddings": True,
+                        "prompt": query_instruction,
+                    }
+                },
+                huggingfacehub_api_token=os.getenv("HUGGINGFACEHUB_API_TOKEN"),
+            )
+            self.cpu_embedding = self.get_hf_embedd()
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        """
+        Embeds a list of documents using the appropriate embedding model.
+
+        Args:
+            texts (List[str]): List of document texts to embed.
+
+        Returns:
+            List[List[float]]: List of embedded document vectors.
+        """
+        try:
+            embed = self.hosted_embedding.embed_documents(texts)
+        except Exception as e:
+            logging.warning(f"Issue with batch hosted embedding, moving to CPU: {e}")
+            embed = self.cpu_embedding.embed_documents(texts)
+        return (
+            [e[: self.matryoshka_dim] for e in embed] if self.matryoshka_dim else embed
+        )
+
+    def embed_query(self, text: str) -> List[float]:
+        """
+        Embeds a single query using the appropriate embedding model.
+
+        Args:
+            text (str): The query text to embed.
+
+        Returns:
+            List[float]: The embedded query vector.
+        """
+        try:
+            logging.info(text)
+            embed = self.hosted_embedding.embed_query(text)
+        except Exception as e:
+            logging.warning(f"Issue with hosted embedding, moving to CPU: {e}")
+            embed = self.cpu_embedding.embed_query(text)
+        logging.warning(text)
+        return embed[: self.matryoshka_dim] if self.matryoshka_dim else embed

src/utilities/llm_models.py

@@ -0,0 +1,46 @@
+import os
+from enum import Enum
+
+from langchain_groq import ChatGroq
+from langchain_ollama import ChatOllama, OllamaEmbeddings
+
+from .embedding import CustomEmbedding
+
+
+class LLMModel(Enum):
+    OLLAMA = "ChatOllama"
+    GROQ = "ChatGroq"
+
+
+def get_llm_model_chat(temperature=0.01, max_tokens: int = None):
+    if str(os.getenv("USE_OLLAMA_CHAT")) == "1":
+        return ChatOllama(
+            model=os.getenv("OLLAMA_MODEL"),
+            temperature=temperature,
+            num_predict=max_tokens,
+        )
+    return ChatGroq(
+        model=os.getenv("GROQ_MODEL_NAME"),
+        temperature=temperature,
+        max_tokens=max_tokens,
+    )
+
+
+def get_llm_model_embedding():
+    if str(os.getenv("USE_HF_EMBEDDING")) == "1":
+        return CustomEmbedding()
+    return OllamaEmbeddings(
+        model=os.getenv("OLLAM_EMB"),
+        base_url=(
+            os.getenv("OLLAMA_HOST") if os.getenv("OLLAMA_HOST") is not None else None
+        ),
+        client_kwargs=(
+            {
+                "headers": {
+                    "Authorization": "Bearer " + (os.getenv("OLLAMA_TOKEN") or "")
+                }
+            }
+            if os.getenv("OLLAMA_HOST") is not None
+            else None
+        ),
+    )

src/vector_store/bivector_store.py

@@ -0,0 +1,138 @@
+import os
+from typing import List, Union
+
+from langchain.retrievers import EnsembleRetriever, MultiQueryRetriever
+from langchain_chroma import Chroma
+from langchain_community.retrievers import BM25Retriever
+from langchain_core.documents import Document
+from tqdm import tqdm
+from transformers import AutoTokenizer
+
+from ..utilities.llm_models import get_llm_model_embedding
+from .document_loader import DocumentLoader
+from .vector_store import get_collection_name
+from .prompts import DEFAULT_QUERY_PROMPT
+
+class VectorStoreManager:
+    """
+    Manages vector store initialization, updates, and retrieval.
+    """
+
+    def __init__(self, persist_directory: str, batch_size: int = 64):
+        """
+        Initializes the VectorStoreManager with the given parameters.
+
+        Args:
+            persist_directory (str): Directory to persist the vector store.
+            batch_size (int): Number of documents to process in each batch.
+        """
+        self.persist_directory = persist_directory
+        self.batch_size = batch_size
+        self.embeddings = get_llm_model_embedding()
+        self.collection_name = get_collection_name()
+        self.vector_stores: dict[str, Union[Chroma, BM25Retriever]] = {
+            "chroma": None,
+            "bm25": None,
+        }
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            os.getenv("HF_MODEL", "meta-llama/Llama-3.2-1B")
+        )
+        self.vs_initialized = False
+        self.vector_store = None
+
+    def _batch_process_documents(self, documents: List[Document]):
+        """
+        Processes documents in batches for vector store initialization.
+
+        Args:
+            documents (List[Document]): List of documents to process.
+        """
+        for i in tqdm(
+            range(0, len(documents), self.batch_size), desc="Processing documents"
+        ):
+            batch = documents[i : i + self.batch_size]
+
+            if not self.vs_initialized:
+                self.vector_stores["chroma"] = Chroma.from_documents(
+                    collection_name=self.collection_name,
+                    documents=batch,
+                    embedding=self.embeddings,
+                    persist_directory=self.persist_directory,
+                )
+                self.vs_initialized = True
+            else:
+                self.vector_stores["chroma"].add_documents(batch)
+
+        self.vector_stores["bm25"] = BM25Retriever.from_documents(
+            documents, tokenizer=self.tokenizer
+        )
+
+    def initialize_vector_store(self, documents: List[Document] = None):
+        """
+        Initializes or loads the vector store.
+
+        Args:
+            documents (List[Document], optional): List of documents to initialize the vector store. Defaults to None.
+        """
+        if documents:
+            self._batch_process_documents(documents)
+        else:
+            self.vector_stores["chroma"] = Chroma(
+                collection_name=self.collection_name,
+                persist_directory=self.persist_directory,
+                embedding_function=self.embeddings,
+            )
+            all_documents = self.vector_stores["chroma"].get(
+                include=["documents", "metadatas"]
+            )
+            documents = [
+                Document(page_content=content, id=doc_id, metadata=metadata)
+                for content, doc_id, metadata in zip(
+                    all_documents["documents"],
+                    all_documents["ids"],
+                    all_documents["metadatas"],
+                )
+            ]
+            self.vector_stores["bm25"] = BM25Retriever.from_documents(documents)
+        self.vs_initialized = True
+
+    def create_retriever(
+        self, llm, n_documents: int, bm25_portion: float = 0.8
+    ) -> EnsembleRetriever:
+        """
+        Creates an ensemble retriever combining Chroma and BM25.
+
+        Args:
+            llm: Language model to use for retrieval.
+            n_documents (int): Number of documents to retrieve.
+            bm25_portion (float): Proportion of BM25 retriever in the ensemble.
+
+        Returns:
+            EnsembleRetriever: The created ensemble retriever.
+        """
+        self.vector_stores["bm25"].k = n_documents
+        self.vector_store = MultiQueryRetriever.from_llm(
+            retriever=EnsembleRetriever(
+                retrievers=[
+                    self.vector_stores["bm25"],
+                    self.vector_stores["chroma"].as_retriever(
+                        search_kwargs={"k": n_documents}
+                    ),
+                ],
+                weights=[bm25_portion, 1 - bm25_portion],
+            ),
+            llm=llm,
+            include_original=True,
+            prompt=DEFAULT_QUERY_PROMPT
+        )
+        return self.vector_store
+
+    def load_and_process_documents(self, doc_dir) -> List[Document]:
+        """
+        Loads and processes documents from the specified directory.
+
+        Returns:
+            List[Document]: List of loaded and processed documents.
+        """
+        loader = DocumentLoader(doc_dir)
+        return loader.load_documents()

src/vector_store/document_loader.py

@@ -0,0 +1,68 @@
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+from glob import glob
+from typing import List
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import DirectoryLoader, TextLoader
+from langchain_core.documents import Document
+from tqdm import tqdm
+
+
+def sanitize_metadata(metadata: dict) -> dict:
+    sanitized = {}
+    for key, value in metadata.items():
+        if isinstance(value, list):
+            sanitized[key] = ", ".join(value)
+        elif isinstance(value, (str, int, float, bool)):
+            sanitized[key] = value
+        else:
+            raise ValueError(
+                f"Unsupported metadata type for key '{key}': {type(value)}"
+            )
+    return sanitized
+
+
+class DocumentLoader:
+    """
+    Handles loading and splitting documents from directories.
+    """
+
+    def __init__(self, docs_dir: str):
+        self.docs_dir = docs_dir
+
+    def load_text_documents(self) -> List[Document]:
+        """Loads and splits text documents."""
+        loader = DirectoryLoader(self.docs_dir, glob="**/*.txt", loader_cls=TextLoader)
+        documents = loader.load()
+        splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+        return splitter.split_documents(documents)
+
+    def load_json_documents(self) -> List[Document]:
+        """Loads and processes JSON documents."""
+        files = glob(os.path.join(self.docs_dir, "*.json"))
+
+        def load_json_file(file_path):
+            with open(file_path, "r") as f:
+                data = json.load(f)["kwargs"]
+            return Document.model_validate(
+                {**data, "metadata": sanitize_metadata(data["metadata"])}
+            )
+
+        with ThreadPoolExecutor() as executor:
+            documents = list(
+                tqdm(
+                    executor.map(load_json_file, files),
+                    total=len(files),
+                    desc="Loading JSON documents",
+                )
+            )
+
+        return documents
+
+    def load_documents(self) -> List[Document]:
+        """Determines and loads documents based on file type."""
+        if glob(os.path.join(self.docs_dir, "*.json")):
+            return self.load_json_documents()
+        return self.load_text_documents()

src/vector_store/prompts.py

@@ -0,0 +1,23 @@
+from langchain_core.prompts.prompt import PromptTemplate
+
+DEFAULT_QUERY_PROMPT = PromptTemplate(
+    input_variables=["question"],
+    template="""You are an AI language model assistant tasked with generating alternative versions of a given user question. Your goal is to create 3 different perspectives on the original question to help retrieve relevant documents from a vector database. This approach aims to overcome some limitations of distance-based similarity search.
+
+When generating alternative questions, follow these guidelines:
+1. Maintain the core intent of the original question
+2. Use different phrasing, synonyms, or sentence structures
+3. Consider potential related aspects or implications of the question
+4. Avoid introducing new topics or drastically changing the subject matter
+
+Here is the original question:
+
+{question}
+
+Generate 3 alternative versions of this question. Provide your output as a numbered list, with each alternative question on a new line. Do not include any additional explanation or commentary.
+
+Remember, the purpose of these alternative questions is to broaden the search scope while staying relevant to the user's original intent. This will help in retrieving a diverse set of potentially relevant documents from the vector database.
+
+Do not include any additional explanation or commentary, just give 3 alternative questions.
+""",
+)

src/vector_store/vector_store.py

@@ -0,0 +1,114 @@
+import os
+from typing import List
+
+from langchain.retrievers import MultiQueryRetriever
+from langchain_chroma import Chroma
+from langchain_core.documents import Document
+from tqdm import tqdm
+
+from ..utilities.llm_models import get_llm_model_embedding
+from .document_loader import DocumentLoader
+from .prompts import DEFAULT_QUERY_PROMPT
+
+def get_collection_name() -> str:
+    """
+    Derives the collection name from an environment variable.
+
+    Returns:
+        str: Processed collection name.
+    """
+    return "medivocate-" + os.getenv("HF_MODEL", "default_model").split(":")[0].split("/")[-1]
+
+
+class VectorStoreManager:
+    """
+    Manages vector store initialization, updates, and retrieval.
+    """
+
+    def __init__(self, persist_directory: str, batch_size: int = 64):
+        """
+        Initializes the VectorStoreManager with the given parameters.
+
+        Args:
+            persist_directory (str): Directory to persist the vector store.
+            batch_size (int): Number of documents to process in each batch.
+        """
+        self.persist_directory = persist_directory
+        self.batch_size = batch_size
+        self.embeddings = get_llm_model_embedding()
+        self.collection_name = get_collection_name()
+        self.vector_stores: dict[str, Chroma] = {"chroma": None}
+        self.vs_initialized = False
+
+    def _batch_process_documents(self, documents: List[Document]):
+        """
+        Processes documents in batches for vector store initialization.
+
+        Args:
+            documents (List[Document]): List of documents to process.
+        """
+        for i in tqdm(
+            range(0, len(documents), self.batch_size), desc="Processing documents"
+        ):
+            batch = documents[i : i + self.batch_size]
+            if not self.vs_initialized:
+                self.vector_stores["chroma"] = Chroma.from_documents(
+                    collection_name=self.collection_name,
+                    documents=batch,
+                    embedding=self.embeddings,
+                    persist_directory=self.persist_directory,
+                )
+                self.vs_initialized = True
+            else:
+                self.vector_stores["chroma"].add_documents(batch)
+
+    def initialize_vector_store(self, documents: List[Document] = None):
+        """
+        Initializes or loads the vector store.
+
+        Args:
+            documents (List[Document], optional): List of documents to initialize the vector store with.
+        """
+        if documents:
+            self._batch_process_documents(documents)
+        else:
+            self.vector_stores["chroma"] = Chroma(
+                collection_name=self.collection_name,
+                persist_directory=self.persist_directory,
+                embedding_function=self.embeddings,
+            )
+        self.vs_initialized = True
+
+    def create_retriever(
+        self, llm, n_documents: int, bm25_portion: float = 0.8
+    ) -> MultiQueryRetriever:
+        """
+        Creates a retriever using Chroma.
+
+        Args:
+            llm: Language model to use for the retriever.
+            n_documents (int): Number of documents to retrieve.
+            bm25_portion (float): Portion of BM25 to use in the retriever.
+
+        Returns:
+            MultiQueryRetriever: Configured retriever.
+        """
+        self.vector_store = MultiQueryRetriever.from_llm(
+            retriever=self.vector_stores["chroma"].as_retriever(
+                search_kwargs={"k": n_documents}
+            ),
+            llm=llm,
+            include_original=True,
+            prompt=DEFAULT_QUERY_PROMPT
+        )
+        return self.vector_store
+
+    def load_and_process_documents(self, doc_dir: str) -> List[Document]:
+        """
+        Loads and processes documents from the specified directory.
+
+        Returns:
+            List[Document]: List of processed documents.
+        """
+        loader = DocumentLoader(doc_dir)
+        return loader.load_documents()