first commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,119 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.idea/*
+.files/*
+
+tmp
+secret.*
+volumes/
+.chainlit
+.DS_Store
+__init__.py
+data/
+data*
+record_manager_cache.sql

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+COPY ./requirements.txt ~/app/requirements.txt
+RUN pip install -r requirements.txt
+COPY --chown=user . .
+RUN --mount=type=secret,id=GOOGLE_API_KEY,mode=0444,required=true \
+    export GOOGLE_API_KEY=$(cat /run/secrets/GOOGLE_API_KEY) &&\
+    python scrape_data.py
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

README.md

@@ -0,0 +1,25 @@
+---
+title: FinChat
+emoji: 🤑
+colorFrom: yellow
+colorTo: purple
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+# FinChat
+
+FinChat est un chatbot conçu par [data354](https://data354.com/) pour répondre aux questions sur l'actualité économique et financière. 
+
+
+## How to run ?
+
+1. Executer le script pour scraper et stoker les données:
+```shell
+python scrape_data.py
+```
+2. Lancez la démo et commencer à interagir avec l'agent.
+
+```shell
+chainlit run app.py 

app.py

@@ -0,0 +1,121 @@
+import chainlit as cl
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.chains.query_constructor.schema import AttributeInfo
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.schema import StrOutputParser
+from langchain.schema.runnable import Runnable, RunnableConfig, RunnablePassthrough
+from langchain.vectorstores.chroma import Chroma
+from langchain_google_genai import (
+    GoogleGenerativeAI,
+    GoogleGenerativeAIEmbeddings,
+    HarmBlockThreshold,
+    HarmCategory,
+)
+
+import config
+from prompts import prompt
+
+metadata_field_info = [
+    AttributeInfo(
+        name="title",
+        description="Le titre de l'article",
+        type="string",
+    ),
+    AttributeInfo(
+        name="date",
+        description="Date de publication",
+        type="string",
+    ),
+    AttributeInfo(name="link", description="Source de l'article", type="string"),
+]
+document_content_description = "Articles sur l'actualité."
+
+model = GoogleGenerativeAI(
+    model=config.GOOGLE_CHAT_MODEL,
+    google_api_key=config.GOOGLE_API_KEY,
+    safety_settings={
+        HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+    },
+)  # type: ignore
+
+# Load vector database that was persisted earlier
+embedding = embeddings_model = GoogleGenerativeAIEmbeddings(
+    model="models/embedding-001", google_api_key=config.GOOGLE_API_KEY
+)  # type: ignore
+
+vectordb = Chroma(persist_directory=config.STORAGE_PATH, embedding_function=embedding)
+
+retriever = SelfQueryRetriever.from_llm(
+    model,
+    vectordb,
+    document_content_description,
+    metadata_field_info,
+)
+
+
+@cl.on_chat_start
+async def on_chat_start():
+
+    def format_docs(docs):
+        return "\n\n".join(doc.page_content for doc in docs)
+
+    rag_chain = (
+        {
+            "context": vectordb.as_retriever() | format_docs,
+            "question": RunnablePassthrough(),
+        }
+        | prompt
+        | model
+        | StrOutputParser()
+    )
+
+    cl.user_session.set("rag_chain", rag_chain)
+
+    msg = cl.Message(
+        content=f"Vous pouvez poser vos questions sur les articles de SIKAFINANCE",
+    )
+    await msg.send()
+
+
+@cl.on_message
+async def on_message(message: cl.Message):
+    runnable = cl.user_session.get("rag_chain")  # type: Runnable # type: ignore
+    msg = cl.Message(content="")
+
+    class PostMessageHandler(BaseCallbackHandler):
+        """
+        Callback handler for handling the retriever and LLM processes.
+        Used to post the sources of the retrieved documents as a Chainlit element.
+        """
+
+        def __init__(self, msg: cl.Message):
+            BaseCallbackHandler.__init__(self)
+            self.msg = msg
+            self.sources = []
+
+        def on_retriever_end(self, documents, *, run_id, parent_run_id, **kwargs):
+            for d in documents:
+                source_doc = d.page_content + "\nSource: " + d.metadata["link"]
+                self.sources.append(source_doc)
+
+        def on_llm_end(self, response, *, run_id, parent_run_id, **kwargs):
+            if len(self.sources):
+                # Display the reference docs with a Text widget
+                sources_element = [
+                    cl.Text(name=f"source_{idx+1}", content=content)
+                    for idx, content in enumerate(self.sources)
+                ]
+                source_names = [el.name for el in sources_element]
+                self.msg.elements += sources_element
+                self.msg.content += f"\nSources: {', '.join(source_names)}"
+
+    async with cl.Step(type="run", name="QA Assistant"):
+        async for chunk in runnable.astream(
+            message.content,
+            config=RunnableConfig(
+                callbacks=[cl.LangchainCallbackHandler(), PostMessageHandler(msg)]
+            ),
+        ):
+            await msg.stream_token(chunk)
+
+    await msg.send()

chainlit.md

@@ -0,0 +1,5 @@
+# Welcome to FinChat! 🚀🤖
+
+FinChat est un chatbot conçu par [data354](https://data354.com/) pour répondre aux questions sur l'actualité économique et financière. 
+
+> Voilà, c'est fait ! Vous pouvez maintenant posez vos questions 💻😊.

config.py

@@ -0,0 +1,7 @@
+import os
+
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+GOOGLE_CHAT_MODEL = "gemini-pro"
+GOOGLE_EMBEDDING_MODEL = "models/embedding-001"
+STORAGE_PATH = "data/chroma/"
+HIISTORY_FILE = "./data/qa_history.txt"

prompts.py

@@ -0,0 +1,12 @@
+from langchain.prompts import ChatPromptTemplate
+
+template = """
+Répondez à la question en vous basant uniquement sur le contexte suivant:
+
+{context}
+
+Question : {question}
+
+"""
+
+prompt = ChatPromptTemplate.from_template(template)

requirements.txt

@@ -0,0 +1,7 @@
+langchain==0.1.14
+langchain-google-genai==1.0.1
+chainlit==1.0.500
+chromadb==0.4.24
+lark==1.1.9
+bs4==0.0.2
+selenium==4.19.0

scrape_data.py

@@ -0,0 +1,149 @@
+import os
+from datetime import date, timedelta
+
+import bs4
+from langchain.indexes import SQLRecordManager, index
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores.chroma import Chroma
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+import config
+
+DATA_URL = "https://www.sikafinance.com/marches/actualites_bourse_brvm"
+
+embeddings_model = GoogleGenerativeAIEmbeddings(
+    model=config.GOOGLE_EMBEDDING_MODEL
+)  # type: ignore
+
+
+options = webdriver.ChromeOptions()
+options.add_argument("--headless")
+options.add_argument("--no-sandbox")
+options.add_argument("--disable-dev-shm-usage")
+driver = webdriver.Chrome(options=options)
+
+
+def scrap_articles(
+    url="https://www.sikafinance.com/marches/actualites_bourse_brvm", num_days_past=5
+):
+
+    today = date.today()
+
+    driver.get(url)
+
+    all_articles = []
+    for i in range(num_days_past + 1):
+        past_date = today - timedelta(days=i)
+        date_str = past_date.strftime("%Y-%m-%d")
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.ID, "dateActu"))
+        )
+        text_box = driver.find_element(By.ID, "dateActu")
+        text_box.send_keys(date_str)
+
+        submit_btn = WebDriverWait(driver, 10).until(
+            EC.element_to_be_clickable((By.ID, "btn"))
+        )
+        submit_btn.click()
+
+        dates = driver.find_elements(By.CLASS_NAME, "sp1")
+        titles = driver.find_elements(By.XPATH, "//td/a")
+
+        articles = []
+        for i in range(len(titles)):
+            art = {
+                "title": titles[i].text.strip(),
+                "date": dates[i].text,
+                "link": titles[i].get_attribute("href"),
+            }
+            articles.append(art)
+
+        all_articles += articles
+    # driver.quit()
+
+    return all_articles
+
+
+def set_metadata(documents, metadatas):
+    """
+    #Edit a metadata of lanchain Documents object
+    """
+    for doc in documents:
+        idx = documents.index(doc)
+        doc.metadata = metadatas[idx]
+    print("Metadata successfully changed")
+    print(documents[0].metadata)
+
+
+def process_docs(
+    articles, persist_directory, embeddings_model, chunk_size=1000, chunk_overlap=100
+):
+    """
+    #Scrap all articles urls content and save on a vector DB
+    """
+    article_urls = [a["link"] for a in articles]
+
+    print("Starting to scrap ..")
+
+    loader = WebBaseLoader(
+        web_paths=article_urls,
+        bs_kwargs=dict(
+            parse_only=bs4.SoupStrainer(
+                class_=("inarticle txtbig", "dt_sign", "innerUp")
+            )
+        ),
+    )
+
+    print("After scraping Loading ..")
+    docs = loader.load()
+
+    # Update metadata: add title,
+    set_metadata(documents=docs, metadatas=articles)
+
+    print("Successfully loaded to document")
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n"]
+    )
+    splits = text_splitter.split_documents(docs)
+
+    # Create the storage path if it doesn't exist
+    if not os.path.exists(persist_directory):
+        os.makedirs(persist_directory)
+
+    doc_search = Chroma.from_documents(
+        documents=splits,
+        embedding=embeddings_model,
+        persist_directory=persist_directory,
+    )
+
+    # Indexing data
+    namespace = "chromadb/my_documents"
+    record_manager = SQLRecordManager(
+        namespace, db_url="sqlite:///record_manager_cache.sql"
+    )
+    record_manager.create_schema()
+
+    index_result = index(
+        docs,
+        record_manager,
+        doc_search,
+        cleanup="incremental",
+        source_id_key="link",
+    )
+
+    print(f"Indexing stats: {index_result}")
+
+    return doc_search
+
+
+if __name__ == "__main__":
+
+    data = scrap_articles(DATA_URL, num_days_past=2)
+    vectordb = process_docs(data, config.STORAGE_PATH, embeddings_model)
+    ret = vectordb.as_retriever()