First commit

app.py

@@ -1,4 +1,83 @@
+import os
 import streamlit as st
+import time
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+from langchain_openai import ChatOpenAI  # Nouvel import pour OpenAI
+from langchain.prompts import PromptTemplate
+from pinecone import Pinecone, ServerlessSpec
+
+from vectore_store import get_retreive_answer
+
+
+PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY")
+index_name = os.environ.get("PINECONE_INDEX_NAME")
+
+pc = Pinecone(api_key=PINECONE_API_KEY)
+
+
+existing_indexes = [index_info["name"] for index_info in pc.list_indexes()]
+
+if index_name not in existing_indexes:
+    pc.create_index(
+        name=index_name,
+        dimension=3072,
+        metric="cosine",
+        spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+    )
+    while not pc.describe_index(index_name).status["ready"]:
+        time.sleep(1)
+
+index = pc.Index(index_name)
+
+# Créez un modèle OpenAI
+llm = ChatOpenAI(model="gpt-4o-mini")
+
+# Créez un template de prompt pour l'IA
+prompt = PromptTemplate(
+    input_variables=["question"],
+    template="Voici une question : {question}. Réponds de manière concise et claire."
+)
+
+# Chaîne qui combine le modèle LLM et le prompt
+llm_chain = prompt | llm  # Nouvelle façon d'utiliser LLMChain
+
+# Initialiser l'historique s'il n'existe pas déjà
+if "history" not in st.session_state:
+    st.session_state["history"] = []
+
+# Interface Streamlit
+st.title("Application LLM avec LangChain")
+st.write("Posez une question au modèle de langage et obtenez une réponse.")
+
+# Champ pour entrer la question
+question = st.text_input("Votre question :")
+
+# Bouton pour soumettre la question
+if st.button("Envoyer"):
+    if question:
+
+        context = get_retreive_answer(index, namespace="chat", prompt=question)
+        if not context:
+            print("no context found for prompt reormatting")
+
+        for chunk in context:
+            print(chunk.metadata)
+            chunk_name = chunk.metadata["filename"]
+
+
+        # Utilisation de la chaîne pour obtenir une réponse
+        response = llm_chain.invoke({"question": question, "context": context})  # Utilisation de invoke() au lieu de run()
+        
+        # Ajouter la question et la réponse à l'historique
+        st.session_state["history"].append({"question": question, "response": response.content})
+    else:
+        st.write("Veuillez poser une question.")
+
+# Affichage de l'historique des échanges
+if st.session_state["history"]:
+    st.write("### Historique des échanges")
+    for i, exchange in enumerate(st.session_state["history"]):
+        st.write(f"**Échange {i + 1}**")
+        st.write(f"**Question :** {exchange['question']}")
+        st.write(f"**Réponse :** {exchange['response']}")
+        st.write("---")

requirements.txt

@@ -0,0 +1,8 @@
+streamlit
+langchain
+langchain-core 
+langchain-pinecone
+langchain-pinecone 
+pinecone-notebooks
+pinecone-client[grpc]
+pandas

vectore_store.py

@@ -0,0 +1,206 @@
+import os
+import json
+import pandas as pd
+from langchain_openai import OpenAIEmbeddings
+from langchain_pinecone import PineconeVectorStore
+from langchain_core.documents import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def get_text_chunks(text):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500, # the character length of the chunck
+        chunk_overlap=100, # the character length of the overlap between chuncks
+        length_function=len # the length function - in this case, character length (aka the python len() fn.)
+    )
+    chunks = text_splitter.split_text(text)
+    return chunks
+
+
+def get_vectorstore_from_csv(csv_file, text_columns, namespace, index):
+    """
+    Function to process CSV with multiple text columns and insert embeddings into Pinecone.
+    
+    Parameters:
+    - csv_file: Path to the CSV file.
+    - text_columns: List of column names that contain text data to be embedded.
+    - namespace: Pinecone namespace to store the vectors.
+    - index: Pinecone index object.
+    
+    Returns:
+    - dict with filename_id upon success or False on failure.
+    """
+    try:
+        # Load the CSV file into a DataFrame
+        df = pd.read_csv(csv_file)
+        
+        # Ensure the specified text columns exist
+        for column in text_columns:
+            if column not in df.columns:
+                raise ValueError(f"'{column}' column not found in the CSV file")
+
+        # Initialize the OpenAI embedding model
+        embedding = OpenAIEmbeddings(model="text-embedding-3-large")  # Replace with your model choice
+        vector_store = PineconeVectorStore(index=index, embedding=embedding, namespace=namespace)
+
+        # Clean up filename (for unique ids)
+        filename = csv_file.split(".")[0]
+        clean_filename = filename.replace(" ", "_").replace("-", "_").replace(".", "_").replace("/", "_").replace("\\", "_").strip()
+
+        # Prepare documents and UUIDs
+        documents = []
+        uuids = []
+        combined_text = " "
+
+        # Iterate through each row in the CSV and process the text columns
+        for i, row in df.iterrows():
+            # Concatenate text from the specified columns
+            combined_text += " ".join(str(row[col]) for col in text_columns)
+
+        text_chunks = get_text_chunks(combined_text)
+
+        print(text_chunks)
+
+        for i, chunk in enumerate(text_chunks):
+
+            # Create a Document object for each combined text chunk
+            document = Document(
+                page_content=chunk,
+                metadata={"filename": csv_file, "filename_id": clean_filename}
+            )
+
+            # Generate a unique ID for each document (row)
+            uuid = f"{clean_filename}_{i}"
+            uuids.append(uuid)
+            documents.append(document)
+            
+        # Insert the documents into Pinecone
+        vector_store.add_documents(documents=documents, ids=uuids)
+
+        # Return filename ID after successful insertion
+        return {"filename_id": clean_filename}
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return False
+    
+
+def get_vectorstore(text_chunks, filename, namespace, index):
+    try:
+        embedding = OpenAIEmbeddings(model="text-embedding-3-large")
+        vector_store = PineconeVectorStore(index=index, embedding=embedding,namespace=namespace)
+
+        file_name = filename.split(".")[0].replace(" ","_").replace("-","_").replace(".","_").replace("/","_").replace("\\","_").strip()
+        clean_filename = file_name
+
+        documents = []
+        uuids = []
+
+        for i, chunk in enumerate(text_chunks):
+
+            document = Document(
+                page_content=chunk,
+                metadata={"filename":filename, "filename_id":clean_filename},
+            )
+    
+            uuid = f"{clean_filename}_{i}"
+            uuids.append(uuid)
+            documents.append(document)
+        
+        vector_store.add_documents(documents=documents, ids=uuids)
+
+        return {"filename_id":clean_filename}
+    
+    except Exception as e:
+        print(e)
+        return False
+    
+def get_vectorstore_from_json(json_file, namespace, index):
+    """
+    Fonction pour traiter un fichier JSON et insérer les embeddings dans Pinecone.
+    
+    Parameters:
+    - json_file: Chemin vers le fichier JSON.
+    - namespace: Namespace Pinecone pour stocker les vecteurs.
+    - index: Objet d'index Pinecone.
+    
+    Returns:
+    - dict avec filename_id en cas de succès ou False en cas d'échec.
+    """
+    try:
+        # Charger le fichier JSON dans une liste de dictionnaires
+        with open(json_file, 'r', encoding='utf-8') as file:
+            data = json.load(file)
+
+        # print(data)
+        # Initialiser le modèle d'embeddings OpenAI
+        embedding = OpenAIEmbeddings(model="text-embedding-3-large")  # Remplacez par votre choix de modèle
+        vector_store = PineconeVectorStore(index=index, embedding=embedding, namespace=namespace)
+
+        # Nettoyer le nom de fichier (pour des IDs uniques)
+        filename = json_file.split(".")[0]
+        clean_filename = filename.replace(" ", "_").replace("-", "_").replace(".", "_").replace("/", "_").replace("\\", "_").strip()
+
+        # Préparer les documents et UUIDs
+        documents = []
+        uuids = []
+
+        # # Itérer à travers chaque entrée du JSON
+        # for i, entry in enumerate(data):
+        #     # Concaténer toutes les propriétés de l'entrée
+        #     combined_text = " ".join(str(value) for value in entry.values())
+
+        text_chunks = get_text_chunks(json.dumps(data, indent=4))
+
+        for j, chunk in enumerate(text_chunks):
+            # Créer un objet Document pour chaque morceau de texte combiné
+            document = Document(
+                page_content=chunk,
+                metadata={"filename": json_file, "filename_id": clean_filename, "chunk_index": j}  # Ajout de toutes les propriétés de l'entrée
+            )
+
+            # Générer un ID unique pour chaque document (entrée)
+            uuid = f"{clean_filename}_{j}"  # Ajouter l'index à l'ID unique
+            uuids.append(uuid)
+            documents.append(document)
+
+        # Insérer les documents dans Pinecone
+        vector_store.add_documents(documents=documents, ids=uuids)
+
+        # Retourner l'ID de fichier après insertion réussie
+        return {"filename_id": clean_filename}
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return False
+    
+
+def get_retreive_answer(index,namespace,prompt):
+    try:
+        
+        embedding = OpenAIEmbeddings(model="text-embedding-3-large")
+        vector_store_chat = PineconeVectorStore(index=index, embedding=embedding,namespace=namespace)
+
+        retriever_chat = vector_store_chat.as_retriever(
+            search_type="similarity_score_threshold",
+            search_kwargs={"k": 3, "score_threshold": 0.6},
+        )
+        
+        vector_store_tic = PineconeVectorStore(index=index, embedding=embedding,namespace=namespace)
+
+        retriever_tic = vector_store_tic.as_retriever(
+            search_type="similarity_score_threshold",
+            search_kwargs={"k": 3, "score_threshold": 0.6},
+        )
+
+        response = retriever_chat.invoke(prompt) + retriever_tic.invoke(prompt)
+
+        if response:
+            print("found enterprise context")
+        else:
+            print("no enterprise context")
+
+        return response
+    
+    except Exception as e:
+        print(e)
+        return False