embed_docs.py

from langchain_community.vectorstores import FAISS
import os
from datetime import datetime

vector_store_path = "/home/user/VectorStoreDB"
index_name = "faiss_index"
full_index_path = os.path.join(vector_store_path, index_name)
start = ""
end = ""


def embed_docs(documents, embedder):

    # Ensure the directory exists
    os.makedirs(vector_store_path, exist_ok=True)

    # just query if it exists
    if os.path.exists(full_index_path):
        print(f"Loading existing vector store at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        saved_vector = FAISS.load_local(full_index_path, 
                                        embeddings=embedder, 
                                        allow_dangerous_deserialization=True)

        return saved_vector
    else:
        print(f"Embedding documents at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        embedded_vector = FAISS.from_documents(documents=documents, embedding=embedder)
        embedded_vector.save_local(full_index_path)
        print(f"Vector store saved at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        
        return embedded_vector