File size: 1,072 Bytes
2ce377c 44863bb 698b982 1c5311a 8fbd8e0 7602a4e 8fbd8e0 b7543fa 2ce377c c4d10a1 8fbd8e0 ff65140 8fbd8e0 ff65140 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import UnstructuredFileLoader
from langchain.vectorstores.faiss import FAISS
from langchain.embeddings import OpenAIEmbeddings
import pickle
import os
import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
def ingest_data(vector_file_path):
# Load Data
loader = UnstructuredFileLoader("cleaned_site_contents2023-02-24.txt")
raw_documents = loader.load()
# Split text
text_splitter = RecursiveCharacterTextSplitter()
documents = text_splitter.split_documents(raw_documents)
# Load Data to vectorstore
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_documents(documents, embeddings)
# Save vectorstore
with open(vector_file_path, "wb") as f:
pickle.dump(vectorstore, f)
return vector_file_path
def get_vectorstore(vector_file_path):
if os.path.isfile(vector_file_path):
return vector_file_path
else:
return ingest_data(vector_file_path)
|