Spaces:
Sleeping
Sleeping
File size: 657 Bytes
7590ece |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
# %%
from langchain.document_loaders import DirectoryLoader, TextLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import FAISS
# %%
loader = DirectoryLoader(
"../../../docs", glob="./*.md", loader_cls=TextLoader, recursive=True
)
documents = loader.load()
# %%
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
texts = text_splitter.split_documents(documents)
# %%
embedding = OpenAIEmbeddings()
vectordb = FAISS.from_documents(documents=texts, embedding=embedding)
# %%
vectordb.save_local("docs.faiss")
# %%
|