Spaces:
Sleeping
Sleeping
arjunanand13
commited on
Commit
•
09192a6
1
Parent(s):
077ef59
Update app.py
Browse files
app.py
CHANGED
@@ -21,11 +21,13 @@ class DocumentRetrievalAndGeneration:
|
|
21 |
|
22 |
def load_documents(self, folder_path):
|
23 |
loader = DirectoryLoader(folder_path, loader_cls=TextLoader)
|
24 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=5000, chunk_overlap=250)
|
25 |
documents = loader.load()
|
|
|
26 |
all_splits = text_splitter.split_documents(documents)
|
27 |
print('Length of documents:', len(documents))
|
28 |
print("LEN of all_splits", len(all_splits))
|
|
|
|
|
29 |
return all_splits
|
30 |
|
31 |
def load_faiss_index(self, faiss_index_path):
|
|
|
21 |
|
22 |
def load_documents(self, folder_path):
|
23 |
loader = DirectoryLoader(folder_path, loader_cls=TextLoader)
|
|
|
24 |
documents = loader.load()
|
25 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=5000, chunk_overlap=250)
|
26 |
all_splits = text_splitter.split_documents(documents)
|
27 |
print('Length of documents:', len(documents))
|
28 |
print("LEN of all_splits", len(all_splits))
|
29 |
+
for i in range(5):
|
30 |
+
print(all_splits[i].page_content)
|
31 |
return all_splits
|
32 |
|
33 |
def load_faiss_index(self, faiss_index_path):
|