Update utils.py

utils.py

@@ -117,17 +117,7 @@ def create_chunk_documents():
 
     source_chunks = splitter.split_documents(sources)
 
-    for chunk in source_chunks:
-        print("Size of chunk: " + str(len(chunk.page_content) + len(chunk.metadata)))
-        if chunk.page_content is None or chunk.page_content == '':
-            print("removing chunk: "+ chunk.page_content)
-            source_chunks.remove(chunk)
-        elif len(chunk.page_content) >=1000:
-            print("splitting document")
-            source_chunks.extend(splitter.split_documents([chunk]))
-    # print("Chunks: " + str(len(source_chunks)) + "and type " + str(type(source_chunks)))
-    return source_chunks
-
+   
 
 def fetch_data_for_embeddings(url, book_file, book_url):
     sources = get_website_data(url)
@@ -176,7 +166,8 @@ def get_links(index_url, paths):
     for path in paths:
         url = urljoin(index_url, path)
         parsed_url = urlparse(url)
-        if parsed_url.scheme in ["http", "https"] and "squarespace" not in parsed_url.netloc:
+        if parsed_url.scheme in ["http", "https"] and "shreyasachdev" in parsed_url.netloc:
+ 
             links.append(url)
     return links