antfraia commited on
Commit
1859f3d
·
1 Parent(s): f7e9189

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +14 -3
app.py CHANGED
@@ -1,5 +1,5 @@
1
  import gradio as gr
2
- from langchain.document_loaders.base import Document
3
  from langchain.indexes import VectorstoreIndexCreator
4
  from apify_client import ApifyClient
5
  import os
@@ -31,10 +31,21 @@ def fetch_website_content(website_url):
31
  items = list(apify_client.dataset(run["defaultDatasetId"]).iterate_items())
32
  return items if items else None
33
 
34
- # Fetch and index website content
 
 
 
 
 
 
 
 
35
  content = fetch_website_content("https://python.langchain.com/en/latest/")
36
  documents = [Document(page_content=item["content"] or "", metadata={"source": item.get("url", "Unknown URL")}) for item in content]
37
- index = VectorstoreIndexCreator().from_loaders([documents])
 
 
 
38
 
39
  # Function for the Gradio UI
40
  def ask_langchain(question):
 
1
  import gradio as gr
2
+ from langchain.document_loaders.base import Document, BaseLoader
3
  from langchain.indexes import VectorstoreIndexCreator
4
  from apify_client import ApifyClient
5
  import os
 
31
  items = list(apify_client.dataset(run["defaultDatasetId"]).iterate_items())
32
  return items if items else None
33
 
34
+ # Custom loader for our documents
35
+ class CustomLoader(BaseLoader):
36
+ def __init__(self, documents):
37
+ self.documents = documents
38
+
39
+ def load(self):
40
+ return self.documents
41
+
42
+ # Fetch content
43
  content = fetch_website_content("https://python.langchain.com/en/latest/")
44
  documents = [Document(page_content=item["content"] or "", metadata={"source": item.get("url", "Unknown URL")}) for item in content]
45
+
46
+ # Use custom loader
47
+ loader = CustomLoader(documents)
48
+ index = VectorstoreIndexCreator().from_loaders([loader])
49
 
50
  # Function for the Gradio UI
51
  def ask_langchain(question):