Spaces:
Paused
Paused
File size: 1,589 Bytes
bd2cf7b 9359dde bd2cf7b 7b6bc08 bd2cf7b ea780f0 9359dde bd2cf7b 9359dde |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
import streamlit as st
import pickle
import os
import torch
from tqdm.auto import tqdm
from langchain.text_splitter import RecursiveCharacterTextSplitter
# from langchain.vectorstores import Chroma
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain import HuggingFacePipeline
from langchain.chains import RetrievalQA
st.set_page_config(
page_title = 'aitGPT',
page_icon = '✅')
st.markdown("# Hello")
with open("ait-web-document", "rb") as fp:
ait_web_documents = pickle.load(fp)
text_splitter = RecursiveCharacterTextSplitter(
# Set a really small chunk size, just to show.
chunk_size = 500,
chunk_overlap = 100,
length_function = len,
)
chunked_text = text_splitter.create_documents([doc for doc in tqdm(ait_web_documents)])
st.markdown(f"Number of Documents: {len(ait_web_documents)}")
st.markdown(f"Number of chunked texts: {len(chunked_text)}")
embedding_model = HuggingFaceInstructEmbeddings(model_name='hkunlp/instructor-base',
model_kwargs = {'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu')})
vector_database = FAISS.load_local("faiss_index", embedding_model)
print("load done")
query_input = st.text_input(label= 'your question')
def retrieve_document(query_input):
related_doc = vector_database.similarity_search(query_input)
return related_doc
output = st.text_area(label = "Here is the relevant documents",
value = retrieve_document(query_input))
|