Update app.py

app.py

@@ -1,75 +1,42 @@
-import os 
-#os.system('pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html')
+import os
+import openai
+import pinecone
+from langchain.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Pinecone
+from langchain.llms import OpenAI
+from langchain.chains.question_answering import load_qa_chain
 
-import transformers
 import streamlit as st
 
-from transformers import AutoTokenizer, AutoModelWithLMHead
-from transformers import pipeline
+st.header("Document Question Answering")
 
-sentiment_tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment")
-    
-def load_text_gen_model():
-    generator = pipeline("text-generation", model="gpt2-medium")
-    return generator 
-    
-@st.cache
-def get_sentiment_model():
-    sentiment_model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment")
-    return sentiment_model 
-
-def get_summarizer_model():
-    summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
-    return summarizer
+directory = st.text_area("")
 
-      
-def get_sentiment(text):
-    input_ids = sentiment_tokenizer .encode(text + '</s>', return_tensors='pt')
-    output = sentiment_extractor.generate(input_ids=input_ids,max_length=2)
-    dec = [sentiment_tokenizer.decode(ids) for ids in output]
-    label = dec[0]
-    return label
-    
-
-def get_qa_model():
-    model_name = "deepset/roberta-base-squad2"
+#directory = '/content/data'
 
-    qa_pipeline = pipeline('question-answering', model=model_name, tokenizer=model_name)
-    return qa_pipeline
-        
-sentiment_extractor   = get_sentiment_model()
-summarizer = get_summarizer_model()
-answer_generator = get_qa_model()
+def load_docs(directory):
+  loader = DirectoryLoader(directory)
+  documents = loader.load()
+  return documents
 
 
-st.header("Review Analyzer")
+def split_docs(documents, chunk_size=1000, chunk_overlap=20):
+  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+  docs = text_splitter.split_documents(documents)
+  return docs
 
-#action = st.sidebar.selectbox("Pick an Action", ["Analyse a Review","Generate an Article","Create an Image"])
 
-#if action == "Analyse a Review":
-st.subheader("Paste/write a review here..")
-review = st.text_area("")
+if directory:
+    documents = load_docs(directory)
+    st.write(len(documents))
 
-if review:
-    
-    start_sentiment_analysis = st.button("Get the Sentiment of the Review")
-    start_summarizing = st.button("Summarize the review")
-    start_topic_extraction = st.button("Find the key topic")
+    docs = split_docs(documents)
+    print(len(docs))
 
-    if start_sentiment_analysis:
-        sentiment = get_sentiment(review)
-        st.write(sentiment)
-        
-    if start_summarizing:
-        summary = summarizer(review, max_length=130, min_length=30, do_sample=False)
-        st.write(summary)
+    embeddings = OpenAIEmbeddings(model_name="ada")
     
-    if start_topic_extraction:
-        QA_input = {'question': 'what is the review about?',
-                    'context': review}
-        answer = answer_generator(QA_input)
-        st.write(answer)
+    query_result = embeddings.embed_query("Hello world")
+    st.write(len(query_result))
 
-        
-         
-