colpali fix

pages/Multimodal_Conversational_Search.py

@@ -15,16 +15,12 @@ import random
 import string
 import rag_DocumentLoader 
 import rag_DocumentSearcher
-#import colpali
 import pandas as pd
 from PIL import Image 
 import shutil
 import base64
 import time
 import botocore
-#from langchain.callbacks.base import BaseCallbackHandler
-#import streamlit_nested_layout
-#from IPython.display import clear_output, display, display_markdown, Markdown
 from requests_aws4auth import AWS4Auth
 import colpali
 from requests.auth import HTTPBasicAuth
@@ -41,14 +37,14 @@ USER_ICON = "images/user.png"
 AI_ICON = "images/opensearch-twitter-card.png"
 REGENERATE_ICON = "images/regenerate.png"
 s3_bucket_ = "pdf-repo-uploads"
-            #"pdf-repo-uploads"
+           
 # polly_client = boto3.Session(
 #             region_name='us-east-1').client('polly')
 
 # Check if the user ID is already stored in the session state
 if 'user_id' in st.session_state:
     user_id = st.session_state['user_id']
-    #print(f"User ID: {user_id}")
+    
 
 # If the user ID is not yet stored in the session state, generate a random UUID
 else:
@@ -105,10 +101,6 @@ if "input_query" not in st.session_state:
         st.session_state.input_query="How many aged above 85 years died due to covid ?"#"What is the projected energy percentage from renewable sources in future?"#"Which city in United Kingdom has the highest average housing price ?"#"How many aged above 85 years died due to covid ?"# What is the projected energy from renewable sources ?"
 
 
-    
-
-
-
 st.markdown("""
     <style>
     [data-testid=column]:nth-of-type(2) [data-testid=stVerticalBlock]{
@@ -120,43 +112,11 @@ st.markdown("""
     </style>
     """,unsafe_allow_html=True)
 
-################ OpenSearch Py client #####################
-    
-# credentials = boto3.Session().get_credentials()
-# awsauth = AWSV4SignerAuth(credentials, region, service)
-
-# ospy_client = OpenSearch(
-#     hosts = [{'host': 'search-opensearchservi-75ucark0bqob-bzk6r6h2t33dlnpgx2pdeg22gi.us-east-1.es.amazonaws.com', 'port': 443}],
-#     http_auth = awsauth,
-#     use_ssl = True,
-#     verify_certs = True,
-#     connection_class = RequestsHttpConnection,
-#     pool_maxsize = 20
-# )
-
 ################# using boto3 credentials ###################
 
 awsauth = HTTPBasicAuth('master',st.secrets['ml_search_demo_api_access'])
 
 
-################# using boto3 credentials ####################
-
-
-
-# if "input_searchType" not in st.session_state:
-#     st.session_state.input_searchType = "Conversational Search (RAG)"
-
-# if "input_temperature" not in st.session_state:
-#     st.session_state.input_temperature = "0.001"
-
-# if "input_topK" not in st.session_state:
-#     st.session_state.input_topK = 200
-
-# if "input_topP" not in st.session_state:
-#     st.session_state.input_topP = 0.95
-
-# if "input_maxTokens" not in st.session_state:
-#     st.session_state.input_maxTokens = 1024
 
 
 def write_logo():
@@ -186,20 +146,14 @@ if clear:
     st.session_state.questions_ = []
     st.session_state.answers_ = []
     st.session_state.input_query=""
-    # st.session_state.input_searchType="Conversational Search (RAG)"
-    # st.session_state.input_temperature = "0.001"
-    # st.session_state.input_topK = 200
-    # st.session_state.input_topP = 0.95
-    # st.session_state.input_maxTokens = 1024
+
 
 
 def handle_input(state,dummy):
     if(state == 'colpali_show_similarity_map'):
         st.session_state.show_columns = True    
-    # st.session_state.answer_ready = True
-    # st.session_state.show_columns = False  # reset column display
     print("Question: "+st.session_state.input_query)
-    print("-----------")
+    print("-"*20)
     print("\n\n")
     # if(st.session_state.input_query==''):
     #     return ""
@@ -234,8 +188,6 @@ def write_user_message(md):
     with col1:
         st.image(USER_ICON, use_column_width='always')
     with col2:
-        #st.warning(md['question'])
-
         st.markdown("<div style='color:#e28743';font-size:18px;padding:3px 7px 3px 7px;borderWidth: 0px;borderColor: red;borderStyle: solid;width: fit-content;height: fit-content;border-radius: 10px;font-style: italic;'>"+md['question']+"</div>", unsafe_allow_html = True)
        
 
@@ -251,25 +203,12 @@ def render_answer(question,answer,index,res_img):
         st.write(ans_)
         rdn_key_1 = ''.join([random.choice(string.ascii_letters)
                               for _ in range(10)])
-        # def show_maxsim():
-        #     st.session_state.show_columns = True
-        #     # st.session_state.input_query = st.session_state.questions_[-1]["question"]
-        #     # st.session_state.answers_.pop()
-        #     # st.session_state.questions_.pop()
-        #     handle_input()
-        #     print("*"*20)
-        #     print(st.session_state.input_query) 
-        #     print(st.session_state.answers_)
-        #     print(st.session_state.questions_)
-        #     print("*"*20) 
-        #     with placeholder.container():
-        #         render_all() 
+
             
         if(st.session_state.input_is_colpali):
             placeholder__ = st.empty()
             placeholder__.button("Show similarity map",key=rdn_key_1,on_click=handle_input,args=('colpali_show_similarity_map',True))
-            # with placeholder.container():
-            #     render_all()    
+   
 
     colu1,colu2,colu3 = st.columns([4,82,20])
     with colu2:
@@ -320,19 +259,13 @@ def render_answer(question,answer,index,res_img):
                     st.table(df)
                 #with st.expander("Raw sources:"):
                 st.write(answer["source"])
-                
-        
-        
     with col_3:
         if(index == len(st.session_state.questions_)):
 
             rdn_key = ''.join([random.choice(string.ascii_letters)
                               for _ in range(10)])
-            # rdn_key_1 = ''.join([random.choice(string.ascii_letters)
-            #                   for _ in range(10)])
             currentValue = ''.join(st.session_state.input_rag_searchType)+str(st.session_state.input_is_rerank)+str(st.session_state.input_table_with_sql)+st.session_state.input_index
             oldValue = ''.join(st.session_state.inputs_["rag_searchType"])+str(st.session_state.inputs_["is_rerank"])+str(st.session_state.inputs_["table_with_sql"])+str(st.session_state.inputs_["index"])
-            #print("changing values-----------------")
             def on_button_click():
                 if(currentValue!=oldValue or 1==1): 
                     st.session_state.input_query = st.session_state.questions_[-1]["question"]
@@ -341,14 +274,7 @@ def render_answer(question,answer,index,res_img):
                     handle_input("regenerate_",None)
                     with placeholder.container():
                         render_all()
-                    
-                    
-            # def show_maxsim():
-            #     st.session_state.show_columns = True
-            #     st.session_state.maxSimImages = colpali.img_highlight(st.session_state.top_img, st.session_state.query_token_vectors, st.session_state.query_tokens)
-            #     handle_input()
-            #     with placeholder.container():
-            #         render_all()
+            
                 
 
             if("currentValue"  in st.session_state):
@@ -361,8 +287,7 @@ def render_answer(question,answer,index,res_img):
 
             placeholder__ = st.empty()
             placeholder__.button("🔄",key=rdn_key,on_click=on_button_click)
-            #placeholder__.button("Show similarity map",key=rdn_key_1,on_click = show_maxsim)
-        
+            
         
      
 #Each answer will have context of the question asked in order to associate the provided feedback with the respective question
@@ -371,8 +296,6 @@ def write_chat_message(md, q,index):
     #st.session_state['session_id'] = res['session_id']   to be added in memory
     chat = st.container()
     with chat:
-        #print("st.session_state.input_index------------------")
-        #print(st.session_state.input_index)
         render_answer(q,md,index,res_img)
     
 def render_all():  
@@ -389,12 +312,8 @@ with placeholder.container():
 
 st.markdown("")
 col_2, col_3 = st.columns([75,20])
-#col_1, col_2, col_3 = st.columns([7.5,71.5,22])
-# with col_1:
-#     st.markdown("<p style='padding:0px 0px 0px 0px; color:#FF9900;font-size:120%'><b>Ask:</b></p>",unsafe_allow_html=True, help = 'Enter the questions and click on "GO"')
     
 with col_2:
-    #st.markdown("")
     input = st.text_input( "Ask here",label_visibility = "collapsed",key="input_query")
 with col_3:
     #hidden = st.button("RUN",disabled=True,key = "hidden")
@@ -403,12 +322,6 @@ with st.sidebar:
     st.page_link("app.py", label=":orange[Home]", icon="🏠")
     st.subheader(":blue[Sample Data]")
     coln_1,coln_2 = st.columns([70,30])
-    # index_select = st.radio("Choose one index",["UK Housing","Covid19 impacts on Ireland","Environmental Global Warming","BEIR Research"],
-    #                         captions = ['[preview](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/HPI-Jan-2024-Hometrack.pdf)',
-    #                                     '[preview](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/covid19_ie.pdf)',
-    #                                     '[preview](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/global_warming.pdf)',
-    #                                     '[preview](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/BEIR.pdf)'],
-    #                         key="input_rad_index")
     with coln_1:
         index_select = st.radio("Choose one index",["UK Housing","Global Warming stats","Covid19 impacts on Ireland"],key="input_rad_index")
     with coln_2:
@@ -416,7 +329,6 @@ with st.sidebar:
         st.write("[:eyes:](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/HPI-Jan-2024-Hometrack.pdf)")
         st.write("[:eyes:](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/global_warming.pdf)")
         st.write("[:eyes:](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/covid19_ie.pdf)")
-        #st.write("[:eyes:](https://github.com/aws-samples/AI-search-with-amazon-opensearch-service/blob/b559f82c07dfcca973f457c0a15d6444752553ab/rag/sample_pdfs/BEIR.pdf)")
     st.markdown("""
     <style>
     [data-testid=column]:nth-of-type(2) [data-testid=stVerticalBlock]{
@@ -435,41 +347,29 @@ with st.sidebar:
     
     
     # st.subheader(":blue[Your multi-modal documents]")
-    pdf_doc_ = st.file_uploader(
-        "Upload your PDFs here and click on 'Process'", accept_multiple_files=False)
+    # pdf_doc_ = st.file_uploader(
+    #     "Upload your PDFs here and click on 'Process'", accept_multiple_files=False)
                     
                 
-    pdf_docs = [pdf_doc_]
-    if st.button("Process"):
-        with st.spinner("Processing"):
-            if os.path.isdir(parent_dirname+"/pdfs") == False:
-                os.mkdir(parent_dirname+"/pdfs")
+    # pdf_docs = [pdf_doc_]
+    # if st.button("Process"):
+    #     with st.spinner("Processing"):
+    #         if os.path.isdir(parent_dirname+"/pdfs") == False:
+    #             os.mkdir(parent_dirname+"/pdfs")
             
-            for pdf_doc in pdf_docs:
-                print(type(pdf_doc))
-                pdf_doc_name = (pdf_doc.name).replace(" ","_")
-                with open(os.path.join(parent_dirname+"/pdfs",pdf_doc_name),"wb") as f: 
-                    f.write(pdf_doc.getbuffer())  
+    #         for pdf_doc in pdf_docs:
+    #             print(type(pdf_doc))
+    #             pdf_doc_name = (pdf_doc.name).replace(" ","_")
+    #             with open(os.path.join(parent_dirname+"/pdfs",pdf_doc_name),"wb") as f: 
+    #                 f.write(pdf_doc.getbuffer())  
                     
-                request_ = { "bucket": s3_bucket_,"key": pdf_doc_name}
-                # if(st.session_state.input_copali_rerank):
-                #     copali.process_doc(request_)
-                # else:
-                rag_DocumentLoader.load_docs(request_)
-                print('lambda done')
-        st.success('you can start searching on your PDF')
-        
-    ############## haystach demo temporary addition ############    
-    # st.subheader(":blue[Multimodality]")
-    # colu1,colu2 = st.columns([50,50])
-    # with colu1:
-    #     in_images = st.toggle('Images', key = 'in_images', disabled = False)
-    # with colu2:
-    #     in_tables = st.toggle('Tables', key = 'in_tables', disabled = False)   
-    # if(in_tables):
-    #     st.session_state.input_table_with_sql = True
-    # else:
-    #     st.session_state.input_table_with_sql = False
+    #             request_ = { "bucket": s3_bucket_,"key": pdf_doc_name}
+    #             # if(st.session_state.input_copali_rerank):
+    #             #     copali.process_doc(request_)
+    #             # else:
+    #             rag_DocumentLoader.load_docs(request_)
+    #             print('lambda done')
+    #     st.success('you can start searching on your PDF')
         
      ############## haystach demo temporary addition ############       
     #if(pdf_doc_ is None or pdf_doc_ == ""):
@@ -509,7 +409,6 @@ with st.sidebar:
     
     st.subheader(":blue[Multi-vector retrieval]")
     
-    #st.write("Dataset indexed: https://huggingface.co/datasets/vespa-engine/gpfg-QA")
     colpali_search_rerank =  st.checkbox('Try Colpali multi-vector retrieval on the [sample dataset](https://huggingface.co/datasets/vespa-engine/gpfg-QA)', key = 'input_colpali', disabled = False, value = False, help = "Checking this box will use colpali as the embedding model and retrieval is performed using multi-vectors followed by re-ranking using MaxSim")
         
     if(colpali_search_rerank):
@@ -524,12 +423,6 @@ with st.sidebar:
         st.write("1. Proportion of female new hires 2021-2023? \n\n 2. First-half 2021 return on unlisted real estate investments? \n\n 3. Trend of the fund's expected absolute volatility between January 2014 and January 2016? \n\n 4. Fund return percentage in 2017? \n\n 5. Annualized gross return of the fund from 1997 to 2008?")
     
         
-    # copali_rerank = st.checkbox("Search and Re-rank with Token level vectors",key = 'copali_rerank',help = "Enabling this option uses 'Copali' model's page level image embeddings to retrieve documents and MaxSim to re-rank the pages.\n\n Hugging Face Model: https://huggingface.co/vidore/colpali")
-        
-    # if(copali_rerank):
-    #     st.session_state.input_copali_rerank = True
-    # else:
-    #     st.session_state.input_copali_rerank = False