search pipeline updated

RAG/rag_DocumentLoader.py

@@ -16,7 +16,6 @@ from requests_aws4auth import AWS4Auth
 import re_ranker
 import utilities.invoke_models as invoke_models
 from requests.auth import HTTPBasicAuth
-
 import generate_csv_for_tables
 from pdf2image import convert_from_bytes,convert_from_path
 #import langchain

RAG/rag_DocumentSearcher.py

@@ -66,11 +66,6 @@ def query_(awsauth,inputs, session_id,search_types):
         images.append({'file':hit['_source']['image'],'caption':hit['_source']['processed_element']})
     
     ####### SEARCH ########
-    
-    
-    path = "_search/pipeline/rag-search-pipeline" 
-    url = host + path
-    
     num_queries = len(search_types)
     
     weights = []
@@ -89,7 +84,8 @@ def query_(awsauth,inputs, session_id,search_types):
         
         #print(weights) 
         
-            
+        path = "_search/pipeline/rag-search-pipeline" 
+        url = host + path 
         s_pipeline_payload = {
                 "description": "Post processor for hybrid search",
                 "phase_results_processors": [
@@ -110,10 +106,6 @@ def query_(awsauth,inputs, session_id,search_types):
             }
             
         r = requests.put(url, auth=awsauth, json=s_pipeline_payload, headers=headers)
-        #print(r.status_code)
-        #print(r.text)
-        
-    
     
     SIZE = 5
     
@@ -154,7 +146,7 @@ def query_(awsauth,inputs, session_id,search_types):
         
     if('Vector Search' in search_types):
         
-        embedding = embedding = invoke_models.invoke_model(question)
+        embedding  = invoke_models.invoke_model(question)
         
         vector_payload = {
             "knn": {
@@ -172,7 +164,7 @@ def query_(awsauth,inputs, session_id,search_types):
         sparse_payload =  {  "neural_sparse": {
                 "processed_element_embedding_sparse": {
                     "query_text": question,
-                    "model_id": "srrJ-owBQhe1aB-khx2n"
+                    "model_id": "fkol-ZMBTp0efWqBcO2P"
                 }
                 }}
                     
@@ -190,13 +182,6 @@ def query_(awsauth,inputs, session_id,search_types):
         # sparse_ = json.loads(r2.text)
         # query_sparse = sparse_["inference_results"][0]["output"][0]["dataAsMap"]["response"][0]
         
-   
-        
-
-        
-    # print("hybrid_payload") 
-    # print("---------------") 
-    #print(hybrid_payload) 
     hits = []
     if(num_queries>1):
         path = st.session_state.input_index+"/_search?search_pipeline=rag-search-pipeline" 
@@ -208,23 +193,16 @@ def query_(awsauth,inputs, session_id,search_types):
         del hybrid_payload["query"]["hybrid"]
         hybrid_payload["query"] = single_query
         r = requests.get(url, auth=awsauth, json=hybrid_payload, headers=headers)
-        #print(r.status_code)
         response_ = json.loads(r.text)
-        #print("-------------------------------------------------------------------")
-        #print(r.text)
         hits = response_['hits']['hits']
         
     else:
         r = requests.get(url, auth=awsauth, json=hybrid_payload, headers=headers)
-        #print(r.status_code)
         response_ = json.loads(r.text)
-        #print("-------------------------------------------------------------------")
-        #print(response_)
         hits = response_['hits']['hits']
     
     ##### GET reference tables separately like *_mm index search for images  ######
     def lazy_get_table():
-        #print("Forcing table analysis")
         table_ref = []
         any_table_exists = False
         for fname in os.listdir(parent_dirname+"/split_pdf_csv"):
@@ -251,7 +229,7 @@ def query_(awsauth,inputs, session_id,search_types):
             payload_tables = {"query":{"neural_sparse": {
                     "processed_element_embedding_sparse": {
                         "query_text": question,
-                        "model_id": "srrJ-owBQhe1aB-khx2n"
+                        "model_id": "fkol-ZMBTp0efWqBcO2P"
                     }
                     }  }     }
             

pages/Multimodal_Conversational_Search.py

@@ -145,11 +145,6 @@ if clear:
     st.session_state.questions_ = []
     st.session_state.answers_ = []
     st.session_state.input_query=""
-    # st.session_state.input_searchType="Conversational Search (RAG)"
-    # st.session_state.input_temperature = "0.001"
-    # st.session_state.input_topK = 200
-    # st.session_state.input_topP = 0.95
-    # st.session_state.input_maxTokens = 1024
 
 
 def handle_input():
@@ -163,11 +158,6 @@ def handle_input():
         if key.startswith('input_'):
             inputs[key.removeprefix('input_')] = st.session_state[key]
     st.session_state.inputs_ = inputs
-    
-    #######
-    
-    
-    #st.write(inputs) 
     question_with_id = {
         'question': inputs["query"],
         'id': len(st.session_state.questions_)
@@ -175,7 +165,7 @@ def handle_input():
     st.session_state.questions_.append(question_with_id)
     if(st.session_state.input_is_colpali):
         out_ = colpali.colpali_search_rerank(st.session_state.input_query)
-        #print(out_)
+        
     else:
         out_ = rag_DocumentSearcher.query_(awsauth, inputs, st.session_state['session_id'],st.session_state.input_rag_searchType)
     st.session_state.answers_.append({

utilities/re_ranker.py

@@ -6,16 +6,15 @@ import streamlit as st
 from sentence_transformers import CrossEncoder
 
 model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", max_length=512)
-kendra_ranking = boto3.client("kendra-ranking",region_name = 'us-east-1')
-
-
-print("Create a rescore execution plan.")
+####### Add this Kendra Rescore ranking
+#kendra_ranking = boto3.client("kendra-ranking",region_name = 'us-east-1')
+#print("Create a rescore execution plan.")
 
 # Provide a name for the rescore execution plan
-name = "MyRescoreExecutionPlan"
+#name = "MyRescoreExecutionPlan"
 # Set your required additional capacity units
 # Don't set capacity units if you don't require more than 1 unit given by default
-capacity_units = 2
+#capacity_units = 2
 
 # try:
 #     rescore_execution_plan_response = kendra_ranking.create_rescore_execution_plan(
@@ -45,7 +44,7 @@ capacity_units = 2
 #         print("%s" % e)
 
 # print("Program ends.")
-
+#########################
 
 
 def re_rank(self_, rerank_type, search_type, question, answers):
@@ -78,52 +77,29 @@ def re_rank(self_, rerank_type, search_type, question, answers):
             
 
     re_ranked = [{}]
-
-  
-
-    
-
-    if(rerank_type == 'Kendra Rescore'):
-
-
-        
-
-        rescore_response = kendra_ranking.rescore(
-            RescoreExecutionPlanId = 'b2a4d4f3-98ff-4e17-8b69-4c61ed7d91eb',
-            SearchQuery = query,
-            Documents = ans
-        )
-    
-            
-        #[{'DocumentId': 'DocId1', 'Score': 2.0}, {'DocumentId': 'DocId2', 'Score': 1.0}]   
-            
+    ####### Add this Kendra Rescore ranking
+    # if(rerank_type == 'Kendra Rescore'):
+    #     rescore_response = kendra_ranking.rescore(
+    #         RescoreExecutionPlanId = 'b2a4d4f3-98ff-4e17-8b69-4c61ed7d91eb',
+    #         SearchQuery = query,
+    #         Documents = ans
+    #     )
+    #     re_ranked[0]['answer']=[]
+    #     for result in rescore_response["ResultItems"]:
+
+    #         pos_ = ids.index(result['DocumentId'])
+
+    #         re_ranked[0]['answer'].append(answers[0]['answer'][pos_])
+    #     re_ranked[0]['search_type']=search_type,
+    #     re_ranked[0]['id'] = len(question)
+    #     return re_ranked
         
-        re_ranked[0]['answer']=[]
-        for result in rescore_response["ResultItems"]:
-
-            pos_ = ids.index(result['DocumentId'])
-
-            re_ranked[0]['answer'].append(answers[0]['answer'][pos_])
-        re_ranked[0]['search_type']=search_type,
-        re_ranked[0]['id'] = len(question)
-
-        #st.session_state.answers_none_rank = st.session_state.answers
-        return re_ranked
-        
-
-    # if(rerank_type == 'None'):
-        
-    #     st.session_state.answers = st.session_state.answers_none_rank 
-        
-
     if(rerank_type == 'Cross Encoder'):
 
         scores = model.predict(
                     ques_ans
                         )
         
-        print("scores")
-        print(scores)
         index__ = 0
         for i in ans:
             i['new_score'] = scores[index__]
@@ -148,9 +124,6 @@ def re_rank(self_, rerank_type, search_type, question, answers):
         return re_ranked
 
 
-
-
-    #return st.session_state.answers