Set bm25 in functions explicitly. Some API updates. Now can get connection params on startup.

app.py

@@ -8,7 +8,7 @@ PandasDataFrame = Type[pd.DataFrame]
 from search_funcs.bm25_functions import prepare_bm25_input_data, prepare_bm25, bm25_search
 from search_funcs.semantic_ingest_functions import csv_excel_text_to_docs
 from search_funcs.semantic_functions import docs_to_bge_embed_np_array, bge_simple_retrieval
-from search_funcs.helper_functions import display_info, initial_data_load, put_columns_in_join_df, get_temp_folder_path, empty_folder, output_folder
+from search_funcs.helper_functions import display_info, initial_data_load, put_columns_in_join_df, get_temp_folder_path, empty_folder, get_connection_params, output_folder
 from search_funcs.spacy_search_funcs import spacy_fuzzy_search
 from search_funcs.aws_functions import load_data_from_aws
 
@@ -30,6 +30,7 @@ with block:
     embeddings_state = gr.State(np.array([])) # globals()["embeddings"]
     search_index_state = gr.State()
     tokenised_state = gr.State()
+    bm25_search_object_state = gr.State()
 
     k_val = gr.State(9999)
     out_passages = gr.State(9999)
@@ -46,6 +47,9 @@ with block:
     orig_semantic_data_state = gr.State(pd.DataFrame())
     semantic_data_state = gr.State(pd.DataFrame())
 
+    session_hash_state = gr.State("")
+    s3_output_folder_state = gr.State("")
+
     in_k1_info = gr.State("""k1: Constant used for influencing the term frequency saturation. After saturation is reached, additional
 presence for the term adds a significantly less additional score. According to [1]_, experiments suggest
 that 1.2 < k1 < 2 yields reasonably good results, although the optimal value depends on factors such as
@@ -92,7 +96,6 @@ depends on factors such as the type of documents or queries. Information taken f
                 output_single_text = gr.Textbox(label="Top result")
                 output_file = gr.File(label="File output")
 
-
     with gr.Tab("Semantic search"):
         gr.Markdown(
     """
@@ -179,20 +182,20 @@ depends on factors such as the type of documents or queries. Information taken f
     
     ### BM25 SEARCH ###
     # Update dropdowns upon initial file load
-    in_bm25_file.change(initial_data_load, inputs=[in_bm25_file], outputs=[in_bm25_column, search_df_join_column, keyword_data_state, orig_keyword_data_state, search_index_state, embeddings_state, tokenised_state, load_finished_message, current_source])
+    in_bm25_file.change(initial_data_load, inputs=[in_bm25_file], outputs=[in_bm25_column, search_df_join_column, keyword_data_state, orig_keyword_data_state, search_index_state, embeddings_state, tokenised_state, load_finished_message, current_source], api_name="initial_load")
     in_join_file.change(put_columns_in_join_df, inputs=[in_join_file], outputs=[in_join_column, join_data_state, in_join_message])
  
     # Load in BM25 data
     load_bm25_data_button.click(fn=prepare_bm25_input_data, inputs=[in_bm25_file, in_bm25_column, keyword_data_state, tokenised_state, in_clean_data, return_intermediate_files], outputs=[corpus_state, load_finished_message, keyword_data_state, output_file, output_file, keyword_data_list_state, in_bm25_column], api_name="load_keyword").\
-    then(fn=prepare_bm25, inputs=[corpus_state, in_bm25_file, in_bm25_column, search_index_state, in_clean_data, return_intermediate_files, in_k1, in_b, in_alpha], outputs=[load_finished_message, output_file], api_name="prepare_keyword")#.\
+    then(fn=prepare_bm25, inputs=[corpus_state, in_bm25_file, in_bm25_column, search_index_state, in_clean_data, return_intermediate_files, in_k1, in_b, in_alpha], outputs=[load_finished_message, output_file, bm25_search_object_state], api_name="prepare_keyword")
     
     
     # BM25 search functions on click or enter
-    keyword_search_button.click(fn=bm25_search, inputs=[keyword_query, in_no_search_results, orig_keyword_data_state, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file], api_name="keyword_search")
-    keyword_query.submit(fn=bm25_search, inputs=[keyword_query, in_no_search_results, orig_keyword_data_state, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file])
+    keyword_search_button.click(fn=bm25_search, inputs=[keyword_query, in_no_search_results, orig_keyword_data_state, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, bm25_search_object_state, in_join_column, search_df_join_column], outputs=[output_single_text, output_file], api_name="keyword_search")
+    keyword_query.submit(fn=bm25_search, inputs=[keyword_query, in_no_search_results, orig_keyword_data_state, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, bm25_search_object_state, in_join_column, search_df_join_column], outputs=[output_single_text, output_file])
 
     # Fuzzy search functions on click
-    fuzzy_search_button.click(fn=spacy_fuzzy_search, inputs=[keyword_query, keyword_data_list_state, keyword_data_state, in_bm25_column, join_data_state, search_df_join_column, in_join_column, no_spelling_mistakes], outputs=[output_single_text, output_file], api_name="fuzzy")
+    fuzzy_search_button.click(fn=spacy_fuzzy_search, inputs=[keyword_query, keyword_data_list_state, keyword_data_state, in_bm25_column, join_data_state, search_df_join_column, in_join_column, no_spelling_mistakes], outputs=[output_single_text, output_file], api_name="fuzzy_search")
     
     ### SEMANTIC SEARCH ###
 
@@ -203,30 +206,15 @@ depends on factors such as the type of documents or queries. Information taken f
         then(docs_to_bge_embed_np_array, inputs=[ingest_docs, in_semantic_file, embeddings_state, output_file_state, in_clean_data, return_intermediate_files, embedding_super_compress], outputs=[semantic_load_progress, vectorstore_state, semantic_output_file, output_file_state])
     
     # Semantic search query
-    semantic_submit.click(bge_simple_retrieval, inputs=[semantic_query, vectorstore_state, ingest_docs, in_semantic_column, k_val, out_passages, semantic_min_distance, vec_weight, join_data_state, in_join_column, search_df_join_column], outputs=[semantic_output_single_text, semantic_output_file], api_name="semantic")
+    semantic_submit.click(bge_simple_retrieval, inputs=[semantic_query, vectorstore_state, ingest_docs, in_semantic_column, k_val, out_passages, semantic_min_distance, vec_weight, join_data_state, in_join_column, search_df_join_column], outputs=[semantic_output_single_text, semantic_output_file], api_name="semantic_search")
     semantic_query.submit(bge_simple_retrieval, inputs=[semantic_query, vectorstore_state, ingest_docs, in_semantic_column, k_val, out_passages, semantic_min_distance, vec_weight, join_data_state, in_join_column, search_df_join_column], outputs=[semantic_output_single_text, semantic_output_file])
 
-# Simple run for HF spaces or local on your computer
-#block.queue().launch(debug=True) 
-
-# def get_params(request: gr.Request):
-#    if request:
-#        print("Request headers dictionary:", request.headers)
-#        print("IP address:", request.client.host)
-#        print("Query parameters:", dict(request.query_params))
-#    return request.query_params
-
-# request_params = get_params()
-# print(request_params)
+    block.load(get_connection_params, inputs=None, outputs=[session_hash_state, s3_output_folder_state])
 
-# Running on server (e.g. AWS) without specifying port
-block.queue().launch(ssl_verify=False) # root_path="/data-text-search" # server_name="0.0.0.0", 
-
-
-# Running on local server without https
-#block.queue().launch(server_name="0.0.0.0", server_port=7861, ssl_verify=False)
+# Launch the Gradio app
+if __name__ == "__main__":
+    block.queue().launch(show_error=True) # root_path="/data-text-search" # server_name="0.0.0.0", 
     
 # Running on local server with https: https://discuss.huggingface.co/t/how-to-run-gradio-with-0-0-0-0-and-https/38003 or https://dev.to/rajshirolkar/fastapi-over-https-for-development-on-windows-2p7d # Need to download OpenSSL and create own keys 
 # block.queue().launch(ssl_verify=False, share=False, debug=False, server_name="0.0.0.0",server_port=443,
-#                      ssl_certfile="cert.pem", ssl_keyfile="key.pem") # port 443 for https. Certificates currently not valid
-
+#                      ssl_certfile="cert.pem", ssl_keyfile="key.pem") # port 443 for https. Certificates currently not valid

search_funcs/bm25_functions.py

@@ -40,6 +40,7 @@ tokenizer = nlp.tokenizer
 PARAM_K1 = 1.5
 PARAM_B = 0.75
 IDF_CUTOFF = -inf
+bm25 = "" # Placeholder just so initial load doesn't fail
 
 # Class built off https://github.com/Inspirateur/Fast-BM25
 
@@ -263,6 +264,8 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 	tokenised_file_names = [string for string in file_list if "tokenised" in string.lower()]
 	search_index_file_names = [string for string in file_list if "gz" in string.lower()]
 
+	print("Dataframe columns:", df.columns)
+
 	df[text_column] = df[text_column].astype(str).str.lower()
 
 	if "copy_of_case_note_id" in df.columns:
@@ -386,8 +389,6 @@ def prepare_bm25(corpus, in_file, text_column, search_index, clean, return_inter
 		print(out_message)
 		return  out_message, None
 
-
-
 	file_list = [string.name for string in in_file]
 
 	#print(file_list)
@@ -444,13 +445,13 @@ def prepare_bm25(corpus, in_file, text_column, search_index, clean, return_inter
 
 		message = "Search parameters loaded."
 
-		return message, bm25_search_file_name
+		return message, bm25_search_file_name, bm25
 
 	message = "Search parameters loaded."
 
 	print(message)
 
-	return message, None
+	return message, None, bm25
 
 def convert_bm25_query_to_tokens(free_text_query, clean="No"):
     '''
@@ -473,7 +474,7 @@ def convert_bm25_query_to_tokens(free_text_query, clean="No"):
 
     return out_query
 
-def bm25_search(free_text_query, in_no_search_results, original_data, searched_data, text_column, in_join_file, clean,  in_join_column = "", search_df_join_column = "", progress=gr.Progress(track_tqdm=True)):   
+def bm25_search(free_text_query, in_no_search_results, original_data, searched_data, text_column, in_join_file, clean, bm25, in_join_column = "", search_df_join_column = "", progress=gr.Progress(track_tqdm=True)):   
 
 	progress(0, desc = "Conducting keyword search")
 	

search_funcs/helper_functions.py

@@ -15,6 +15,8 @@ from openpyxl.cell.text import InlineFont
 from openpyxl.cell.rich_text import TextBlock, CellRichText
 from openpyxl.styles import Font, Alignment
 
+from search_funcs.aws_functions import bucket_name
+
 megabyte = 1024 * 1024  # Bytes in a megabyte
 file_size_mb = 500  # Size in megabytes
 file_size_bytes_500mb =  megabyte * file_size_mb
@@ -49,6 +51,41 @@ def ensure_output_folder_exists(output_folder):
     else:
         print(f"The output folder already exists:", folder_name)
 
+def get_connection_params(request: gr.Request):
+        if request:
+            #request_data = request.json()  # Parse JSON body
+            #print("All request data:", request_data)
+            #context_value = request_data.get('context') 
+            #if 'context' in request_data:
+            #     print("Request context dictionary:", request_data['context'])
+
+            #print("Request headers dictionary:", request.headers)
+            #print("All host elements", request.client)           
+            #print("IP address:", request.client.host)
+            #print("Query parameters:", dict(request.query_params))
+            # To get the underlying FastAPI items you would need to use await and some fancy @ stuff for a live query: https://fastapi.tiangolo.com/vi/reference/request/
+            #print("Request dictionary to object:", request.request.body())
+            print("Session hash:", request.session_hash)
+
+            if 'x-cognito-id' in request.headers:
+                out_session_hash = request.headers['x-cognito-id']
+                base_folder = "user-files/"
+                print("Cognito ID found:", out_session_hash)
+
+            else:
+                out_session_hash = request.session_hash
+                base_folder = "temp-files/"
+                print("Cognito ID not found. Using session hash as save folder.")
+
+            output_folder = base_folder + out_session_hash + "/"
+            if bucket_name:
+                print("S3 output folder is: " + "s3://" + bucket_name + "/" + output_folder)
+
+            return out_session_hash, output_folder
+        else:
+            print("No session parameters found.")
+            return "", ""
+
 # Attempt to delete content of gradio temp folder
 def get_temp_folder_path():
     username = getpass.getuser()