Improved xlsx output formatting. Deals better with cleaning data then analysing in same session.

.gitignore

@@ -16,6 +16,7 @@
 *.pkl
 *.pkl.gz
 *.pem
+docs/*
 build/*
 dist/*
 __pycache__/*

app.py

@@ -35,9 +35,13 @@ with block:
     vec_weight = gr.State(1)
 
     corpus_state = gr.State()
-    keyword_data_state = gr.State(pd.DataFrame())
     keyword_data_list_state = gr.State([])
     join_data_state = gr.State(pd.DataFrame())
+
+    orig_keyword_data_state = gr.State(pd.DataFrame())
+    keyword_data_state = gr.State(pd.DataFrame())
+   
+    orig_semantic_data_state = gr.State(pd.DataFrame())
     semantic_data_state = gr.State(pd.DataFrame())
 
     in_k1_info = gr.State("""k1: Constant used for influencing the term frequency saturation. After saturation is reached, additional
@@ -157,13 +161,14 @@ depends on factors such as the type of documents or queries. Information taken f
     
     ### BM25 SEARCH ###
     # Update dropdowns upon initial file load
-    in_bm25_file.upload(initial_data_load, inputs=[in_bm25_file], outputs=[in_bm25_column, search_df_join_column, keyword_data_state, search_index_state, embeddings_state, tokenised_state, load_finished_message, current_source])
+    in_bm25_file.upload(initial_data_load, inputs=[in_bm25_file], outputs=[in_bm25_column, search_df_join_column, keyword_data_state, orig_keyword_data_state, search_index_state, embeddings_state, tokenised_state, load_finished_message, current_source])
     in_join_file.upload(put_columns_in_join_df, inputs=[in_join_file], outputs=[in_join_column, join_data_state, in_join_message])
  
     # Load in BM25 data
-    load_bm25_data_button.click(fn=prepare_bm25_input_data, inputs=[in_bm25_file, in_bm25_column, keyword_data_state, tokenised_state, in_clean_data, return_intermediate_files], outputs=[corpus_state, load_finished_message, keyword_data_state, output_file, output_file, keyword_data_list_state]).\
+    load_bm25_data_button.click(fn=prepare_bm25_input_data, inputs=[in_bm25_file, in_bm25_column, keyword_data_state, tokenised_state, in_clean_data, return_intermediate_files], outputs=[corpus_state, load_finished_message, keyword_data_state, output_file, output_file, keyword_data_list_state, in_bm25_column]).\
     then(fn=prepare_bm25, inputs=[corpus_state, in_bm25_file, in_bm25_column, search_index_state, in_clean_data, return_intermediate_files, in_k1, in_b, in_alpha], outputs=[load_finished_message, output_file])#.\
     
+    
     # BM25 search functions on click or enter
     keyword_search_button.click(fn=bm25_search, inputs=[keyword_query, in_no_search_results, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file], api_name="keyword")
     keyword_query.submit(fn=bm25_search, inputs=[keyword_query, in_no_search_results, keyword_data_state, in_bm25_column, join_data_state, in_clean_data, in_join_column, search_df_join_column], outputs=[output_single_text, output_file])
@@ -174,10 +179,10 @@ depends on factors such as the type of documents or queries. Information taken f
     ### SEMANTIC SEARCH ###
 
     # Load in a csv/excel file for semantic search
-    in_semantic_file.upload(initial_data_load, inputs=[in_semantic_file], outputs=[in_semantic_column,  search_df_join_column, semantic_data_state, search_index_state, embeddings_state, tokenised_state, semantic_load_progress, current_source_semantic])
+    in_semantic_file.upload(initial_data_load, inputs=[in_semantic_file], outputs=[in_semantic_column,  search_df_join_column,  semantic_data_state, orig_semantic_data_state, search_index_state, embeddings_state, tokenised_state, semantic_load_progress, current_source_semantic])
     load_semantic_data_button.click(
         csv_excel_text_to_docs, inputs=[semantic_data_state, in_semantic_file, in_semantic_column, in_clean_data, return_intermediate_files], outputs=[ingest_docs, semantic_load_progress]).\
-        then(docs_to_bge_embed_np_array, inputs=[ingest_docs, in_semantic_file, embeddings_state, return_intermediate_files, embedding_super_compress], outputs=[semantic_load_progress, vectorstore_state, semantic_output_file])
+        then(docs_to_bge_embed_np_array, inputs=[ingest_docs, in_semantic_file, embeddings_state, in_clean_data, return_intermediate_files, embedding_super_compress], outputs=[semantic_load_progress, vectorstore_state, semantic_output_file])
     
     # Semantic search query
     semantic_submit.click(bge_simple_retrieval, inputs=[semantic_query, vectorstore_state, ingest_docs, in_semantic_column, k_val, out_passages, semantic_min_distance, vec_weight, join_data_state, in_join_column, search_df_join_column], outputs=[semantic_output_single_text, semantic_output_file], api_name="semantic")

search_funcs/bm25_functions.py

@@ -231,11 +231,11 @@ class BM25:
 # These following functions are my own work
 
 def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, clean="No",  return_intermediate_files = "No", progress=gr.Progress(track_tqdm=True)):
-	print(in_file)
+	#print(in_file)
 
 	if not in_file:
 		print("No input file found. Please load in at least one file.")
-		return None, "No input file found. Please load in at least one file.", data_state, None, None, None, []
+		return None, "No input file found. Please load in at least one file.", data_state, None, None, [], gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
 	progress(0, desc = "Loading in data")
 	file_list = [string.name for string in in_file]
@@ -245,10 +245,10 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 	data_file_names = [string for string in file_list if "tokenised" not in string.lower() and "npz" not in string.lower() and "gz" not in string.lower()]
 
 	if not data_file_names:
-		return None, "Please load in at least one csv/Excel/parquet data file.", data_state, None, None, None, []
+		return None, "Please load in at least one csv/Excel/parquet data file.", data_state, None, None, [], gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
 	if not text_column:
-		return None, "Please enter a column name to search.", data_state, None, None, None, []
+		return None, "Please enter a column name to search.", data_state, None, None,[], gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
 	data_file_name = data_file_names[0]
 
@@ -268,7 +268,7 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 		corpus = list(df[text_column])
 		message = "Tokenisation skipped - loading search index from file."
 		print(message)
-		return corpus, message, df, None, None, None
+		return corpus, message, df, None, None, [], gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
 	
 	
@@ -282,7 +282,7 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 		df_list = initial_clean(df_list)
 
 		# Save to file if you have cleaned the data
-		out_file_name, text_column = save_prepared_bm25_data(data_file_name, df_list, df, text_column)
+		out_file_name, text_column, df  = save_prepared_bm25_data(data_file_name, df_list, df, text_column)
 	
 		clean_toc = time.perf_counter()
 		clean_time_out = f"Cleaning the text took {clean_toc - clean_tic:0.1f} seconds."
@@ -328,9 +328,9 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 
 		pd.DataFrame(data={"Corpus":corpus}).to_parquet(tokenised_data_file_name)
 
-		return corpus, message, df, out_file_name, tokenised_data_file_name, df_list
+		return corpus, message, df, out_file_name, tokenised_data_file_name, df_list, gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
-	return corpus, message, df, out_file_name, None, df_list
+	return corpus, message, df, out_file_name, None, df_list, gr.Dropdown(allow_custom_value=True, value=text_column, choices=data_state.columns.to_list())
 
 def save_prepared_bm25_data(in_file_name, prepared_text_list, in_df, in_bm25_column, progress=gr.Progress(track_tqdm=True)):
 
@@ -356,7 +356,7 @@ def save_prepared_bm25_data(in_file_name, prepared_text_list, in_df, in_bm25_col
 		prepared_df.to_parquet(file_name)
 	else: file_name = None
 
-	return file_name, new_text_column
+	return file_name, new_text_column, prepared_df
 
 def prepare_bm25(corpus, in_file, text_column, search_index, clean, return_intermediate_files, k1=1.5, b = 0.75, alpha=-5, progress=gr.Progress(track_tqdm=True)):
 	#bm25.save("saved_df_bm25")

search_funcs/clean_funcs.py

@@ -32,68 +32,27 @@ num_pattern_regex = r'[0-9]+'
 postcode_pattern_regex = r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)'
 warning_pattern_regex = r'caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.'
 nbsp_pattern_regex = r' '
+multiple_spaces_regex = r'\s{2,}'
 
 # Pre-compiling the regular expressions for efficiency
-email_start_pattern = re.compile(email_start_pattern_regex)
-email_end_pattern = re.compile(email_end_pattern_regex)
-html_pattern = re.compile(html_pattern_regex)
-email_pattern = re.compile(email_end_pattern_regex)
-num_pattern = re.compile(num_pattern_regex)
-postcode_pattern = re.compile(postcode_pattern_regex)
-warning_pattern = re.compile(warning_pattern_regex)
-nbsp_pattern = re.compile(nbsp_pattern_regex)
-
-# def stem_sentence(sentence):
-
-#     words = sentence.split()
-#     stemmed_words = [stemmer.stem(word).lower().rstrip("'") for word in words]
-#     return stemmed_words
-
-# def stem_sentences(sentences, progress=gr.Progress()):
-#         """Stem each sentence in a list of sentences."""
-#         stemmed_sentences = [stem_sentence(sentence) for sentence in progress.tqdm(sentences)]
-#         return stemmed_sentences
-
-# def get_lemma_text(text):
-#     # Tokenize the input string into words
-#     tokens = word_tokenize(text)
-    
-#     lemmas = []
-#     for word in tokens:
-#         if len(word) > 3:
-#             lemma = wn.morphy(word)
-#         else:
-#             lemma = None
-        
-#         if lemma is None:
-#             lemmas.append(word)
-#         else:
-#             lemmas.append(lemma)
-#     return lemmas
-
-# def get_lemma_tokens(tokens):
-    # Tokenize the input string into words
-    
-    # lemmas = []
-    # for word in tokens:
-    #     if len(word) > 3:
-    #         lemma = wn.morphy(word)
-    #     else:
-    #         lemma = None
-        
-    #     if lemma is None:
-    #         lemmas.append(word)
-    #     else:
-    #         lemmas.append(lemma)
-    # return lemmas
+# email_start_pattern = re.compile(email_start_pattern_regex)
+# email_end_pattern = re.compile(email_end_pattern_regex)
+# html_pattern = re.compile(html_pattern_regex)
+# email_pattern = re.compile(email_end_pattern_regex)
+# num_pattern = re.compile(num_pattern_regex)
+# postcode_pattern = re.compile(postcode_pattern_regex)
+# warning_pattern = re.compile(warning_pattern_regex)
+# nbsp_pattern = re.compile(nbsp_pattern_regex)
+
 
 def initial_clean(texts , progress=gr.Progress()):
     texts = pl.Series(texts)#[]
 
-    text = texts.str.replace_all(email_start_pattern_regex, '')
+    text = texts.str.replace_all(html_pattern_regex, '')
+    text = text.str.replace_all(email_start_pattern_regex, '')
     text = text.str.replace_all(email_end_pattern_regex, '')
-    text = text.str.replace_all(html_pattern_regex, '')
     text = text.str.replace_all(email_pattern_regex, '')
+    text = text.str.replace_all(multiple_spaces_regex, ' ')
 
     text = text.to_list()
     

search_funcs/helper_functions.py

@@ -13,7 +13,7 @@ import numpy as np
 from openpyxl import Workbook
 from openpyxl.cell.text import InlineFont 
 from openpyxl.cell.rich_text import TextBlock, CellRichText
-from openpyxl.styles import Font
+from openpyxl.styles import Font, Alignment
 
 # Attempt to delete content of gradio temp folder
 def get_temp_folder_path():
@@ -103,6 +103,7 @@ def initial_data_load(in_file):
     tokenised_load =[]
     out_message = ""
     current_source = ""
+    df = pd.DataFrame()
 
     file_list = [string.name for string in in_file]
 
@@ -113,25 +114,25 @@ def initial_data_load(in_file):
     if not data_file_names:
         out_message = "Please load in at least one csv/Excel/parquet data file."
         print(out_message)
-        return gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), pd.DataFrame(), index_load, out_message
+        return gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), pd.DataFrame(), pd.DataFrame(), index_load, out_message
 
-    data_file_name = data_file_names[0]
+    # This if you have loaded in a documents object for the semantic search
+    if "pkl" in data_file_names[0]: 
+        df = read_file(data_file_names[0])
+        new_choices = list(df[0].metadata.keys()) #["Documents"] #["page_contents"] + 
+        current_source = get_file_path_end_with_ext(data_file_names[0])
 
-    current_source = get_file_path_end_with_ext(data_file_name)
-   
-    
-    df = read_file(data_file_name)
-
-    if "pkl" not in data_file_name:
+    # This if you have loaded in a csv/parquets/xlsx
+    else:
+        for file in data_file_names:
 
-        new_choices = list(df.columns)
+            current_source = current_source + get_file_path_end_with_ext(file) + " "
+        
+            df_new = read_file(file)
 
-    elif "search_index" in data_file_name:
-        # If only the search_index found, need a data file too
-        new_choices = []
+            df = pd.concat([df, df_new], ignore_index = True)
 
-    else: new_choices = ["page_contents"] + list(df[0].metadata.keys()) #["Documents"]
-    #print(new_choices)
+        new_choices = list(df.columns)
 
     concat_choices.extend(new_choices)
 
@@ -161,7 +162,7 @@ def initial_data_load(in_file):
     out_message = "Initial data check successful. Next, choose a data column to search in the drop down above, then click 'Load data'"
     print(out_message)
         
-    return gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), df, index_load, embed_load, tokenised_load, out_message, current_source
+    return gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), df, df, index_load, embed_load, tokenised_load, out_message, current_source
 
 def put_columns_in_join_df(in_file):
     '''
@@ -291,12 +292,19 @@ def create_highlighted_excel_wb(df, search_text, column_to_highlight):
     for cell in sheet[1]:
         cell.font = Font(bold=True)
 
+    column_width = 150  # Adjust as needed
+    relevant_column_no = (df.columns == column_to_highlight).argmax() + 1
+    print(relevant_column_no)
+    sheet.column_dimensions[sheet.cell(row=1, column=relevant_column_no).column_letter].width = column_width
+
     # Find substrings in cells and highlight
     for r_idx, row in enumerate(df.itertuples(), start=2):
         for c_idx, cell_value in enumerate(row[1:], start=1):
             sheet.cell(row=r_idx, column=c_idx, value=cell_value)
             if df.columns[c_idx - 1] == column_to_highlight:
+
                 html_text, combined_positions = highlight_found_text(search_text, cell_value)
                 sheet.cell(row=r_idx, column=c_idx).value = create_rich_text_cell_from_positions(cell_value, combined_positions)
+                sheet.cell(row=r_idx, column=c_idx).alignment = Alignment(wrap_text=True)
 
     return wb

search_funcs/semantic_functions.py

@@ -25,7 +25,7 @@ else:
 
 print("Device used is: ", torch_device)
 
-#from search_funcs.helper_functions import get_file_path_end
+from search_funcs.helper_functions import create_highlighted_excel_wb
 
 PandasDataFrame = Type[pd.DataFrame]
 
@@ -45,106 +45,10 @@ PandasDataFrame = Type[pd.DataFrame]
 embeddings_name = "BAAI/bge-small-en-v1.5"
 local_embeddings_location = "model/bge/"
 
-#try:
-#    tokenizer = AutoTokenizer.from_pretrained(embeddings_name)
-#    embeddings_model = AutoModel.from_pretrained(local_embeddings_location, local_files_only=True).to(torch_device)
-#except:
-#    tokenizer = AutoTokenizer.from_pretrained(embeddings_name)
-#    embeddings_model = AutoModel.from_pretrained(embeddings_name).to(torch_device)
-
 # Not using SentenceTransformer here
 embeddings_model = SentenceTransformer(embeddings_name)
     
-# def calc_bge_norm_embeddings(docs, embeddings_model=embeddings_model, tokenizer=tokenizer, progress=gr.Progress(track_tqdm=True)):
-#     # Tokenize sentences
-#     print("Tokenising")
-#     encoded_input = tokenizer(docs, padding=True, truncation=True, return_tensors='pt', max_length=32).to(torch_device)
-
-#     # Compute token embeddings
-#     print("Calculating embeddings")
-#     with torch.no_grad():
-#         model_output = embeddings_model(**encoded_input).to(torch_device)
-#         # Perform pooling. In this case, cls pooling.
-#         embeddings_out = model_output[0][:, 0]
-#     # normalize embeddings
-#     embeddings_out = torch.nn.functional.normalize(embeddings_out, p=2, dim=1)
-#     #print("Sentence embeddings:", embeddings_out)
-
-#     return embeddings_out
-
-
-def docs_to_jina_embed_np_array(docs_out, in_file, embeddings_state, return_intermediate_files = "No", embeddings_super_compress = "No", embeddings = embeddings_model, progress=gr.Progress(track_tqdm=True)):
-    '''
-    Takes a Langchain document class and saves it into a Chroma sqlite file.
-    '''
-    if not in_file:
-        out_message = "No input file found. Please load in at least one file."
-        print(out_message)
-        return out_message, None, None
-        
-
-    progress(0.6, desc = "Loading/creating embeddings")
-
-    print(f"> Total split documents: {len(docs_out)}")
-
-    #print(docs_out)
-
-    page_contents = [doc.page_content for doc in docs_out]
-
-    ## Load in pre-embedded file if exists
-    file_list = [string.name for string in in_file]
-
-    #print(file_list)
-
-    embeddings_file_names = [string for string in file_list if "embedding" in string.lower()]
-    data_file_names = [string for string in file_list if "tokenised" not in string.lower() and "npz" not in string.lower()]# and "gz" not in string.lower()]
-    data_file_name = data_file_names[0]
-    data_file_name_no_ext = get_file_path_end(data_file_name)
-
-    out_message = "Document processing complete. Ready to search."
-
-     # print("embeddings loaded: ", embeddings_out)
-
-    if embeddings_state.size == 0:
-        tic = time.perf_counter()
-        print("Starting to embed documents.")
-        #embeddings_list = []
-        #for page in progress.tqdm(page_contents, desc = "Preparing search index", unit = "rows"):
-        #    embeddings_list.append(embeddings.encode(sentences=page, max_length=1024).tolist())
-
-        embeddings_out = embeddings.encode(sentences=page_contents, max_length=1024, show_progress_bar = True, batch_size = 32) # For Jina embeddings
-        #embeddings_list = embeddings.encode(sentences=page_contents, normalize_embeddings=True).tolist() # For BGE embeddings
-        #embeddings_list = embeddings.encode(sentences=page_contents).tolist() # For minilm
-
-        toc = time.perf_counter()
-        time_out = f"The embedding took {toc - tic:0.1f} seconds"
-        print(time_out)
-
-        # If you want to save your files for next time
-        if return_intermediate_files == "Yes":
-            progress(0.9, desc = "Saving embeddings to file")
-            if embeddings_super_compress == "No":
-                semantic_search_file_name = data_file_name_no_ext + '_' + 'embeddings.npz'
-                np.savez_compressed(semantic_search_file_name, embeddings_out)
-            else:
-                semantic_search_file_name = data_file_name_no_ext + '_' + 'embedding_compress.npz'
-                embeddings_out_round = np.round(embeddings_out, 3) 
-                embeddings_out_round *= 100 # Rounding not currently used
-                np.savez_compressed(semantic_search_file_name, embeddings_out_round)
-
-            return out_message, embeddings_out, semantic_search_file_name
-
-        return out_message, embeddings_out, None
-    else:
-        # Just return existing embeddings if already exist
-        embeddings_out = embeddings_state
-    
-    print(out_message)
-
-    return out_message, embeddings_out, None#, None
-
-
-def docs_to_bge_embed_np_array(docs_out, in_file, embeddings_state, return_intermediate_files = "No", embeddings_super_compress = "No", embeddings_model = embeddings_model, progress=gr.Progress(track_tqdm=True)):
+def docs_to_bge_embed_np_array(docs_out, in_file, embeddings_state, clean, return_intermediate_files = "No", embeddings_super_compress = "No", embeddings_model = embeddings_model, progress=gr.Progress(track_tqdm=True)):
     '''
     Takes a Langchain document class and saves it into a Chroma sqlite file.
     '''
@@ -197,6 +101,9 @@ def docs_to_bge_embed_np_array(docs_out, in_file, embeddings_state, return_inter
 
         # If you want to save your files for next time
         if return_intermediate_files == "Yes":
+            if clean == "Yes": data_file_name_no_ext = data_file_name_no_ext + "_cleaned"
+            else: data_file_name_no_ext = data_file_name_no_ext
+
             progress(0.9, desc = "Saving embeddings to file")
             if embeddings_super_compress == "No":
                 semantic_search_file_name = data_file_name_no_ext + '_bge_embeddings.npz'
@@ -273,7 +180,7 @@ def process_data_from_scores_df(df_docs, in_join_file, out_passages, vec_score_c
     # Concatenate the original DataFrame with the expanded metadata DataFrame
     results_df_out = pd.concat([length_more_limit.drop('metadatas', axis=1), df_metadata_expanded], axis=1)
 
-    results_df_out = results_df_out.rename(columns={"documents":orig_df_col})
+    results_df_out = results_df_out.rename(columns={"documents":"search_text"})
 
     results_df_out = results_df_out.drop(["page_section", "row", "source", "id"], axis=1, errors="ignore")
     results_df_out['distances'] = round(results_df_out['distances'].astype(float), 3)
@@ -371,7 +278,11 @@ def bge_simple_retrieval(query_str:str, vectorstore, docs, orig_df_col:str, k_va
     print("Saving search output to file")
     progress(0.7, desc = "Saving search output to file")
 
-    results_df_out.to_excel(results_df_name, index= None)
+    # Highlight found text and save to file
+    results_df_out_wb = create_highlighted_excel_wb(results_df_out, query_str, "search_text")
+    results_df_out_wb.save(results_df_name)
+
+    #results_df_out.to_excel(results_df_name, index= None)
     results_first_text = results_df_out.iloc[0, 1]
 
     print("Returning results")
@@ -379,7 +290,77 @@ def bge_simple_retrieval(query_str:str, vectorstore, docs, orig_df_col:str, k_va
     return results_first_text, results_df_name
 
 
-def jina_simple_retrieval(query_str:str, vectorstore, docs, orig_df_col:str, k_val:int, out_passages:int,
+def docs_to_jina_embed_np_array_deprecated(docs_out, in_file, embeddings_state, return_intermediate_files = "No", embeddings_super_compress = "No", embeddings = embeddings_model, progress=gr.Progress(track_tqdm=True)):
+    '''
+    Takes a Langchain document class and saves it into a Chroma sqlite file.
+    '''
+    if not in_file:
+        out_message = "No input file found. Please load in at least one file."
+        print(out_message)
+        return out_message, None, None
+        
+
+    progress(0.6, desc = "Loading/creating embeddings")
+
+    print(f"> Total split documents: {len(docs_out)}")
+
+    #print(docs_out)
+
+    page_contents = [doc.page_content for doc in docs_out]
+
+    ## Load in pre-embedded file if exists
+    file_list = [string.name for string in in_file]
+
+    #print(file_list)
+
+    embeddings_file_names = [string for string in file_list if "embedding" in string.lower()]
+    data_file_names = [string for string in file_list if "tokenised" not in string.lower() and "npz" not in string.lower()]# and "gz" not in string.lower()]
+    data_file_name = data_file_names[0]
+    data_file_name_no_ext = get_file_path_end(data_file_name)
+
+    out_message = "Document processing complete. Ready to search."
+
+     # print("embeddings loaded: ", embeddings_out)
+
+    if embeddings_state.size == 0:
+        tic = time.perf_counter()
+        print("Starting to embed documents.")
+        #embeddings_list = []
+        #for page in progress.tqdm(page_contents, desc = "Preparing search index", unit = "rows"):
+        #    embeddings_list.append(embeddings.encode(sentences=page, max_length=1024).tolist())
+
+        embeddings_out = embeddings.encode(sentences=page_contents, max_length=1024, show_progress_bar = True, batch_size = 32) # For Jina embeddings
+        #embeddings_list = embeddings.encode(sentences=page_contents, normalize_embeddings=True).tolist() # For BGE embeddings
+        #embeddings_list = embeddings.encode(sentences=page_contents).tolist() # For minilm
+
+        toc = time.perf_counter()
+        time_out = f"The embedding took {toc - tic:0.1f} seconds"
+        print(time_out)
+
+        # If you want to save your files for next time
+        if return_intermediate_files == "Yes":
+            progress(0.9, desc = "Saving embeddings to file")
+            if embeddings_super_compress == "No":
+                semantic_search_file_name = data_file_name_no_ext + '_' + 'embeddings.npz'
+                np.savez_compressed(semantic_search_file_name, embeddings_out)
+            else:
+                semantic_search_file_name = data_file_name_no_ext + '_' + 'embedding_compress.npz'
+                embeddings_out_round = np.round(embeddings_out, 3) 
+                embeddings_out_round *= 100 # Rounding not currently used
+                np.savez_compressed(semantic_search_file_name, embeddings_out_round)
+
+            return out_message, embeddings_out, semantic_search_file_name
+
+        return out_message, embeddings_out, None
+    else:
+        # Just return existing embeddings if already exist
+        embeddings_out = embeddings_state
+    
+    print(out_message)
+
+    return out_message, embeddings_out, None#, None
+
+def jina_simple_retrieval_deprecated(query_str:str, vectorstore, docs, orig_df_col:str, k_val:int, out_passages:int,
                            vec_score_cut_off:float, vec_weight:float, in_join_file, in_join_column = None, search_df_join_column = None, device = torch_device, embeddings = embeddings_model, progress=gr.Progress(track_tqdm=True)): # ,vectorstore, embeddings
 
     # print("vectorstore loaded: ", vectorstore)
@@ -464,6 +445,7 @@ def jina_simple_retrieval(query_str:str, vectorstore, docs, orig_df_col:str, k_v
 #if os.path.isfile(chromadb_file):
 #    os.remove(chromadb_file)
 
+
 def docs_to_chroma_save_deprecated(docs_out, embeddings = embeddings_model, progress=gr.Progress()):
     '''
     Takes a Langchain document class and saves it into a Chroma sqlite file. Not currently used.

search_funcs/semantic_ingest_functions.py

@@ -309,7 +309,7 @@ def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_interm
         elif clean == "Yes":
             #pd.DataFrame(data = {"Documents":page_content_series_string}).to_parquet(file_name + "_prepared_docs_clean.parquet")
 
-            with gzip.open(file_name + "cleaned_prepared_docs.pkl.gz", 'wb') as file:
+            with gzip.open(file_name + "_cleaned_prepared_docs.pkl.gz", 'wb') as file:
                 pickle.dump(doc_sections, file)
 
             #pd.Series(doc_sections).to_pickle(file_name + "_prepared_docs_clean.pkl")