Allowed for manual modification of output topic table. Did some fixes to deduplication and Excel file input. Allowed for General topic specification in zero shot topics

app.py

@@ -3,7 +3,7 @@ import socket
 import spaces
 from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df, get_connection_params, output_folder, get_or_create_env_var, reveal_feedback_buttons, wipe_logs, model_full_names, view_table, empty_output_vars_extract_topics, empty_output_vars_summarise, RUN_LOCAL_MODEL
 from tools.aws_functions import upload_file_to_s3, RUN_AWS_FUNCTIONS
-from tools.llm_api_call import extract_topics, load_in_data_file, load_in_previous_data_files, sample_reference_table_summaries, summarise_output_topics, batch_size_default, deduplicate_topics
+from tools.llm_api_call import extract_topics, load_in_data_file, load_in_previous_data_files, sample_reference_table_summaries, summarise_output_topics, batch_size_default, deduplicate_topics, modify_existing_output_tables
 from tools.auth import authenticate_user
 from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt, add_existing_topics_system_prompt, add_existing_topics_prompt
 #from tools.aws_functions import load_data_from_aws
@@ -44,14 +44,20 @@ with app:
     ###
 
     text_output_file_list_state = gr.State([])
+    text_output_modify_file_list_state = gr.State([])
     log_files_output_list_state = gr.State([]) 
     first_loop_state = gr.State(True)
     second_loop_state = gr.State(False)
+    modified_unique_table_change_bool = gr.State(True) # This boolean is used to flag whether a file upload should change just the modified unique table object on the second tab
 
-    file_data_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="file_data_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
-    master_topic_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_topic_df_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
-    master_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_reference_df_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
-    master_unique_topics_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_unique_topics_df_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
+    file_data_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="file_data_state", visible=False, type="pandas")
+    master_topic_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_topic_df_state", visible=False, type="pandas")
+    master_unique_topics_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_unique_topics_df_state", visible=False, type="pandas")
+    master_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_reference_df_state", visible=False, type="pandas")
+
+    master_modify_unique_topics_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_modify_unique_topics_df_state", visible=False, type="pandas")
+    master_modify_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_modify_reference_df_state", visible=False, type="pandas")
+    
  
     session_hash_state = gr.State()
     s3_output_folder_state = gr.State()
@@ -67,14 +73,15 @@ with app:
     feedback_s3_logs_loc_state = gr.State(feedback_data_folder)
 
     # Summary state objects
-    summary_reference_table_sample_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="summary_reference_table_sample_state", visible=False, type="pandas") # gr.State(pd.DataFrame())
-    master_reference_df_revised_summaries_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_reference_df_revised_summaries_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
-    master_unique_topics_df_revised_summaries_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_unique_topics_df_revised_summaries_state", visible=False, type="pandas") #gr.State(pd.DataFrame())
+    summary_reference_table_sample_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="summary_reference_table_sample_state", visible=False, type="pandas")
+    master_reference_df_revised_summaries_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_reference_df_revised_summaries_state", visible=False, type="pandas")
+    master_unique_topics_df_revised_summaries_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_unique_topics_df_revised_summaries_state", visible=False, type="pandas")
     summarised_references_markdown = gr.Markdown("", visible=False)
     summarised_outputs_list = gr.Dropdown(value=[], choices=[], visible=False, label="List of summarised outputs", allow_custom_value=True)
     latest_summary_completed_num = gr.Number(0, visible=False)
 
-    unique_topics_table_file_textbox = gr.Textbox(label="unique_topics_table_file_textbox", visible=False)
+    reference_data_file_name_textbox = gr.Textbox(label = "Reference data file name", value="", visible=False)
+    unique_topics_table_file_name_textbox = gr.Textbox(label="Unique topics data file name textbox", visible=False)
 
     ###
     # UI LAYOUT
@@ -108,15 +115,15 @@ with app:
         in_colnames = gr.Dropdown(choices=["Choose column with responses"], multiselect = False, label="Select the open text column of interest.", allow_custom_value=True, interactive=True)
         
         with gr.Accordion("I have my own list of topics (zero shot topic modelling).", open = False):
-            candidate_topics = gr.File(height=file_input_height, label="Input topics from file (csv). File should have a single column with a header, and all topic keywords below.")
+            candidate_topics = gr.File(height=file_input_height, label="Input topics from file (csv). File should have at least one column with a header, and all topic titles below this. Using the headers 'General Topic' and/or 'Subtopic' will allow for either or both of these lists to be specified.")
 
         context_textbox = gr.Textbox(label="Write up to one sentence giving context to the large language model for your task (e.g. 'Consultation for the construction of flats on Main Street')")
 
         sentiment_checkbox = gr.Radio(label="Choose sentiment categories to split responses", value="Negative, Neutral, or Positive", choices=["Negative, Neutral, or Positive", "Negative or Positive", "Do not assess sentiment"])
 
-        extract_topics_btn = gr.Button("Extract topics from open text", variant="primary")
+        extract_topics_btn = gr.Button("Extract topics", variant="primary")
         
-        text_output_summary = gr.Markdown(value="### Language model response will appear here")
+        text_output_summary = gr.Markdown(value="### Language model response will appear here", show_copy_button=True)
         text_output_file = gr.File(height=file_input_height, label="Output files")
         latest_batch_completed = gr.Number(value=0, label="Number of files prepared", interactive=False, visible=False)
         # Duplicate version of the above variable for when you don't want to initiate the summarisation loop
@@ -131,14 +138,28 @@ with app:
         with gr.Row():
             s3_logs_output_textbox = gr.Textbox(label="Feedback submission logs", visible=False)
 
-    with gr.Tab(label="Deduplicate and summarise topics"):
+    with gr.Tab(label="Modify, deduplicate, and summarise topic outputs"):
         gr.Markdown(
         """
-        ### Load in previously completed Extract Topics output files ('reference_table', and 'unique_topics' files) to summarise the outputs.
+        Load in previously completed Extract Topics output files ('reference_table', and 'unique_topics' files) to modify topics, deduplicate topics, or summarise the outputs. If you want pivot table outputs, please load in the original data file along with the selected open text column on the first tab before deduplicating or summarising.
         """)
+
+
+
+        with gr.Accordion("Modify existing topics", open = False):
+            modification_input_files = gr.File(height=file_input_height, label="Upload files to modify topics", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
+
+            modifiable_unique_topics_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=(4, "fixed"), row_count = (1, "fixed"), visible=True, type="pandas")
+
+            save_modified_files_button = gr.Button(value="Save modified topic names")
+
+
         with gr.Accordion("Upload reference data file and unique data files", open = True):
-            summarisation_in_previous_data_files = gr.File(height=file_input_height, label="Upload files to deduplicate topics", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
-            summarisation_in_previous_data_files_status = gr.Textbox(value = "", label="Previous file input", visible=False)
+
+            
+            ### DEDUPLICATION
+            deduplication_input_files = gr.File(height=file_input_height, label="Upload files to deduplicate topics", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
+            deduplication_input_files_status = gr.Textbox(value = "", label="Previous file input", visible=False)
 
             with gr.Row():
                 merge_general_topics_drop = gr.Dropdown(label="Merge general topic values together for duplicate subtopics.", value="No", choices=["Yes", "No"])
@@ -146,19 +167,21 @@ with app:
                 deduplicate_score_threshold = gr.Number(label="Similarity threshold with which to determine duplicates.", value = 90, minimum=5, maximum=100, precision=0)
 
             deduplicate_previous_data_btn = gr.Button("Deduplicate topics", variant="primary")
-            
-            duplicate_output_files = gr.File(height=file_input_height, label="Upload files to summarise", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
+
+
+            ### SUMMARISATION            
+            summarisation_input_files = gr.File(height=file_input_height, label="Upload files to summarise", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
 
             summarise_format_radio = gr.Radio(label="Choose summary type", value="Return a summary up to two paragraphs long that includes as much detail as possible from the original text", choices=["Return a summary up to two paragraphs long that includes as much detail as possible from the original text", "Return a concise summary up to one paragraph long that summarises only the most important themes from the original text"])
             
             summarise_previous_data_btn = gr.Button("Summarise topics", variant="primary")
             summary_output_files = gr.File(height=file_input_height, label="Summarised output files", interactive=False)
-            summarised_output_markdown = gr.Markdown(value="### Summarised table will appear here")
+            summarised_output_markdown = gr.Markdown(value="### Summarised table will appear here", show_copy_button=True)
 
-    with gr.Tab(label="Continue previous topic extraction"):
+    with gr.Tab(label="Continue unfinished topic extraction"):
         gr.Markdown(
         """
-        ### Load in data files from a previous attempt at extracting topics to continue it.
+        ### Load in output files from a previous topic extraction process and continue topic extraction with new data.
         """)
 
         with gr.Accordion("Upload reference data file and unique data files", open = True):
@@ -174,7 +197,7 @@ with app:
         """)
     
         in_view_table = gr.File(height=file_input_height, label="Choose unique topic csv files", file_count= "single", file_types=['.csv', '.parquet', '.csv.gz'])
-        view_table_markdown = gr.Markdown(value = "", label="View table")
+        view_table_markdown = gr.Markdown(value = "", label="View table", show_copy_button=True)
 
     with gr.Tab(label="Topic extraction settings"):
         gr.Markdown(
@@ -187,7 +210,7 @@ with app:
             random_seed = gr.Number(value=42, label="Random seed for LLM generation", visible=False)            
 
         with gr.Accordion("Prompt settings", open = True):
-            number_of_prompts = gr.Number(value=1, label="Number of prompts to send to LLM in sequence", minimum=1, maximum=3)
+            number_of_prompts = gr.Number(value=1, label="Number of prompts to send to LLM in sequence", minimum=1, maximum=3, visible=False)
             system_prompt_textbox = gr.Textbox(label="Initial system prompt", lines = 4, value = system_prompt)
             initial_table_prompt_textbox = gr.Textbox(label = "Initial topics prompt", lines = 8, value = initial_table_prompt)
             prompt_2_textbox = gr.Textbox(label = "Prompt 2", lines = 8, value = prompt2, visible=False)
@@ -200,7 +223,7 @@ with app:
 
         # Invisible text box to hold the session hash/username just for logging purposes
         session_hash_textbox = gr.Textbox(label = "Session hash", value="", visible=False) 
-        data_file_names_textbox = gr.Textbox(label = "Data file name", value="", visible=False)
+        
         estimated_time_taken_number = gr.Number(label= "Estimated time taken (seconds)", value=0.0, precision=1, visible=False) # This keeps track of the time taken to redact files for logging purposes.
         total_number_of_batches = gr.Number(label = "Current batch number", value = 1, precision=0, visible=False)
         
@@ -224,40 +247,46 @@ with app:
     ###
 
      # Tabular data upload
-    in_data_files.upload(fn=put_columns_in_df, inputs=[in_data_files], outputs=[in_colnames, in_excel_sheets, data_file_names_textbox])
+    in_data_files.upload(fn=put_columns_in_df, inputs=[in_data_files], outputs=[in_colnames, in_excel_sheets, reference_data_file_name_textbox])
 
     extract_topics_btn.click(fn=empty_output_vars_extract_topics, inputs=None, outputs=[master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number]).\
-    then(load_in_data_file,                           
-        inputs = [in_data_files, in_colnames, batch_size_number], outputs = [file_data_state, data_file_names_textbox, total_number_of_batches], api_name="load_data").then(\
-        fn=extract_topics,                           
-        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, text_output_summary, data_file_names_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, text_output_summary, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox],        
-        outputs=[text_output_summary, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, summarisation_in_previous_data_files, duplicate_output_files], api_name="extract_topics")
+    success(load_in_data_file,                           
+        inputs = [in_data_files, in_colnames, batch_size_number, in_excel_sheets], outputs = [file_data_state, reference_data_file_name_textbox, total_number_of_batches], api_name="load_data").\
+    success(fn=extract_topics,                           
+        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, text_output_summary, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, text_output_summary, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox],        
+        outputs=[text_output_summary, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, deduplication_input_files, summarisation_input_files, modifiable_unique_topics_df_state, modification_input_files], api_name="extract_topics")
     
-    # return display_table, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths
     
     # If the output file count text box changes, keep going with redacting each data file until done. Then reveal the feedback buttons.
     latest_batch_completed.change(fn=extract_topics,                                  
-        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, text_output_summary, data_file_names_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, text_output_summary, text_output_file_list_state, log_files_output_list_state, second_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox],
-        outputs=[text_output_summary, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, summarisation_in_previous_data_files, duplicate_output_files]).\
-        then(fn = reveal_feedback_buttons,
-        outputs=[data_feedback_radio, data_further_details_text, data_submit_feedback_btn, data_feedback_title], scroll_to_output=True)
+        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, text_output_summary, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, text_output_summary, text_output_file_list_state, log_files_output_list_state, second_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox],
+        outputs=[text_output_summary, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, deduplication_input_files, summarisation_input_files, modifiable_unique_topics_df_state, modification_input_files]).\
+        success(fn = reveal_feedback_buttons,
+            outputs=[data_feedback_radio, data_further_details_text, data_submit_feedback_btn, data_feedback_title], scroll_to_output=True)
+    
+    # If you upload data into the deduplication input box, the modifiable topic dataframe box is updated
+    modification_input_files.upload(fn=load_in_previous_data_files, inputs=[modification_input_files, modified_unique_table_change_bool], outputs=[modifiable_unique_topics_df_state, master_modify_reference_df_state, master_modify_unique_topics_df_state, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, text_output_modify_file_list_state])
+   
+
+    # Modify output table with custom topic names
+    save_modified_files_button.click(fn=modify_existing_output_tables, inputs=[master_modify_unique_topics_df_state, modifiable_unique_topics_df_state, master_modify_reference_df_state, text_output_modify_file_list_state], outputs=[master_unique_topics_df_state, master_reference_df_state, text_output_file, text_output_file_list_state, deduplication_input_files, summarisation_input_files, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, summarised_output_markdown])
     
     # When button pressed, deduplicate data
-    deduplicate_previous_data_btn.click(load_in_previous_data_files, inputs=[summarisation_in_previous_data_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, summarisation_in_previous_data_files_status, data_file_names_textbox, unique_topics_table_file_textbox]).\
-    then(deduplicate_topics, inputs=[master_reference_df_state, master_unique_topics_df_state, data_file_names_textbox, unique_topics_table_file_textbox, merge_sentiment_drop, merge_general_topics_drop, deduplicate_score_threshold, in_data_files, in_colnames], outputs=[master_reference_df_state, master_unique_topics_df_state, duplicate_output_files, log_files_output])
+    deduplicate_previous_data_btn.click(load_in_previous_data_files, inputs=[deduplication_input_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, deduplication_input_files_status, reference_data_file_name_textbox, unique_topics_table_file_name_textbox]).\
+        success(deduplicate_topics, inputs=[master_reference_df_state, master_unique_topics_df_state, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, in_excel_sheets, merge_sentiment_drop, merge_general_topics_drop, deduplicate_score_threshold, in_data_files, in_colnames], outputs=[master_reference_df_state, master_unique_topics_df_state, summarisation_input_files, log_files_output, summarised_output_markdown], scroll_to_output=True)
     
     # When button pressed, summarise previous data
     summarise_previous_data_btn.click(empty_output_vars_summarise, inputs=None, outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox]).\
-    then(load_in_previous_data_files, inputs=[duplicate_output_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, summarisation_in_previous_data_files_status, data_file_names_textbox, unique_topics_table_file_textbox]).\
-    then(sample_reference_table_summaries, inputs=[master_reference_df_state, master_unique_topics_df_state, random_seed], outputs=[summary_reference_table_sample_state, summarised_references_markdown, master_reference_df_state, master_unique_topics_df_state]).\
-    then(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
+        success(load_in_previous_data_files, inputs=[summarisation_input_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, deduplication_input_files_status, reference_data_file_name_textbox, unique_topics_table_file_name_textbox]).\
+            success(sample_reference_table_summaries, inputs=[master_reference_df_state, master_unique_topics_df_state, random_seed], outputs=[summary_reference_table_sample_state, summarised_references_markdown, master_reference_df_state, master_unique_topics_df_state]).\
+                success(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
 
-    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
+    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output], scroll_to_output=True)
 
     # If uploaded partially completed consultation files do this. This should then start up the 'latest_batch_completed' change action above to continue extracting topics.
     continue_previous_data_files_btn.click(
-            load_in_data_file, inputs = [in_data_files, in_colnames, batch_size_number], outputs = [file_data_state, data_file_names_textbox, total_number_of_batches]).\
-            then(load_in_previous_data_files, inputs=[in_previous_data_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed, in_previous_data_files_status, data_file_names_textbox])
+            load_in_data_file, inputs = [in_data_files, in_colnames, batch_size_number, in_excel_sheets], outputs = [file_data_state, reference_data_file_name_textbox, total_number_of_batches]).\
+            success(load_in_previous_data_files, inputs=[in_previous_data_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed, in_previous_data_files_status, reference_data_file_name_textbox])
 
     ###
     # LOGGING AND ON APP LOAD FUNCTIONS
@@ -268,21 +297,21 @@ with app:
     access_callback = gr.CSVLogger(dataset_file_name=log_file_name)
     access_callback.setup([session_hash_textbox], access_logs_data_folder)
     session_hash_textbox.change(lambda *args: access_callback.flag(list(args)), [session_hash_textbox], None, preprocess=False).\
-    then(fn = upload_file_to_s3, inputs=[access_logs_state, access_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+        success(fn = upload_file_to_s3, inputs=[access_logs_state, access_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
     # Log usage usage when making a query
     usage_callback = gr.CSVLogger(dataset_file_name=log_file_name)
-    usage_callback.setup([session_hash_textbox, data_file_names_textbox, model_choice, conversation_metadata_textbox, estimated_time_taken_number], usage_data_folder)
+    usage_callback.setup([session_hash_textbox, reference_data_file_name_textbox, model_choice, conversation_metadata_textbox, estimated_time_taken_number], usage_data_folder)
 
-    conversation_metadata_textbox.change(lambda *args: usage_callback.flag(list(args)), [session_hash_textbox, data_file_names_textbox, model_choice, conversation_metadata_textbox, estimated_time_taken_number], None, preprocess=False).\
-    then(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+    conversation_metadata_textbox.change(lambda *args: usage_callback.flag(list(args)), [session_hash_textbox, reference_data_file_name_textbox, model_choice, conversation_metadata_textbox, estimated_time_taken_number], None, preprocess=False).\
+        success(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
     # User submitted feedback
     feedback_callback = gr.CSVLogger(dataset_file_name=log_file_name)
-    feedback_callback.setup([data_feedback_radio, data_further_details_text, data_file_names_textbox, model_choice, temperature_slide, text_output_summary, conversation_metadata_textbox], feedback_data_folder)
+    feedback_callback.setup([data_feedback_radio, data_further_details_text, reference_data_file_name_textbox, model_choice, temperature_slide, text_output_summary, conversation_metadata_textbox], feedback_data_folder)
 
-    data_submit_feedback_btn.click(lambda *args: feedback_callback.flag(list(args)), [data_feedback_radio, data_further_details_text, data_file_names_textbox, model_choice, temperature_slide, text_output_summary, conversation_metadata_textbox], None, preprocess=False).\
-    then(fn = upload_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[data_further_details_text])
+    data_submit_feedback_btn.click(lambda *args: feedback_callback.flag(list(args)), [data_feedback_radio, data_further_details_text, reference_data_file_name_textbox, model_choice, temperature_slide, text_output_summary, conversation_metadata_textbox], None, preprocess=False).\
+        success(fn = upload_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[data_further_details_text])
 
     in_view_table.upload(view_table, inputs=[in_view_table], outputs=[view_table_markdown])
 

tools/helper_functions.py

@@ -106,22 +106,31 @@ def detect_file_type(filename):
     else:
         raise ValueError("Unsupported file type.")
 
-def read_file(filename):
+def read_file(filename:str, sheet:str=""):
     """Read the file based on its detected type."""
     file_type = detect_file_type(filename)
     
     if file_type == 'csv':
         return pd.read_csv(filename, low_memory=False)
     elif file_type == 'xlsx':
-        return pd.read_excel(filename)
+        if sheet:
+            return pd.read_excel(filename, sheet_name=sheet)
+        else:
+            return pd.read_excel(filename)
     elif file_type == 'parquet':
         return pd.read_parquet(filename)
     
 # Wrap text in each column to the specified max width, including whole words
-def wrap_text(text, max_width=60):
+def wrap_text(text:str, max_width=60, max_text_length=None):
     if not isinstance(text, str):
         return text
         
+    # If max_text_length is set, truncate the text and add ellipsis
+    if max_text_length and len(text) > max_text_length:
+        return text[:max_text_length] + '...'
+    
+    text = text.replace('\r\n', '<br>').replace('\n', '<br>')
+    
     words = text.split()
     if not words:
         return text

tools/llm_api_call.py

@@ -1,4 +1,5 @@
 import os
+import os
 import google.generativeai as ai
 import pandas as pd
 import numpy as np
@@ -57,7 +58,7 @@ def normalise_string(text):
     
     return text
 
-def load_in_file(file_path: str, colname:str=""):
+def load_in_file(file_path: str, colname:str="", excel_sheet:str=""):
     """
     Loads in a tabular data file and returns data and file name.
 
@@ -68,7 +69,7 @@ def load_in_file(file_path: str, colname:str=""):
     #print("File type is:", file_type)
 
     file_name = get_file_name_no_ext(file_path)
-    file_data = read_file(file_path)
+    file_data = read_file(file_path, excel_sheet)
 
     if colname:
         file_data[colname] = file_data[colname].fillna("")
@@ -79,11 +80,11 @@ def load_in_file(file_path: str, colname:str=""):
 
     return file_data, file_name
 
-def load_in_data_file(file_paths:List[str], in_colnames:List[str], batch_size:int=50):
+def load_in_data_file(file_paths:List[str], in_colnames:List[str], batch_size:int=50, in_excel_sheets:str=""):
     '''Load in data table, work out how many batches needed.'''
 
     try:
-        file_data, file_name = load_in_file(file_paths[0], colname=in_colnames)
+        file_data, file_name = load_in_file(file_paths[0], colname=in_colnames, excel_sheet=in_excel_sheets)
         num_batches = math.ceil(len(file_data) / batch_size)
         print("Total number of batches:", num_batches)
 
@@ -95,7 +96,7 @@ def load_in_data_file(file_paths:List[str], in_colnames:List[str], batch_size:in
     
     return file_data, file_name, num_batches
 
-def load_in_previous_data_files(file_paths_partial_output:List[str]):
+def load_in_previous_data_files(file_paths_partial_output:List[str], for_modified_table:bool=False):
     '''Load in data table from a partially completed consultation summary to continue it.'''
 
     reference_file_data = pd.DataFrame()
@@ -106,24 +107,25 @@ def load_in_previous_data_files(file_paths_partial_output:List[str]):
     latest_batch = 0
 
     for file in file_paths_partial_output:
+
         # If reference table
         if 'reference_table' in file.name:
             try:
                 reference_file_data, reference_file_name = load_in_file(file)
                 #print("reference_file_data:", reference_file_data.head(2))
-                out_message = out_message + " Reference file load successful"
+                out_message = out_message + " Reference file load successful."
             except Exception as e:
                 out_message = "Could not load reference file data:" + str(e)
-                print("Could not load reference file data:", e)
+                raise Exception("Could not load reference file data:", e)
         # If unique table
         if 'unique_topics' in file.name:
             try:
                 unique_file_data, unique_file_name = load_in_file(file)
                 #print("unique_topics_file:", unique_file_data.head(2))
-                out_message = out_message + " Unique table file load successful"
+                out_message = out_message + " Unique table file load successful."
             except Exception as e:
                 out_message = "Could not load unique table file data:" + str(e)
-                print("Could not load unique table file data:", e)
+                raise Exception("Could not load unique table file data:", e)
         if 'batch_' in file.name:
             latest_batch = re.search(r'batch_(\d+)', file.name).group(1)
             print("latest batch:", latest_batch)
@@ -133,12 +135,37 @@ def load_in_previous_data_files(file_paths_partial_output:List[str]):
         out_message = out_message + " Latest batch number not found."
     if reference_file_data.empty:
         out_message = out_message + " No reference data table provided."
+        raise Exception(out_message)
     if unique_file_data.empty:
         out_message = out_message + " No unique data table provided."   
 
     print(out_message)
-            
-    return reference_file_data, unique_file_data, latest_batch, out_message, reference_file_name, unique_file_name
+
+    # Return all data if using for deduplication task. Return just modified unique table if using just for table modification
+    if for_modified_table == False:            
+        return reference_file_data, unique_file_data, latest_batch, out_message, reference_file_name, unique_file_name
+    else:
+        
+        reference_file_data.drop("Topic_number", axis=1, inplace=True, errors="ignore")
+
+        unique_file_data = create_unique_table_df_from_reference_table(reference_file_data)
+
+        unique_file_data.drop("Summary",axis=1, inplace=True)
+
+        # Then merge the topic numbers back to the original dataframe
+        reference_file_data = reference_file_data.merge(
+            unique_file_data[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
+            on=['General Topic', 'Subtopic', 'Sentiment'],
+            how='left'
+        )        
+
+        out_file_names = [reference_file_name + ".csv"]
+        out_file_names.append(unique_file_name + ".csv")
+
+        print("reference_file_name:", reference_file_name)
+        print("unique_file_name:", unique_file_name)
+
+        return gr.Dataframe(value=unique_file_data, headers=None, col_count=(unique_file_data.shape[1], "fixed"), row_count = (unique_file_data.shape[0], "fixed"), visible=True, type="pandas"), reference_file_data, unique_file_data, reference_file_name, unique_file_name, out_file_names
 
 def get_basic_response_data(file_data:pd.DataFrame, chosen_cols:List[str]) -> pd.DataFrame:
     basic_response_data = file_data[[chosen_cols]].reset_index(names="Reference")
@@ -190,7 +217,7 @@ def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_co
     batch_basic_response_data = basic_response_data[start_row:end_row]  # Select the current batch
 
     # Now replace the reference numbers with numbers starting from 1
-    batch_basic_response_data["Reference"] = batch_basic_response_data["Reference"] - start_row
+    batch_basic_response_data.loc[:, "Reference"] = batch_basic_response_data["Reference"] - start_row
 
     #print("batch_basic_response_data:", batch_basic_response_data)
 
@@ -612,30 +639,43 @@ def clean_column_name(column_name, max_length=20):
     return column_name[:max_length]
 
 def create_unique_table_df_from_reference_table(reference_df:pd.DataFrame):
-    new_unique_topics_df = reference_df[["General Topic", "Subtopic", "Sentiment"]]
 
-    new_unique_topics_df = new_unique_topics_df.rename(columns={new_unique_topics_df.columns[0]: "General Topic", new_unique_topics_df.columns[1]: "Subtopic", new_unique_topics_df.columns[2]: "Sentiment"})
+    out_unique_topics_df = (reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
+            .agg({
+                'Response References': 'size',  # Count the number of references
+                'Summary': lambda x: '<br>'.join(
+                    sorted(set(x), key=lambda summary: reference_df.loc[reference_df['Summary'] == summary, 'Start row of group'].min())
+                )
+            })
+            .reset_index()
+            .sort_values('Response References', ascending=False)  # Sort by size, biggest first
+            .assign(Topic_number=lambda df: np.arange(1, len(df) + 1))  # Add numbering 1 to x
+        )
+
+    # new_unique_topics_df = reference_df[["General Topic", "Subtopic", "Sentiment"]]
+
+    # new_unique_topics_df = new_unique_topics_df.rename(columns={new_unique_topics_df.columns[0]: "General Topic", new_unique_topics_df.columns[1]: "Subtopic", new_unique_topics_df.columns[2]: "Sentiment"})
     
-    # Join existing and new unique topics
-    out_unique_topics_df = new_unique_topics_df
+    # # Join existing and new unique topics
+    # out_unique_topics_df = new_unique_topics_df
 
-    out_unique_topics_df = out_unique_topics_df.rename(columns={out_unique_topics_df.columns[0]: "General Topic", out_unique_topics_df.columns[1]: "Subtopic", out_unique_topics_df.columns[2]: "Sentiment"})
+    # out_unique_topics_df = out_unique_topics_df.rename(columns={out_unique_topics_df.columns[0]: "General Topic", out_unique_topics_df.columns[1]: "Subtopic", out_unique_topics_df.columns[2]: "Sentiment"})
 
-    #print("out_unique_topics_df:", out_unique_topics_df)
+    # #print("out_unique_topics_df:", out_unique_topics_df)
 
-    out_unique_topics_df = out_unique_topics_df.drop_duplicates(["General Topic", "Subtopic", "Sentiment"]).\
-            drop(["Response References", "Summary"], axis = 1, errors="ignore") 
+    # out_unique_topics_df = out_unique_topics_df.drop_duplicates(["General Topic", "Subtopic", "Sentiment"]).\
+    #         drop(["Response References", "Summary"], axis = 1, errors="ignore") 
 
-    # Get count of rows that refer to particular topics
-    reference_counts = reference_df.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
-    'Response References': 'size',  # Count the number of references
-    'Summary': lambda x: '<br>'.join(
-        sorted(set(x), key=lambda summary: reference_df.loc[reference_df['Summary'] == summary, 'Start row of group'].min())
-    )
-    }).reset_index()
+    # # Get count of rows that refer to particular topics
+    # reference_counts = reference_df.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
+    # 'Response References': 'size',  # Count the number of references
+    # 'Summary': lambda x: '<br>'.join(
+    #     sorted(set(x), key=lambda summary: reference_df.loc[reference_df['Summary'] == summary, 'Start row of group'].min())
+    # )
+    # }).reset_index()
 
-    # Join the counts to existing_unique_topics_df
-    out_unique_topics_df = out_unique_topics_df.merge(reference_counts, how='left', on=["General Topic", "Subtopic", "Sentiment"]).sort_values("Response References", ascending=False)
+    # # Join the counts to existing_unique_topics_df
+    # out_unique_topics_df = out_unique_topics_df.merge(reference_counts, how='left', on=["General Topic", "Subtopic", "Sentiment"]).sort_values("Response References", ascending=False)
 
     return out_unique_topics_df
 
@@ -762,13 +802,11 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
     batch_file_path_details = f"{file_name}_batch_{latest_batch_completed + 1}_size_{batch_size_number}_col_{in_column_cleaned}"
     row_number_string_start = f"Rows {start_row_reported} to {end_row}: "
 
-    print("batch_file_path_details:", batch_file_path_details)
-
     whole_conversation_path = output_folder + batch_file_path_details + "_full_conversation_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
     whole_conversation_path_meta = output_folder + batch_file_path_details + "_metadata_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
 
-    #with open(whole_conversation_path, "w", encoding='utf-8', errors='replace') as f:
-    #    f.write(whole_conversation_str)
+    with open(whole_conversation_path, "w", encoding='utf-8', errors='replace') as f:
+        f.write(whole_conversation_str)
 
     with open(whole_conversation_path_meta, "w", encoding='utf-8', errors='replace') as f:
         f.write(whole_conversation_metadata_str)
@@ -799,9 +837,9 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
     topic_with_response_df["Response References"] = topic_with_response_df["Response References"].astype(str).str.replace(".0", "", regex=False)
 
     # Strip and lower case topic names to remove issues where model is randomly capitalising topics/sentiment
-    topic_with_response_df["General Topic"] = topic_with_response_df["General Topic"].str.strip().str.lower().str.capitalize()
-    topic_with_response_df["Subtopic"] = topic_with_response_df["Subtopic"].str.strip().str.lower().str.capitalize()
-    topic_with_response_df["Sentiment"] = topic_with_response_df["Sentiment"].str.strip().str.lower().str.capitalize()
+    topic_with_response_df["General Topic"] = topic_with_response_df["General Topic"].astype(str).str.strip().str.lower().str.capitalize()
+    topic_with_response_df["Subtopic"] = topic_with_response_df["Subtopic"].astype(str).str.strip().str.lower().str.capitalize()
+    topic_with_response_df["Sentiment"] = topic_with_response_df["Sentiment"].astype(str).str.strip().str.lower().str.capitalize()
     
     topic_table_out_path = output_folder + batch_file_path_details + "_topic_table_" + model_choice_clean + "_temp_" + str(temperature) + ".csv"
 
@@ -860,7 +898,10 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
     except Exception as e:
         print("Could not convert Response References column to integer due to", e)
 
-    out_reference_df.sort_values(["Start row of group", "Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True) 
+    out_reference_df.sort_values(["Start row of group", "Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
+
+    # Each topic should only be associated with each individual response once
+    out_reference_df.drop_duplicates(["Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
 
     # Save the new DataFrame to CSV
     reference_table_out_path = output_folder + batch_file_path_details + "_reference_table_" + model_choice_clean + "_temp_" + str(temperature) + ".csv"    
@@ -884,9 +925,7 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
     # Get count of rows that refer to particular topics
     reference_counts = out_reference_df.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
     'Response References': 'size',  # Count the number of references
-    'Summary': lambda x: '<br>'.join(
-        sorted(set(x), key=lambda summary: out_reference_df.loc[out_reference_df['Summary'] == summary, 'Start row of group'].min())
-    )
+    'Summary': ' <br> '.join
     }).reset_index()
 
     # Join the counts to existing_unique_topics_df
@@ -1006,7 +1045,8 @@ def extract_topics(in_data_file,
             # Check if files and text exist
             out_message = "Please enter a data file to summarise."
             print(out_message)
-            return out_message, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths#, out_message
+            raise Exception(out_message)
+            #return out_message, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths#, out_message
 
 
     #model_choice_clean = replace_punctuation_with_underscore(model_choice)
@@ -1014,20 +1054,20 @@ def extract_topics(in_data_file,
 
     # If this is the first time around, set variables to 0/blank
     if first_loop_state==True:
-        print("This is the first time through the loop")
+        #print("This is the first time through the loop")
         if (latest_batch_completed == 999) | (latest_batch_completed == 0):
             latest_batch_completed = 0
             out_message = []
             out_file_paths = []
-            print("model_choice_clean:", model_choice_clean)
+            #print("model_choice_clean:", model_choice_clean)
 
             if (model_choice == "gemma_2b_it_local") & (RUN_LOCAL_MODEL == "1"):
                 progress(0.1, "Loading in Gemma 2b model")
                 local_model, tokenizer = load_model()
                 print("Local model loaded:", local_model)
 
-    print("latest_batch_completed at start of function:", str(latest_batch_completed))
-    print("total number of batches:", str(num_batches))
+    #print("latest_batch_completed at start of function:", str(latest_batch_completed))
+    #print("total number of batches:", str(num_batches))
 
     # If we have already redacted the last file, return the input out_message and file list to the relevant components
     if latest_batch_completed >= num_batches:
@@ -1071,8 +1111,11 @@ def extract_topics(in_data_file,
         existing_reference_df.to_csv(reference_table_out_path, index=None)
         out_file_paths.append(reference_table_out_path)
 
+        # Create final unique topics table from reference table to ensure consistent numbers
+        final_out_unique_topics_df = create_unique_table_df_from_reference_table(existing_reference_df)
+
         ## Unique topic list
-        existing_unique_topics_df.to_csv(unique_topics_df_out_path, index=None)
+        final_out_unique_topics_df.to_csv(unique_topics_df_out_path, index=None)
         out_file_paths.append(unique_topics_df_out_path)
 
         # Ensure that we are only returning the final results to outputs
@@ -1107,7 +1150,7 @@ def extract_topics(in_data_file,
 
         # Step 3: Populate the new DataFrame
         missing_df['Response References'] = missing_references['Reference']
-        missing_df = missing_df.fillna(np.nan)  # Fill other columns with NA
+        missing_df = missing_df.fillna(np.nan) #.infer_objects(copy=False)  # Fill other columns with NA
 
         # Display the new DataFrame
         #print("missing_df:", missing_df)
@@ -1120,10 +1163,12 @@ def extract_topics(in_data_file,
         log_files_output_paths = list(set(log_files_output_paths))        
 
         summary_out_file_paths = [file_path for file_path in out_file_paths if "final_" in file_path]
-        print("summary_out_file_paths:", summary_out_file_paths)
+ 
+        # The topic table that can be modified does not need the summary column
+        modifiable_unique_topics_df = final_out_unique_topics_df.drop("Summary", axis=1)
 
         #final_out_message = '\n'.join(out_message)
-        return display_table, existing_topics_table, existing_unique_topics_df, existing_reference_df, summary_out_file_paths, summary_out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths
+        return display_table, existing_topics_table, final_out_unique_topics_df, existing_reference_df, summary_out_file_paths, summary_out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths, gr.Dataframe(value=modifiable_unique_topics_df, headers=None, col_count=(modifiable_unique_topics_df.shape[1], "fixed"), row_count = (modifiable_unique_topics_df.shape[0], "fixed"), visible=True, type="pandas"), out_file_paths
        
     
     if num_batches > 0:
@@ -1144,7 +1189,8 @@ def extract_topics(in_data_file,
     if model_choice == "anthropic.claude-3-sonnet-20240229-v1:0" and file_data.shape[1] > 300:
         out_message = "Your data has more than 300 rows, using the Sonnet model will be too expensive. Please choose the Haiku model instead."
         print(out_message)
-        return out_message, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths#, out_message
+        raise Exception(out_message)
+        #return out_message, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths#, out_message
         
     
     if sentiment_checkbox == "Negative, Neutral, or Positive": sentiment_prompt = "In the third column, write the sentiment of the Subtopic: Negative, Neutral, or Positive"
@@ -1155,9 +1201,6 @@ def extract_topics(in_data_file,
     topics_loop_description = "Extracting topics from response batches (each batch of " + str(batch_size) + " responses)."
     topics_loop = tqdm(range(latest_batch_completed, num_batches), desc = topics_loop_description, unit="batches remaining")
 
-
-
-    
     for i in topics_loop:       
         #for latest_batch_completed in range(num_batches):
         reported_batch_no = latest_batch_completed + 1  
@@ -1167,8 +1210,6 @@ def extract_topics(in_data_file,
         simplified_csv_table_path, normalised_simple_markdown_table, start_row, end_row, batch_basic_response_df = data_file_to_markdown_table(file_data, file_name, chosen_cols, output_folder, latest_batch_completed, batch_size)
         #log_files_output_paths.append(simplified_csv_table_path)
 
-
-
         # Conversation history
         conversation_history = []
 
@@ -1177,7 +1218,7 @@ def extract_topics(in_data_file,
         # If the latest batch of responses contains at least one instance of text
         if not batch_basic_response_df.empty:
 
-            print("latest_batch_completed:", latest_batch_completed)
+            #print("latest_batch_completed:", latest_batch_completed)
 
             #print("candidate_topics:", candidate_topics)
 
@@ -1198,26 +1239,58 @@ def extract_topics(in_data_file,
                 # Preparing candidate topics
                 if candidate_topics and existing_unique_topics_df.empty:
                     progress(0.1, "Creating revised zero shot topics table")
+
                     # 'Zero shot topics' are those supplied by the user
                     max_topic_no = 120
-
                     zero_shot_topics = read_file(candidate_topics.name)
 
-                    if zero_shot_topics.shape[1] == 1:  # Check if there is only one column
-                        zero_shot_topics_series = zero_shot_topics.iloc[:, 0].str.strip().str.lower().str.capitalize()
-                        # Max 120 topics allowed
-                        if len(zero_shot_topics_series) > max_topic_no:
-                            print("Maximum", max_topic_no, "topics allowed to fit within large language model context limits.")
-                            zero_shot_topics_series = zero_shot_topics_series.iloc[:max_topic_no]
-
-                        zero_shot_topics_list = list(zero_shot_topics_series)
+                    # Max 120 topics allowed
+                    if zero_shot_topics.shape[0] > max_topic_no:
+                        print("Maximum", max_topic_no, "topics allowed to fit within large language model context limits.")
+                        zero_shot_topics = zero_shot_topics.iloc[:max_topic_no, :]
+
+                    # Forward slashes in the topic names seems to confuse the model
+                    if zero_shot_topics.shape[1] >= 1:  # Check if there is at least one column                       
+                        for x in zero_shot_topics.columns:
+                            zero_shot_topics.loc[:, x] = (
+                            zero_shot_topics.loc[:, x]
+                            .str.strip()
+                            .str.replace('\n', ' ')
+                            .str.replace('\r', ' ')
+                            .str.replace('/', ' or ')
+                            .str.lower()
+                            .str.capitalize())
+                 
+                        # If number of columns is 1, keep only subtopics
+                        if zero_shot_topics.shape[1] == 1 and "General Topic" not in zero_shot_topics.columns: 
+                            zero_shot_topics_gen_topics_list = [""] * zero_shot_topics.shape[0]
+                            zero_shot_topics_subtopics_list = list(zero_shot_topics.iloc[:, 0])                        
+                        # Allow for possibility that the user only wants to set general topics and not subtopics
+                        elif zero_shot_topics.shape[1] == 1 and "General Topic" in zero_shot_topics.columns: 
+                            zero_shot_topics_gen_topics_list = list(zero_shot_topics["General Topic"])
+                            zero_shot_topics_subtopics_list = [""] * zero_shot_topics.shape[0]
+                        # If general topic and subtopic are specified
+                        elif set(["General Topic", "Subtopic"]).issubset(zero_shot_topics.columns):
+                            zero_shot_topics_gen_topics_list = list(zero_shot_topics["General Topic"])
+                            zero_shot_topics_subtopics_list = list(zero_shot_topics["Subtopic"])
+                        # If number of columns is 2, keep general topics and subtopics
+                        elif zero_shot_topics.shape[1] == 2: 
+                            zero_shot_topics_gen_topics_list = list(zero_shot_topics.iloc[:, 0])
+                            zero_shot_topics_subtopics_list = list(zero_shot_topics.iloc[:, 1])
+                        else:
+                            # If there are more columns, just assume that the first column was meant to be a subtopic
+                            zero_shot_topics_gen_topics_list = [""] * zero_shot_topics.shape[0]
+                            zero_shot_topics_subtopics_list = list(zero_shot_topics.iloc[:, 0])
 
-                        print("Zero shot topics are:", zero_shot_topics_list)
+                        
 
                         if create_revised_general_topics == True:
                             # Create the most up to date list of topics and subtopics.
                             # If there are candidate topics, but the existing_unique_topics_df hasn't yet been constructed, then create.
-                            unique_topics_df = pd.DataFrame(data={"General Topic":[""] * len(zero_shot_topics_list), "Subtopic":zero_shot_topics_list})
+                            unique_topics_df = pd.DataFrame(data={
+                                "General Topic":zero_shot_topics_gen_topics_list,
+                                "Subtopic":zero_shot_topics_subtopics_list
+                                })
                             unique_topics_markdown = unique_topics_df.to_markdown()
 
                             print("unique_topics_markdown:", unique_topics_markdown)
@@ -1244,16 +1317,24 @@ def extract_topics(in_data_file,
                                 zero_shot_revised_path = output_folder + "zero_shot_topics_with_general_topics.csv"
                                 zero_shot_topics_df.to_csv(zero_shot_revised_path, index = None)
                                 out_file_paths.append(zero_shot_revised_path)
+
                             except Exception as e:
-                                print("Error in parsing markdown table from response text:", e)
-                                print("Not adding revised General Topics to table")
-                                zero_shot_topics_df = pd.DataFrame(data={"General Topic":[""] * len(zero_shot_topics_list), "Subtopic":zero_shot_topics_list})
+                                print("Error in parsing markdown table from response text:", e, "Not adding revised General Topics to table")
+                                zero_shot_topics_df = pd.DataFrame(data={
+                                    "General Topic":zero_shot_topics_gen_topics_list,
+                                    "Subtopic":zero_shot_topics_subtopics_list})
 
                             if zero_shot_topics_df.empty:
                                 print("Creation of revised general topics df failed, reverting to original list")
-                                zero_shot_topics_df = pd.DataFrame(data={"General Topic":[""] * len(zero_shot_topics_list), "Subtopic":zero_shot_topics_list})
+                                zero_shot_topics_df = pd.DataFrame(data={
+                                    "General Topic":zero_shot_topics_gen_topics_list,
+                                    "Subtopic":zero_shot_topics_subtopics_list})
                         else:
-                            zero_shot_topics_df = pd.DataFrame(data={"General Topic":[""] * len(zero_shot_topics_list), "Subtopic":zero_shot_topics_list})
+                            zero_shot_topics_df = pd.DataFrame(data={
+                                "General Topic":zero_shot_topics_gen_topics_list,
+                                "Subtopic":zero_shot_topics_subtopics_list})
+                            
+                        print("Zero shot topics are:", zero_shot_topics_df)
 
                         # This part concatenates all zero shot and new topics together, so that for the next prompt the LLM will have the full list available
                         if not existing_unique_topics_df.empty:
@@ -1261,18 +1342,6 @@ def extract_topics(in_data_file,
                         else:
                             existing_unique_topics_df = zero_shot_topics_df
 
-                    # If your zero shot column file already contains General Topic and Subtopic columns
-                    if set(["General Topic", "Subtopic"]).issubset(zero_shot_topics.columns):
-                        # Max 120 topics allowed
-                        if zero_shot_topics.shape[0] > max_topic_no:
-                            print("Maximum", max_topic_no, "topics allowed to fit within large language model context limits.")
-                            zero_shot_topics = zero_shot_topics.iloc[:max_topic_no,:]
-
-                        if existing_unique_topics_df.empty:
-                            existing_unique_topics_df = pd.DataFrame(data={'General Topic':zero_shot_topics.iloc[:,0], 'Subtopic':zero_shot_topics.iloc[:,1]})
-                        
-                        zero_shot_topics_df = zero_shot_topics
-
                 if candidate_topics and not zero_shot_topics_df.empty:
                     # If you have already created revised zero shot topics, concat to the current
                     existing_unique_topics_df = pd.concat([existing_unique_topics_df, zero_shot_topics_df])
@@ -1282,7 +1351,7 @@ def extract_topics(in_data_file,
                 #all_topic_tables_df_merged = existing_unique_topics_df
                 existing_unique_topics_df["Response References"] = ""
 
-                unique_topics_markdown = existing_unique_topics_df[["General Topic", "Subtopic"]].drop_duplicates(["Subtopic"]).to_markdown(index=False)
+                unique_topics_markdown = existing_unique_topics_df[["General Topic", "Subtopic"]].drop_duplicates(["General Topic", "Subtopic"]).to_markdown(index=False)
             
                 #existing_unique_topics_df.to_csv(output_folder + f"{file_name}_existing_unique_topics_df_" + #model_choice_clean + "_temp_" + str(temperature) + "_batch_" + str(latest_batch_completed) + ".csv", index=None)
 
@@ -1317,17 +1386,36 @@ def extract_topics(in_data_file,
                 summary_whole_conversation = []
 
                 # Process requests to large language model
-                master_summary_response, summary_conversation_history, whole_summary_conversation, whole_conversation_metadata, response_text = process_requests(summary_prompt_list, add_existing_topics_system_prompt, summary_conversation_history, summary_whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no, local_model, master = True)
+                responses, summary_conversation_history, whole_conversation, whole_conversation_metadata, response_text = process_requests(summary_prompt_list, add_existing_topics_system_prompt, summary_conversation_history, summary_whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no, local_model, master = True)
 
-                # print("master_summary_response:", master_summary_response[-1].text)
+                # print("responses:", responses[-1].text)
                 # print("Whole conversation metadata:", whole_conversation_metadata)
 
-                topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, new_topic_df, new_markdown_table, new_reference_df, new_unique_topics_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(master_summary_response, whole_summary_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=False)
+                topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, new_topic_df, new_markdown_table, new_reference_df, new_unique_topics_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=False)
+
+                # Write final output to text file for logging purposes
+                try:
+                    final_table_output_path = output_folder + master_batch_out_file_part + "_full_final_response_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+
+                    if isinstance(responses[-1], ResponseObject):
+                        with open(final_table_output_path, "w", encoding='utf-8', errors='replace') as f:
+                            f.write(responses[-1].text)
+                    elif "choices" in responses[-1]:
+                        with open(final_table_output_path, "w", encoding='utf-8', errors='replace') as f:
+                            f.write(responses[-1]["choices"][0]['text'])
+                    else:
+                        with open(final_table_output_path, "w", encoding='utf-8', errors='replace') as f:
+                            f.write(responses[-1].text)
+
+                except Exception as e:
+                    print("Error in returning model response:", e)
+                
 
                 # If error in table parsing, leave function
                 if is_error == True:
                     final_message_out = "Could not complete summary, error in LLM output."
-                    display_table, new_topic_df, new_unique_topics_df, new_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths#, final_message_out
+                    raise Exception(final_message_out)
+                    #return display_table, new_topic_df, new_unique_topics_df, new_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths#, final_message_out
 
                 # Write outputs to csv
                 ## Topics with references
@@ -1347,7 +1435,7 @@ def extract_topics(in_data_file,
                 #all_topic_tables_df.append(new_topic_df)
                 #all_markdown_topic_tables.append(new_markdown_table)
 
-                #display_table = master_summary_response[-1].text
+                #display_table = responses[-1].text
 
                 # Show unique topics alongside document counts as output
                 display_table = new_unique_topics_df.to_markdown(index=False)
@@ -1372,6 +1460,8 @@ def extract_topics(in_data_file,
                 if model_choice in ["gemini-2.0-flash", "gemini-1.5-pro-002"]:
                     print("Using Gemini model:", model_choice)
                     model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=system_prompt, max_tokens=max_tokens)
+                elif model_choice in ["gemma_2b_it_local"]:
+                    print("Using local Gemma 2b model")
                 else:
                     print("Using AWS Bedrock model:", model_choice)
 
@@ -1449,7 +1539,7 @@ def extract_topics(in_data_file,
                     log_files_output_paths.append(final_table_output_path)
 
                 except Exception as e:
-                    print(e)
+                    print("Error in returning model response:", e)
                 
                 new_topic_df = topic_table_df
                 new_reference_df = reference_df
@@ -1476,6 +1566,9 @@ def extract_topics(in_data_file,
         existing_unique_topics_df = new_unique_topics_df.dropna(how='all')
         existing_topics_table = new_topic_df.dropna(how='all')
 
+        # The topic table that can be modified does not need the summary column
+        modifiable_unique_topics_df = existing_unique_topics_df.drop("Summary", axis=1)
+
     out_time = f"{final_time:0.1f} seconds."
     
     out_message.append('All queries successfully completed in')
@@ -1485,10 +1578,8 @@ def extract_topics(in_data_file,
 
     print(final_message_out)
 
-    #print("out_file_paths:", out_file_paths)
-    #print("log_files_output_paths:", log_files_output_paths)
 
-    return display_table, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths
+    return display_table, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths, gr.Dataframe(value=modifiable_unique_topics_df, headers=None, col_count=(modifiable_unique_topics_df.shape[1], "fixed"), row_count = (modifiable_unique_topics_df.shape[0], "fixed"), visible=True, type="pandas"), out_file_paths
 
 def convert_reference_table_to_pivot_table(df:pd.DataFrame, basic_response_data:pd.DataFrame=pd.DataFrame()):
 
@@ -1521,9 +1612,115 @@ def convert_reference_table_to_pivot_table(df:pd.DataFrame, basic_response_data:
     return pivot_table
 
 
-# SUMMARISATION FUNCTIONS
+def join_modified_topic_names_to_ref_table(modified_unique_topics_df:pd.DataFrame, original_unique_topics_df:pd.DataFrame, reference_df:pd.DataFrame):
+    '''
+    Take a unique topic table that has been modified by the user, and apply the topic name changes to the long-form reference table.
+    '''
+
+    # Drop rows where Response References is either NA or null
+    modified_unique_topics_df = modified_unique_topics_df[~modified_unique_topics_df["Response References"].isnull()]
+    modified_unique_topics_df.drop_duplicates(["General Topic", "Subtopic", "Sentiment", "Topic_number"], inplace=True)
+
+    # First, join the modified topics to the original topics dataframe based on index to have the modified names alongside the original names
+    original_unique_topics_df_m = original_unique_topics_df.merge(modified_unique_topics_df[["General Topic", "Subtopic", "Sentiment", "Topic_number"]], on="Topic_number", how="left", suffixes=("", "_mod"))
+
+    original_unique_topics_df_m.to_csv(output_folder + "original_unique_topics_df_m.csv")
+
+    original_unique_topics_df_m.drop_duplicates(["General Topic", "Subtopic", "Sentiment", "Topic_number"], inplace=True)
+
+    reference_df.to_csv(output_folder + "before_join_reference_df.csv")
+
+
+    # Then, join these new topic names onto the reference_df, merge based on the original names
+    modified_reference_df = reference_df.merge(original_unique_topics_df_m[["Topic_number", "General Topic_mod", "Subtopic_mod", "Sentiment_mod"]], on=["Topic_number"], how="left")
+
+    modified_reference_df.to_csv(output_folder + "modified_reference_df.csv")
+
+    # Replace old topic names with new topic names in reference_df
+    # modified_reference_df.rename(columns={"General Topic":"General Topic_old",
+    #                                                              "Subtopic":"Subtopic_old",
+    #                                                              "Sentiment":"Sentiment_old"}, inplace=True)
+    
+    modified_reference_df.drop(["General Topic", "Subtopic", "Sentiment"], axis=1, inplace=True, errors="ignore")
+    
+    modified_reference_df.rename(columns={"General Topic_mod":"General Topic",
+                                                                 "Subtopic_mod":"Subtopic",
+                                                                 "Sentiment_mod":"Sentiment"}, inplace=True)
+
+    modified_reference_df.drop(["General Topic_mod", "Subtopic_mod", "Sentiment_mod"], inplace=True, errors="ignore")  
+    
+
+    #modified_reference_df.drop_duplicates(["Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
+
+    modified_reference_df.sort_values(["Start row of group", "Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
+
+    modified_reference_df.to_csv(output_folder + "test_out_ref_df.csv")
+
+    modified_reference_df = modified_reference_df.loc[:, ["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group", "Topic_number"]]
+
+    # Drop rows where Response References is either NA or null
+    modified_reference_df = modified_reference_df[~modified_reference_df["Response References"].isnull()]
+
+    return modified_reference_df
+
+# MODIFY EXISTING TABLE
+def modify_existing_output_tables(original_unique_topics_df:pd.DataFrame, modifiable_unique_topics_df:pd.DataFrame, reference_df:pd.DataFrame, text_output_file_list_state:List[str]) -> Tuple:
+    '''
+    Take a unique_topics table that has been modified, apply these new topic names to the long-form reference_df, and save both tables to file.
+    '''    
+    
+    reference_file_path = os.path.basename([x for x in text_output_file_list_state if 'reference' in x][0])
+    unique_table_file_path = os.path.basename([x for x in text_output_file_list_state if 'unique' in x][0])
+
+    print("reference_file_path:", reference_file_path)
+
+    output_file_list = []
+
+    if reference_file_path and unique_table_file_path:
+
+        reference_df = join_modified_topic_names_to_ref_table(modifiable_unique_topics_df, original_unique_topics_df, reference_df)
+
+        ## Reference table mapping response numbers to topics
+        reference_table_file_name = reference_file_path.replace(".csv", "_mod") 
+        new_reference_df_file_path = output_folder + reference_table_file_name  + ".csv"
+        reference_df.to_csv(new_reference_df_file_path, index=None)
+        output_file_list.append(new_reference_df_file_path)
+
+        # Drop rows where Response References is NA or null
+        modifiable_unique_topics_df = modifiable_unique_topics_df[~modifiable_unique_topics_df["Response References"].isnull()]
+
+        # Convert 'Response References' to numeric (forcing errors to NaN if conversion fails)
+        modifiable_unique_topics_df["Response References"] = pd.to_numeric(
+            modifiable_unique_topics_df["Response References"], errors='coerce'
+        )
 
-def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, reference_df: pd.DataFrame, merge_sentiment:str="Yes", threshold: float = deduplication_threshold) -> pd.DataFrame:
+        # Drop any rows where conversion failed (original non-numeric values)
+        modifiable_unique_topics_df.dropna(subset=["Response References"], inplace=True)
+
+        # Sort values
+        modifiable_unique_topics_df.sort_values(["Response References"], ascending=False, inplace=True)
+
+        unique_table_file_name = unique_table_file_path.replace(".csv", "_mod")
+        modified_unique_table_file_path = output_folder + unique_table_file_name + ".csv"
+        modifiable_unique_topics_df.to_csv(modified_unique_table_file_path, index=None)
+        output_file_list.append(modified_unique_table_file_path)
+    
+    else:
+        output_file_list = text_output_file_list_state
+        reference_table_file_name = reference_file_path
+        unique_table_file_name = unique_table_file_path
+        raise Exception("Reference and unique topic tables not found.")
+    
+    # Outputs for markdown table output
+    unique_table_df_revised_display = modifiable_unique_topics_df.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
+    deduplicated_unique_table_markdown = unique_table_df_revised_display.to_markdown(index=False)
+    
+
+    return modifiable_unique_topics_df, reference_df, output_file_list, output_file_list, output_file_list, output_file_list, reference_table_file_name, unique_table_file_name, deduplicated_unique_table_markdown
+
+
+# DEDUPLICATION/SUMMARISATION FUNCTIONS
+def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, reference_df: pd.DataFrame, general_topic_series: pd.Series = None, merge_general_topics = "No", merge_sentiment:str="No", threshold: float = 90) -> pd.DataFrame:
     """
     Deduplicates similar category names in a pandas Series based on a fuzzy matching threshold,
     merging smaller topics into larger topics.
@@ -1540,8 +1737,9 @@ def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, r
     # Count occurrences of each category in the reference_df
     category_counts = reference_df['Subtopic'].value_counts().to_dict()
 
-    # Initialize the result dictionary
+    # Initialize dictionaries for both category mapping and scores
     deduplication_map = {}
+    match_scores = {}  # New dictionary to store match scores
 
     # First pass: Handle exact matches
     for category in category_series.unique():
@@ -1556,45 +1754,63 @@ def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, r
             most_common = max(match_counts.items(), key=lambda x: x[1])[0]
             most_common_category = category_series[most_common]
             
-            # Map all exact matches to the most common variant
+            # Map all exact matches to the most common variant and store score
             for match in exact_matches:
                 deduplication_map[category_series[match]] = most_common_category
+                match_scores[category_series[match]] = 100  # Exact matches get score of 100
 
     # Second pass: Handle fuzzy matches for remaining categories
-    for category in category_series.unique():
-        # Skip if the category is already processed
+    # Create a DataFrame to maintain the relationship between categories and general topics
+    categories_df = pd.DataFrame({
+        'category': category_series,
+        'general_topic': general_topic_series
+    }).drop_duplicates()
+
+    for _, row in categories_df.iterrows():
+        category = row['category']
         if category in deduplication_map:
             continue
 
-        # Find close matches to the current category, excluding the current category itself
+        current_general_topic = row['general_topic']
+
+        # Filter potential matches to only those within the same General Topic if relevant
+        if merge_general_topics == "No":
+            potential_matches = categories_df[
+                (categories_df['category'] != category) & 
+                (categories_df['general_topic'] == current_general_topic)
+            ]['category'].tolist()
+        else:
+            potential_matches = categories_df[
+                (categories_df['category'] != category)
+            ]['category'].tolist()
+
         matches = process.extract(category, 
-                                [cat for cat in category_series.unique() if cat != category], 
-                                scorer=fuzz.token_set_ratio, 
+                                potential_matches,
+                                scorer=fuzz.WRatio,
                                 score_cutoff=threshold)
 
-        if matches:  # Check if there are any matches
-            best_match = max(matches, key=lambda x: x[1])  # Get the match with the highest score
-            match, score, _ = best_match  # Unpack the best match
+        if matches:
+            best_match = max(matches, key=lambda x: x[1])
+            match, score, _ = best_match
 
-            # Compare counts to ensure smaller topics merge into larger ones
             if category_counts.get(category, 0) < category_counts.get(match, 0):
-                deduplication_map[category] = match  # Map the smaller category to the larger one
+                deduplication_map[category] = match
+                match_scores[category] = score
             else:
-                deduplication_map[match] = category  # Map the larger category to the smaller one
+                deduplication_map[match] = category
+                match_scores[match] = score
         else:
-            deduplication_map[category] = category  # No match found, keep the category as is
-
-    # Create the result DataFrame
-    if merge_sentiment == "Yes":
-        result_df = pd.DataFrame({
-            'old_category': category_series + " | " + join_series,
-            'deduplicated_category': category_series.map(lambda x: deduplication_map.get(x, x))
-        })
-    else:
-        result_df = pd.DataFrame({
-            'old_category': category_series + " | " + join_series,
-            'deduplicated_category': category_series.map(lambda x: deduplication_map.get(x, x))
-        })
+            deduplication_map[category] = category
+            match_scores[category] = 100
+
+    # Create the result DataFrame with scores
+    result_df = pd.DataFrame({
+        'old_category': category_series + " | " + join_series,
+        'deduplicated_category': category_series.map(lambda x: deduplication_map.get(x, x)),
+        'match_score': category_series.map(lambda x: match_scores.get(x, 100))  # Add scores column
+    })
+
+    #print(result_df)
 
     return result_df
 
@@ -1602,10 +1818,11 @@ def deduplicate_topics(reference_df:pd.DataFrame,
                        unique_topics_df:pd.DataFrame,
                        reference_table_file_name:str,
                        unique_topics_table_file_name:str,
+                       in_excel_sheets:str="",
                        merge_sentiment:str= "No",
                        merge_general_topics:str="No",
-                       score_threshold:int=deduplication_threshold,
-                       in_data_files=[],
+                       score_threshold:int=90,
+                       in_data_files:List[str]=[],
                        chosen_cols:List[str]="",
                        deduplicate_topics:str="Yes"
                        ):
@@ -1614,37 +1831,95 @@ def deduplicate_topics(reference_df:pd.DataFrame,
     '''
     output_files = []
     log_output_files = []
+    file_data = pd.DataFrame()
 
-    reference_table_file_name_no_ext = get_file_name_no_ext(reference_table_file_name)
-    unique_topics_table_file_name_no_ext = get_file_name_no_ext(unique_topics_table_file_name)
+    reference_table_file_name_no_ext = reference_table_file_name #get_file_name_no_ext(reference_table_file_name)
+    unique_topics_table_file_name_no_ext = unique_topics_table_file_name #get_file_name_no_ext(unique_topics_table_file_name)
 
-    if in_data_files and chosen_cols:
-        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1)
+    # For checking that data is not lost during the process
+    initial_unique_references = len(reference_df["Response References"].unique())
 
-    # Run through this x times to try to get all duplicate topics
-    if deduplicate_topics == "Yes":
-        for i in range(0, 5):
-            #print("Deduplication run:", i)
-            
+    if unique_topics_df.empty:
+        unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
 
-            #reference_df_unique[["old_category"]].to_csv(output_folder + "reference_df_unique_old_categories_" + str(i) + ".csv", index=None)
+        # Then merge the topic numbers back to the original dataframe
+        reference_df = reference_df.merge(
+            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
+            on=['General Topic', 'Subtopic', 'Sentiment'],
+            how='left'
+        )     
 
-            if merge_sentiment == "No":
-                # First, combine duplicate topics in reference_df
-                reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                reference_df_unique = reference_df.drop_duplicates("old_category")
+    if in_data_files and chosen_cols:
+        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets)
+    else:
+        out_message = "No file data found, pivot table output will not be created."
+        print(out_message)
+        #raise Exception(out_message)
 
-                # Deduplicate categories within each sentiment group
-                deduplicated_topic_map_df = reference_df_unique.groupby("Sentiment").apply(
-                    lambda group: deduplicate_categories(group["Subtopic"], group["Sentiment"], reference_df, threshold=score_threshold)
-                ).reset_index(drop=True)  # Reset index after groupby
+    # Run through this x times to try to get all duplicate topics
+    if deduplicate_topics == "Yes":
+        for i in range(0, 8):
+            if merge_sentiment == "No":    
+                if merge_general_topics == "No":
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby(["General Topic", "Sentiment"]).apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=group["General Topic"],
+                            merge_general_topics="No",
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
+                else:
+                    # This case should allow cross-topic matching but is still grouping by Sentiment
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby("Sentiment").apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=None,  # Set to None to allow cross-topic matching
+                            merge_general_topics="Yes",
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
             else:
-                # Deduplicate categories by subtopic name only
-                # First, combine duplicate topics in reference_df
-                reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                reference_df_unique = reference_df.drop_duplicates("old_category")
-
-                deduplicated_topic_map_df = deduplicate_categories(reference_df_unique["Subtopic"], reference_df_unique["Sentiment"], reference_df, merge_sentiment=merge_sentiment, threshold=score_threshold).reset_index(drop=True)
+                if merge_general_topics == "No":
+                    # Update this case to maintain general topic boundaries
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby("General Topic").apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=group["General Topic"],
+                            merge_general_topics="No",
+                            merge_sentiment=merge_sentiment, 
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
+                else:
+                    # For complete merging across all categories
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = deduplicate_categories(
+                        reference_df_unique["Subtopic"], 
+                        reference_df_unique["Sentiment"], 
+                        reference_df, 
+                        general_topic_series=None,  # Set to None to allow cross-topic matching
+                        merge_general_topics="Yes",
+                        merge_sentiment=merge_sentiment,
+                        threshold=score_threshold
+                    ).reset_index(drop=True)
            
             if deduplicated_topic_map_df['deduplicated_category'].isnull().all():
             # Check if 'deduplicated_category' contains any values
@@ -1653,8 +1928,9 @@ def deduplicate_topics(reference_df:pd.DataFrame,
             else:
                 # Join deduplicated columns back to original df
                 #deduplicated_topic_map_df.to_csv(output_folder + "deduplicated_topic_map_df_" + str(i) + ".csv", index=None)
+
                 # Remove rows where 'deduplicated_category' is blank or NaN
-                deduplicated_topic_map_df = deduplicated_topic_map_df.loc[(deduplicated_topic_map_df['deduplicated_category'].str.strip() != '') & ~(deduplicated_topic_map_df['deduplicated_category'].isnull()), ['old_category','deduplicated_category']]
+                deduplicated_topic_map_df = deduplicated_topic_map_df.loc[(deduplicated_topic_map_df['deduplicated_category'].str.strip() != '') & ~(deduplicated_topic_map_df['deduplicated_category'].isnull()), ['old_category','deduplicated_category', 'match_score']]
 
                 deduplicated_topic_map_df.to_csv(output_folder + "deduplicated_topic_map_df_" + str(i) + ".csv", index=None)
 
@@ -1675,9 +1951,9 @@ def deduplicate_topics(reference_df:pd.DataFrame,
 
             reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
 
-            reference_df["General Topic"] = reference_df["General Topic"].str.lower().str.capitalize() 
-            reference_df["Subtopic"] = reference_df["Subtopic"].str.lower().str.capitalize() 
-            reference_df["Sentiment"] = reference_df["Sentiment"].str.lower().str.capitalize() 
+            #reference_df["General Topic"] = reference_df["General Topic"].str.lower().str.capitalize() 
+            #reference_df["Subtopic"] = reference_df["Subtopic"].str.lower().str.capitalize() 
+            #reference_df["Sentiment"] = reference_df["Sentiment"].str.lower().str.capitalize() 
 
             if merge_general_topics == "Yes":
                 # Replace General topic names for each Subtopic with that for the Subtopic with the most responses
@@ -1711,15 +1987,42 @@ def deduplicate_topics(reference_df:pd.DataFrame,
 
             reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
 
+        # Update reference summary column with all summaries
+        reference_df["Summary"] = reference_df.groupby(
+        ["Response References", "General Topic", "Subtopic", "Sentiment"]
+        )["Summary"].transform(' <br> '.join)
+
+        # Check that we have not inadvertantly removed some data during the above process
+        end_unique_references = len(reference_df["Response References"].unique())
+
+        if initial_unique_references != end_unique_references:
+            raise Exception(f"Number of unique references changed during processing: Initial={initial_unique_references}, Final={end_unique_references}")
+        
+        # Drop duplicates in the reference table - each comment should only have the same topic referred to once
+        reference_df.drop_duplicates(['Response References', 'General Topic', 'Subtopic', 'Sentiment'], inplace=True)
+
+
         # Remake unique_topics_df based on new reference_df
         unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
 
-        basic_response_data = get_basic_response_data(file_data, chosen_cols)
+        # Then merge the topic numbers back to the original dataframe
+        reference_df = reference_df.merge(
+            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
+            on=['General Topic', 'Subtopic', 'Sentiment'],
+            how='left'
+        ) 
+
+        if not file_data.empty:
+            basic_response_data = get_basic_response_data(file_data, chosen_cols)            
+            reference_df_pivot = convert_reference_table_to_pivot_table(reference_df, basic_response_data)
+
+            reference_pivot_file_path = output_folder + reference_table_file_name_no_ext + "_pivot_dedup.csv"
+            reference_df_pivot.to_csv(reference_pivot_file_path, index=None)
 
-        reference_df_pivot = convert_reference_table_to_pivot_table(reference_df, basic_response_data)
+            log_output_files.append(reference_pivot_file_path)
 
-        reference_table_file_name_no_ext = get_file_name_no_ext(reference_table_file_name)
-        unique_topics_table_file_name_no_ext = get_file_name_no_ext(unique_topics_table_file_name)
+        #reference_table_file_name_no_ext = get_file_name_no_ext(reference_table_file_name)
+        #unique_topics_table_file_name_no_ext = get_file_name_no_ext(unique_topics_table_file_name)
 
         reference_file_path = output_folder + reference_table_file_name_no_ext + "_dedup.csv"
         unique_topics_file_path = output_folder + unique_topics_table_file_name_no_ext + "_dedup.csv"
@@ -1727,14 +2030,14 @@ def deduplicate_topics(reference_df:pd.DataFrame,
         unique_topics_df.to_csv(unique_topics_file_path, index=None)
 
         output_files.append(reference_file_path)
-        output_files.append(unique_topics_file_path)
+        output_files.append(unique_topics_file_path)        
 
-        reference_pivot_file_path = output_folder + reference_table_file_name_no_ext + "_pivot_dedup.csv"
-        reference_df_pivot.to_csv(reference_pivot_file_path, index=None)
+        # Outputs for markdown table output
+        unique_table_df_revised_display = unique_topics_df.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
 
-        log_output_files.append(reference_pivot_file_path)
+        deduplicated_unique_table_markdown = unique_table_df_revised_display.to_markdown(index=False)
 
-    return reference_df, unique_topics_df, output_files, log_output_files
+    return reference_df, unique_topics_df, output_files, log_output_files, deduplicated_unique_table_markdown
 
 def sample_reference_table_summaries(reference_df:pd.DataFrame,
                                      unique_topics_df:pd.DataFrame,
@@ -1750,6 +2053,11 @@ def sample_reference_table_summaries(reference_df:pd.DataFrame,
 
     reference_df_grouped = reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
 
+    if 'Revised summary' in reference_df.columns:
+        out_message = "Summary has already been created for this file"
+        print(out_message)
+        raise Exception(out_message)
+
     for group_keys, reference_df_group in reference_df_grouped:
         #print(f"Group: {group_keys}")
         #print(f"Data: {reference_df_group}")
@@ -1831,6 +2139,7 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
                             latest_summary_completed:int = 0,
                             out_metadata_str:str = "",
                             in_data_files:List[str]=[],
+                            in_excel_sheets:str="",
                             chosen_cols:List[str]=[],
                             log_output_files:list[str]=[],
                             summarise_format_radio:str="Return a summary up to two paragraphs long that includes as much detail as possible from the original text",
@@ -1845,11 +2154,6 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
     local_model = []
     summarised_output_markdown = ""
     
-    print("In summarise_output_topics function.")
-
-    all_summaries = summarised_references["Summary"].tolist()
-
-    length_all_summaries = len(all_summaries)
 
     # Check for data for summarisations
     if not unique_table_df.empty and not reference_table_df.empty:
@@ -1857,17 +2161,24 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
     else:
         out_message = "Please upload a unique topic table and reference table file to continue with summarisation."
         print(out_message)
-        raise(out_message)
-        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
-
+        raise Exception(out_message)
+    
+    if 'Revised summary' in reference_table_df.columns:
+        out_message = "Summary has already been created for this file"
+        print(out_message)
+        raise Exception(out_message)
+    
     # Load in data file and chosen columns if exists to create pivot table later
     if in_data_files and chosen_cols:
-        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1)
+        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets=in_excel_sheets)
     else:
-        out_message = "No file data found, please load a data file on the first tab and select a column."
+        out_message = "No file data found, pivot table output will not be created."
         print(out_message)
-        raise(out_message)
-        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
+        raise Exception(out_message)
+   
+    
+    all_summaries = summarised_references["Summary"].tolist()
+    length_all_summaries = len(all_summaries)
 
     # If all summaries completed, make final outputs
     if latest_summary_completed >= length_all_summaries:
@@ -1908,9 +2219,18 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
         unique_table_df_revised = unique_table_df_revised.loc[unique_table_df_revised["Sentiment"] != "Not Mentioned", :]
         reference_table_df_revised = reference_table_df_revised.loc[reference_table_df_revised["Sentiment"] != "Not Mentioned", :]
         
-        basic_response_data = get_basic_response_data(file_data, chosen_cols)        
+          
+            
 
-        reference_table_df_revised_pivot = convert_reference_table_to_pivot_table(reference_table_df_revised, basic_response_data)
+        if not file_data.empty:
+            basic_response_data = get_basic_response_data(file_data, chosen_cols)
+            reference_table_df_revised_pivot = convert_reference_table_to_pivot_table(reference_table_df_revised, basic_response_data)
+
+            ### Save pivot file to log area
+            reference_table_df_revised_pivot_path = output_folder + batch_file_path_details + "_summarised_reference_table_pivot_" + model_choice_clean + ".csv"
+            reference_table_df_revised_pivot.to_csv(reference_table_df_revised_pivot_path, index=None)
+
+            log_output_files.append(reference_table_df_revised_pivot_path)
 
         # Save to file
         unique_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_unique_topic_table_" + model_choice_clean + ".csv"
@@ -1919,16 +2239,10 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
         reference_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_reference_table_" + model_choice_clean + ".csv"
         reference_table_df_revised.to_csv(reference_table_df_revised_path, index = None)
 
-        output_files.extend([reference_table_df_revised_path, unique_table_df_revised_path])
-
-        ### Save pivot file to log area
-        reference_table_df_revised_pivot_path = output_folder + batch_file_path_details + "_summarised_reference_table_pivot_" + model_choice_clean + ".csv"
-        reference_table_df_revised_pivot.to_csv(reference_table_df_revised_pivot_path, index=None)
-
-        log_output_files.append(reference_table_df_revised_pivot_path)
+        output_files.extend([reference_table_df_revised_path, unique_table_df_revised_path])       
 
         ###
-        unique_table_df_revised_display = unique_table_df_revised.apply(lambda col: col.map(wrap_text))
+        unique_table_df_revised_display = unique_table_df_revised.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
 
         summarised_output_markdown = unique_table_df_revised_display.to_markdown(index=False)
 

tools/prompts.py

@@ -5,7 +5,7 @@ initial_table_prompt = """The open text data is shown in the following table tha
 
 Your task is to create one new markdown table with the headings 'General Topic', 'Subtopic', 'Sentiment', 'Response references', and 'Summary'.
 In the first column identify general topics relevant to responses. Create as many general topics as you can.
-In the second column list subtopics relevant to responses. Make the subtopics as specific as possible and make sure they cover every issue mentioned.
+In the second column list subtopics relevant to responses. Make the subtopics as specific as possible and make sure they cover every issue mentioned. The subtopic should never be blank or empty.
 In the third column write the sentiment of the subtopic: {sentiment_choices}.
 In the fourth column list each specific Response reference number that is relevant to the Subtopic, separated by commas. Do no write any other text in this column.
 In the fifth and final column, write a short summary of the subtopic based on relevant responses. Highlight specific issues that appear in relevant responses.
@@ -29,7 +29,7 @@ Topics known to be relevant to this dataset are shown in the following Topics ta
 
 Your task is to create one new markdown table, assigning responses from the Response table to existing topics, or to create new topics if no existing topics are relevant.  
 Create a new markdown table with the headings 'General Topic', 'Subtopic', 'Sentiment', 'Response references', and 'Summary'.
-In the first and second columns, assign General Topics and Subtopics to Responses. Assign topics from the Topics table above if they are very relevant to the text of the Response. Fill in the General Topic and Sentiment for the Subtopic if they do not already exist. If you find a new topic that does not exist in the Topics table, add a new row to the new table. Make the General Topic and Subtopic as specific as possible.
+In the first and second columns, assign General Topics and Subtopics to Responses. Assign topics from the Topics table above if they are very relevant to the text of the Response. Fill in the General Topic and Sentiment for the Subtopic if they do not already exist. If you find a new topic that does not exist in the Topics table, add a new row to the new table. Make the General Topic and Subtopic as specific as possible. The subtopic should never be blank or empty.
 In the third column, write the sentiment of the Subtopic: {sentiment_choices}.
 In the fourth column list each specific Response reference number that is relevant to the Subtopic, separated by commas. Do no write any other text in this column.
 In the fifth and final column, write a short summary of the Subtopic based on relevant responses. Highlight specific issues that appear in relevant responses.
@@ -50,6 +50,8 @@ Your task is to make a consolidated summary of the above text. {summary_format}.
 Summary:"""
 
 
+## The following didn't work well in testing and so is not currently used
+
 create_general_topics_system_prompt = system_prompt
 
 create_general_topics_prompt = """Subtopics known to be relevant to this dataset are shown in the following Topics table: 
@@ -60,6 +62,10 @@ Your task is to create a General Topic name for each Subtopic. The new Topics ta
 New Topics table:"""
 
 
+
+
+
+
 # example_instruction_prompt_llama3 = """<|start_header_id|>system<|end_header_id|>\n
 # You are an AI assistant that follows instruction extremely well. Help as much as you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n
 # Summarise the following text in less than {length} words: "{text}"\n