Merge pull request #1 from seanpedrick-case/dev

app.py

@@ -139,15 +139,15 @@ with app:
             summarisation_in_previous_data_files_status = gr.Textbox(value = "", label="Previous file input", visible=False)
 
             with gr.Row():
-                merge_sentiment_drop = gr.Dropdown(label="Merge sentiment values together for duplicate subtopics.", value="No", choices=["Yes", "No"])
                 merge_general_topics_drop = gr.Dropdown(label="Merge general topic values together for duplicate subtopics.", value="No", choices=["Yes", "No"])
+                merge_sentiment_drop = gr.Dropdown(label="Merge sentiment values together for duplicate subtopics.", value="No", choices=["Yes", "No"])                
                 deduplicate_score_threshold = gr.Number(label="Similarity threshold with which to determine duplicates.", value = 90, minimum=5, maximum=100, precision=0)
 
             deduplicate_previous_data_btn = gr.Button("Deduplicate topics", variant="primary")
             
             duplicate_output_files = gr.File(height=file_input_height, label="Upload files to summarise", file_count= "multiple", file_types=['.xlsx', '.xls', '.csv', '.parquet', '.csv.gz'])
             
-            summarise_previous_data_btn = gr.Button("Summarise existing topics", variant="primary")
+            summarise_previous_data_btn = gr.Button("Summarise topics", variant="primary")
             summary_output_files = gr.File(height=file_input_height, label="Summarised output files", interactive=False)
             summarised_output_markdown = gr.Markdown(value="### Summarised table will appear here")
 
@@ -246,9 +246,9 @@ with app:
     summarise_previous_data_btn.click(empty_output_vars_summarise, inputs=None, outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox]).\
     then(load_in_previous_data_files, inputs=[duplicate_output_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, summarisation_in_previous_data_files_status, data_file_names_textbox, unique_topics_table_file_textbox]).\
     then(sample_reference_table_summaries, inputs=[master_reference_df_state, master_unique_topics_df_state, random_seed], outputs=[summary_reference_table_sample_state, summarised_references_markdown, master_reference_df_state, master_unique_topics_df_state]).\
-    then(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
+    then(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames, log_files_output_list_state], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
 
-    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
+    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, data_file_names_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_colnames, log_files_output_list_state], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
 
     # If uploaded partially completed consultation files do this. This should then start up the 'latest_batch_completed' change action above to continue extracting topics.
     continue_previous_data_files_btn.click(

requirements.txt

@@ -1,5 +1,5 @@
 pandas==2.2.3
-gradio==5.12.0
+gradio==5.18.0
 spaces==0.31.0
 boto3==1.35.71
 pyarrow==18.1.0
@@ -13,6 +13,6 @@ beautifulsoup4==4.12.3
 rapidfuzz==3.10.1
 torch==2.5.1 --extra-index-url https://download.pytorch.org/whl/cpu
 llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-transformers==4.47.0
+transformers==4.49.0
 numpy==1.26.4
 typing_extensions==4.12.2

requirements_aws.txt

@@ -1,5 +1,5 @@
 pandas==2.2.3
-gradio==5.12.0
+gradio==5.18.0
 spaces==0.31.0
 boto3==1.35.71
 pyarrow==18.1.0
@@ -12,6 +12,6 @@ html5lib==1.1
 beautifulsoup4==4.12.3
 rapidfuzz==3.10.1
 llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-transformers==4.47.0
+transformers==4.49.0
 numpy==1.26.4
 typing_extensions==4.12.2

requirements_gpu.txt

@@ -1,5 +1,5 @@
 pandas==2.2.3
-gradio==5.12.0
+gradio==5.18.0
 spaces==0.31.0
 boto3==1.35.71
 pyarrow==18.1.0
@@ -15,6 +15,6 @@ torch==2.4.1 --extra-index-url https://download.pytorch.org/whl/cu121
 #llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
 # Specify exact llama_cpp wheel for huggingface compatibility
 https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.90-cu121/llama_cpp_python-0.2.90-cp310-cp310-linux_x86_64.whl
-transformers==4.47.0
+transformers==4.49.0
 numpy==1.26.4
 typing_extensions==4.12.2

tools/llm_api_call.py

@@ -196,11 +196,11 @@ def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_co
 
     # Remove problematic characters including ASCII and various quote marks
         # Remove problematic characters including control characters, special characters, and excessive leading/trailing whitespace
-    batch_basic_response_data["Response"] = batch_basic_response_data["Response"].str.replace(r'[\x00-\x1F\x7F]|[""<>]|\\', '', regex=True)  # Remove control and special characters
-    batch_basic_response_data["Response"] = batch_basic_response_data["Response"].str.strip()  # Remove leading and trailing whitespace
-    batch_basic_response_data["Response"] = batch_basic_response_data["Response"].str.replace(r'\s+', ' ', regex=True)  # Replace multiple spaces with a single space
-    batch_basic_response_data["Response"] = batch_basic_response_data["Response"].str.replace(r'\n{2,}', '\n', regex=True)  # Replace multiple line breaks with a single line break
-    batch_basic_response_data["Response"] = batch_basic_response_data["Response"].str.slice(0, max_comment_character_length) # Maximum 1,500 character responses
+    batch_basic_response_data.loc[:, "Response"]= batch_basic_response_data["Response"].str.replace(r'[\x00-\x1F\x7F]|[""<>]|\\', '', regex=True)  # Remove control and special characters
+    batch_basic_response_data.loc[:, "Response"] = batch_basic_response_data["Response"].str.strip()  # Remove leading and trailing whitespace
+    batch_basic_response_data.loc[:, "Response"] = batch_basic_response_data["Response"].str.replace(r'\s+', ' ', regex=True)  # Replace multiple spaces with a single space
+    batch_basic_response_data.loc[:, "Response"] = batch_basic_response_data["Response"].str.replace(r'\n{2,}', '\n', regex=True)  # Replace multiple line breaks with a single line break
+    batch_basic_response_data.loc[:, "Response"] = batch_basic_response_data["Response"].str.slice(0, max_comment_character_length) # Maximum 1,500 character responses
 
     # Remove blank and extremely short responses
     batch_basic_response_data = batch_basic_response_data.loc[~(batch_basic_response_data["Response"].isnull()) &\
@@ -855,6 +855,12 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
     # Remove duplicate Response references for the same topic
     out_reference_df.drop_duplicates(["Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
 
+    # Try converting response references column to int, keep as string if fails
+    try:
+        out_reference_df["Response References"] = out_reference_df["Response References"].astype(int)
+    except Exception as e:
+        print("Could not convert Response References column to integer due to", e)
+
     out_reference_df.sort_values(["Start row of group", "Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True) 
 
     # Save the new DataFrame to CSV
@@ -1817,7 +1823,8 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
                             out_metadata_str:str = "",
                             in_data_files:List[str]=[],
                             chosen_cols:List[str]=[],
-                            output_files:list = [],
+                            log_output_files:list[str]=[],
+                            output_files:list[str] = [],                            
                             summarise_topic_descriptions_prompt:str=summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt:str=summarise_topic_descriptions_system_prompt,
                             do_summaries="Yes",
                             progress=gr.Progress(track_tqdm=True)):
@@ -1826,7 +1833,6 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
     '''
     out_metadata = []
     local_model = []
-    log_output_files = []
     summarised_output_markdown = ""
     
     print("In summarise_output_topics function.")
@@ -1835,12 +1841,23 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
 
     length_all_summaries = len(all_summaries)
 
+    # Check for data for summarisations
+    if not unique_table_df.empty and not reference_table_df.empty:
+        print("Unique table and reference table data found.")
+    else:
+        out_message = "Please upload a unique topic table and reference table file to continue with summarisation."
+        print(out_message)
+        raise(out_message)
+        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
+
     # Load in data file and chosen columns if exists to create pivot table later
     if in_data_files and chosen_cols:
         file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1)
-
-    #print("latest_summary_completed:", latest_summary_completed)
-    #print("length_all_summaries:", length_all_summaries)
+    else:
+        out_message = "No file data found, please load a data file on the first tab and select a column."
+        print(out_message)
+        raise(out_message)
+        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
 
     # If all summaries completed, make final outputs
     if latest_summary_completed >= length_all_summaries:
@@ -1866,6 +1883,7 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
         summarised_references_j = summarised_references[join_plus_summary_cols].drop_duplicates(join_plus_summary_cols)
 
         unique_table_df_revised = unique_table_df.merge(summarised_references_j, on = join_cols, how = "left")
+
         # If no new summary is available, keep the original
         unique_table_df_revised["Revised summary"] = unique_table_df_revised["Revised summary"].combine_first(unique_table_df_revised["Summary"])
 
@@ -1904,6 +1922,10 @@ def summarise_output_topics(summarised_references:pd.DataFrame,
 
         summarised_output_markdown = unique_table_df_revised_display.to_markdown(index=False)
 
+        # Ensure same file name not returned twice
+        output_files = list(set(output_files))
+        log_output_files = list(set(log_output_files))
+
         return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
 
     tic = time.perf_counter()