start analysis button, bug fixes, authentication

.gitignore

@@ -4,4 +4,7 @@
 *.xlsx
 /testing/
 /modules/__pycache__/
-/logs/
+/logs/
+app.log
+logs
+logs/

app.py

@@ -34,7 +34,7 @@ load_dotenv()
 def main():
     # Temporarily set authentication to True for testing
     if 'authenticated' not in st.session_state:
-        st.session_state['authenticated'] = True
+        st.session_state['authenticated'] = False
 
     if st.session_state['authenticated']:
         # Remove login success message for testing
@@ -88,7 +88,8 @@ def main():
                                     applications being filtered out. At the same time, this also \
                                     increases the probability of false negatives (FNs). The rate of \
                                     FNs at the lowest setting is approximately 6 percent, and \
-                                    approaches 13 percent at the highest setting. ', 
+                                    approaches 13 percent at the highest setting. \
+                                    NOTE: changing this setting does not affect the raw data in the CSV output file (only the REVIEW/REJECT labels)', 
                                     options = list(sens_options.keys()), 
                                     horizontal = False)
 
@@ -108,54 +109,78 @@ def main():
 
         uploaded_file = st.file_uploader("Select a file containing MAF application pre-filtering data (see instructions in the sidebar)")
 
-        if uploaded_file is not None:
+        # Add session state variables if they don't exist
+        if 'show_button' not in st.session_state:
+            st.session_state['show_button'] = True
+        if 'processing' not in st.session_state:
+            st.session_state['processing'] = False
+        if 'data_processed' not in st.session_state:
+            st.session_state['data_processed'] = False
+
+        # Only show the button if show_button is True and file is uploaded and not processing
+        if uploaded_file is not None and st.session_state['show_button'] and not st.session_state['processing']:
+            if st.button("Start Analysis", key="start_analysis"):
+                st.session_state['show_button'] = False
+                st.session_state['processing'] = True
+                st.rerun()
+
+        # If we're processing, show the processing logic
+        if st.session_state['processing']:
             try:
                 logger.info(f"File uploaded: {uploaded_file.name}")
                 
                 if not st.session_state['data_processed']:
-                    logger.info("Starting data processing")
+                    logger.info("Starting data processing...")
                     try:
                         st.session_state['df'] = process_data(uploaded_file, sens_level)
                         logger.info("Data processing completed successfully")
                         st.session_state['data_processed'] = True
                     except Exception as e:
                         logger.error(f"Error in process_data: {str(e)}")
-                        raise  # Re-raise the exception to be caught by outer try-except
-    
+                        raise
+
                 df = st.session_state['df']
+                logger.info(f"DataFrame columns: {list(df.columns)}")
+                logger.info(f"DataFrame shape: {df.shape}")
 
-                # Get the current date
                 current_datetime = datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
                 output_filename = f'processed_applications_{current_datetime}.csv'
-    
-                # Convert DataFrame to CSV string buffer instead of file
                 csv_buffer = df.to_csv(index=False).encode()
                 logger.info("CSV buffer created successfully")
                 
+                def reset_button_state():
+                    st.session_state['show_button'] = True
+                    st.session_state['processing'] = False
+                    st.session_state['data_processed'] = False
+                
                 st.download_button(
                     label="Download data as CSV",
                     data=csv_buffer,
                     file_name=output_filename,
                     mime='text/csv',
+                    on_click=reset_button_state
                 )
 
             except Exception as e:
                 logger.error(f"Error processing file: {str(e)}")
                 st.error("Failed to process the file. Please ensure your column names match the template file.")
+                st.session_state['show_button'] = True
+                st.session_state['processing'] = False
+                st.rerun()
 
 
     # Comment out or remove the else block containing login form
-    # else:
-    #     username = st.text_input("Username")
-    #     password = st.text_input("Password", type="password")
-    #     if st.button("Login"):
-    #         if validate_login(username, password):
-    #             st.session_state['authenticated'] = True
-    #             st.experimental_rerun()
-    #         else:
-    #             st.error("Incorrect username or password")
-
-
-# Run the main function
+    else:
+        username = st.text_input("Username")
+        password = st.text_input("Password", type="password")
+        if st.button("Login"):
+            if validate_login(username, password):
+                st.session_state['authenticated'] = True
+                st.rerun()
+            else:
+                st.error("Incorrect username or password")
+
+
+
 main()
 

logs/app.log

@@ -1,103 +0,0 @@
-2025-02-04 20:16:29,467 - datasets - INFO - PyTorch version 2.5.1 available.
-2025-02-04 20:16:31,199 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:16:37,895 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:16:38,062 - modules.utils - INFO - T1 df import
-2025-02-04 20:16:38,062 - modules.utils - INFO - T2 columns renamed
-2025-02-04 20:16:38,249 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:16:38,249 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:16:44,645 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:16:44,645 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:16:49,350 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:16:49,350 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:16:55,639 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:16:55,639 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
-2025-02-04 20:17:00,538 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:17:00,539 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
-2025-02-04 20:21:02,708 - datasets - INFO - PyTorch version 2.5.1 available.
-2025-02-04 20:21:05,106 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:21:10,682 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:21:10,849 - modules.utils - INFO - T1 df import
-2025-02-04 20:21:10,850 - modules.utils - INFO - T2 columns renamed
-2025-02-04 20:21:10,994 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:21:10,994 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:21:17,001 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:21:17,001 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:21:21,035 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:21:21,035 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:21:27,247 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:21:27,247 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
-2025-02-04 20:21:31,121 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:21:31,121 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
-2025-02-04 20:23:19,122 - datasets - INFO - PyTorch version 2.5.1 available.
-2025-02-04 20:23:21,471 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:23:26,565 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:23:26,732 - modules.utils - INFO - T1 df import
-2025-02-04 20:23:26,732 - modules.utils - INFO - T2 columns renamed
-2025-02-04 20:23:26,875 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:23:26,875 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:23:30,614 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:23:30,614 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:23:33,378 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:23:33,378 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:23:37,329 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:23:37,329 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
-2025-02-04 20:23:41,129 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:23:41,130 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
-2025-02-04 20:26:27,528 - datasets - INFO - PyTorch version 2.5.1 available.
-2025-02-04 20:26:29,749 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:26:35,354 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:26:35,512 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
-2025-02-04 20:26:35,513 - __main__ - INFO - Starting data processing...
-2025-02-04 20:26:35,519 - modules.utils - INFO - T1 df import
-2025-02-04 20:26:35,520 - modules.utils - INFO - T2 columns renamed
-2025-02-04 20:26:35,658 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:26:35,658 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:26:39,590 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:26:39,591 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:26:43,283 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:26:43,283 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:26:47,454 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:26:47,455 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
-2025-02-04 20:26:50,569 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:26:50,569 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
-2025-02-04 20:27:00,137 - __main__ - ERROR - Error in process_data: Cannot set a DataFrame with multiple columns to the single column pred_score
-2025-02-04 20:27:00,138 - __main__ - ERROR - Error processing file: Cannot set a DataFrame with multiple columns to the single column pred_score
-2025-02-04 20:31:19,620 - datasets - INFO - PyTorch version 2.5.1 available.
-2025-02-04 20:31:21,779 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:31:26,273 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:31:26,432 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
-2025-02-04 20:31:26,433 - __main__ - INFO - Starting data processing...
-2025-02-04 20:31:26,439 - modules.utils - INFO - T1 df import
-2025-02-04 20:31:26,440 - modules.utils - INFO - T2 columns renamed
-2025-02-04 20:31:26,563 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:31:26,563 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:31:30,934 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:31:30,935 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:31:33,839 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:31:33,839 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:31:37,942 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:31:37,942 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
-2025-02-04 20:31:41,885 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:31:41,886 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
-2025-02-04 20:31:51,715 - __main__ - INFO - Data processing completed successfully
-2025-02-04 20:31:51,715 - __main__ - INFO - DataFrame columns: ['id', 'scope_txt', 'tech_txt', 'fin_txt', 'maf_funding', 'cont_public', 'cont_private', 'cont_other', 'scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2', 'ADAPMIT', 'SECTOR1', 'SECTOR2', 'LANG', 'lev_total', 'lev_gt_0', 'lev_maf_%', 'lev_maf_scale', 'pred_score', 'pred_action']
-2025-02-04 20:31:51,715 - __main__ - INFO - DataFrame shape: (14, 23)
-2025-02-04 20:31:51,718 - __main__ - INFO - CSV buffer created successfully
-2025-02-04 20:31:56,984 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:31:57,147 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
-2025-02-04 20:31:57,148 - __main__ - INFO - DataFrame columns: ['id', 'scope_txt', 'tech_txt', 'fin_txt', 'maf_funding', 'cont_public', 'cont_private', 'cont_other', 'scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2', 'ADAPMIT', 'SECTOR1', 'SECTOR2', 'LANG', 'lev_total', 'lev_gt_0', 'lev_maf_%', 'lev_maf_scale', 'pred_score', 'pred_action']
-2025-02-04 20:31:57,148 - __main__ - INFO - DataFrame shape: (14, 23)
-2025-02-04 20:31:57,150 - __main__ - INFO - CSV buffer created successfully
-2025-02-04 20:33:58,763 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:34:06,910 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
-2025-02-04 20:34:07,067 - __main__ - INFO - File uploaded: MAF_2024_FULL.xlsx
-2025-02-04 20:34:07,068 - __main__ - INFO - Starting data processing
-2025-02-04 20:34:07,121 - modules.utils - INFO - data import successful
-2025-02-04 20:34:07,673 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:34:07,675 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
-2025-02-04 20:34:59,028 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:34:59,028 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
-2025-02-04 20:35:25,589 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:35:25,589 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
-2025-02-04 20:36:08,932 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
-2025-02-04 20:36:08,933 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3

modules/utils.py

@@ -176,9 +176,14 @@ def process_data(uploaded_file, sens_level):
         steps_remaining = total_steps - step_count
         if step_count > 1:
             estimated_time_remaining = (elapsed_time / step_count) * steps_remaining
-            estimated_time_remaining_text.write(f'Estimated Time Remaining: {estimated_time_remaining:.0f} seconds (step {step_count+1} of 9)')
+            estimated_time_remaining_text.markdown(
+                f"Elapsed time: {elapsed_time:.1f}s. "
+                f"Estimated time remaining: {estimated_time_remaining:.1f}s"
+                f" (step {step_count+1} of {len(model_names)})"
+            )
+            # estimated_time_remaining_text.write(f'Estimated Time Remaining: {estimated_time_remaining:.0f} seconds (step {step_count+1} of 9)')
         else:
-            estimated_time_remaining_text.write(f'Calculating time remaining... (step {step_count+1} of 9)')
+            estimated_time_remaining_text.write(f'Calculating time remaining... (step {step_count+1} of {len(model_names)})')
 
         model_processing_text.empty()