pred_score fix, logging

app.py

@@ -12,16 +12,22 @@ except Exception as e:
     print(f"Error checking CUDA availability: {str(e)}")
     print("Continuing with CPU...")
 
+
+from modules.logging_config import setup_logging
+setup_logging()
+import logging
+logger = logging.getLogger(__name__)
+
 import streamlit as st
 import os
 from huggingface_hub import login
 from datetime import datetime
-from modules.auth import validate_login, check_password
+from modules.auth import validate_login
 from modules.utils import create_excel, clean_text, extract_predicted_labels, predict_category, process_data
 
 # Local
-# from dotenv import load_dotenv
-# load_dotenv()
+from dotenv import load_dotenv
+load_dotenv()
 
 
 # Main app logic
@@ -104,26 +110,37 @@ def main():
 
         if uploaded_file is not None:
             try:
+                logger.info(f"File uploaded: {uploaded_file.name}")
+                
                 if not st.session_state['data_processed']:
-                    st.session_state['df'] = process_data(uploaded_file, sens_level)
-                    st.session_state['data_processed'] = True
+                    logger.info("Starting data processing")
+                    try:
+                        st.session_state['df'] = process_data(uploaded_file, sens_level)
+                        logger.info("Data processing completed successfully")
+                        st.session_state['data_processed'] = True
+                    except Exception as e:
+                        logger.error(f"Error in process_data: {str(e)}")
+                        raise  # Re-raise the exception to be caught by outer try-except
     
                 df = st.session_state['df']
-    
+
                 # Get the current date
                 current_datetime = datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
-                output_filename = 'processed_applications_'+current_datetime+'.csv'
+                output_filename = f'processed_applications_{current_datetime}.csv'
     
-                output_file = 'processed_applications.csv'
-                df.to_csv(output_file, index=False)
+                # Convert DataFrame to CSV string buffer instead of file
+                csv_buffer = df.to_csv(index=False).encode()
+                logger.info("CSV buffer created successfully")
+                
                 st.download_button(
                     label="Download data as CSV",
-                    data=open(output_file, 'rb'),
+                    data=csv_buffer,
                     file_name=output_filename,
                     mime='text/csv',
                 )
 
-            except:
+            except Exception as e:
+                logger.error(f"Error processing file: {str(e)}")
                 st.error("Failed to process the file. Please ensure your column names match the template file.")
 
 

logs/app.log

@@ -0,0 +1,99 @@
+2025-02-04 20:16:29,467 - datasets - INFO - PyTorch version 2.5.1 available.
+2025-02-04 20:16:31,199 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:16:37,895 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:16:38,062 - modules.utils - INFO - T1 df import
+2025-02-04 20:16:38,062 - modules.utils - INFO - T2 columns renamed
+2025-02-04 20:16:38,249 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:16:38,249 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:16:44,645 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:16:44,645 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
+2025-02-04 20:16:49,350 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:16:49,350 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
+2025-02-04 20:16:55,639 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:16:55,639 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
+2025-02-04 20:17:00,538 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:17:00,539 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
+2025-02-04 20:21:02,708 - datasets - INFO - PyTorch version 2.5.1 available.
+2025-02-04 20:21:05,106 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:21:10,682 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:21:10,849 - modules.utils - INFO - T1 df import
+2025-02-04 20:21:10,850 - modules.utils - INFO - T2 columns renamed
+2025-02-04 20:21:10,994 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:21:10,994 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:21:17,001 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:21:17,001 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
+2025-02-04 20:21:21,035 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:21:21,035 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
+2025-02-04 20:21:27,247 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:21:27,247 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
+2025-02-04 20:21:31,121 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:21:31,121 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
+2025-02-04 20:23:19,122 - datasets - INFO - PyTorch version 2.5.1 available.
+2025-02-04 20:23:21,471 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:23:26,565 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:23:26,732 - modules.utils - INFO - T1 df import
+2025-02-04 20:23:26,732 - modules.utils - INFO - T2 columns renamed
+2025-02-04 20:23:26,875 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:23:26,875 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:23:30,614 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:23:30,614 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
+2025-02-04 20:23:33,378 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:23:33,378 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
+2025-02-04 20:23:37,329 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:23:37,329 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
+2025-02-04 20:23:41,129 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:23:41,130 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
+2025-02-04 20:26:27,528 - datasets - INFO - PyTorch version 2.5.1 available.
+2025-02-04 20:26:29,749 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:26:35,354 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:26:35,512 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
+2025-02-04 20:26:35,513 - __main__ - INFO - Starting data processing...
+2025-02-04 20:26:35,519 - modules.utils - INFO - T1 df import
+2025-02-04 20:26:35,520 - modules.utils - INFO - T2 columns renamed
+2025-02-04 20:26:35,658 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:26:35,658 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:26:39,590 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:26:39,591 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
+2025-02-04 20:26:43,283 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:26:43,283 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
+2025-02-04 20:26:47,454 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:26:47,455 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
+2025-02-04 20:26:50,569 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:26:50,569 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
+2025-02-04 20:27:00,137 - __main__ - ERROR - Error in process_data: Cannot set a DataFrame with multiple columns to the single column pred_score
+2025-02-04 20:27:00,138 - __main__ - ERROR - Error processing file: Cannot set a DataFrame with multiple columns to the single column pred_score
+2025-02-04 20:31:19,620 - datasets - INFO - PyTorch version 2.5.1 available.
+2025-02-04 20:31:21,779 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:31:26,273 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:31:26,432 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
+2025-02-04 20:31:26,433 - __main__ - INFO - Starting data processing...
+2025-02-04 20:31:26,439 - modules.utils - INFO - T1 df import
+2025-02-04 20:31:26,440 - modules.utils - INFO - T2 columns renamed
+2025-02-04 20:31:26,563 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:31:26,563 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:31:30,934 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:31:30,935 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2
+2025-02-04 20:31:33,839 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:31:33,839 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab1
+2025-02-04 20:31:37,942 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:31:37,942 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_tech_lab3
+2025-02-04 20:31:41,885 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:31:41,886 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_fin_lab2
+2025-02-04 20:31:51,715 - __main__ - INFO - Data processing completed successfully
+2025-02-04 20:31:51,715 - __main__ - INFO - DataFrame columns: ['id', 'scope_txt', 'tech_txt', 'fin_txt', 'maf_funding', 'cont_public', 'cont_private', 'cont_other', 'scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2', 'ADAPMIT', 'SECTOR1', 'SECTOR2', 'LANG', 'lev_total', 'lev_gt_0', 'lev_maf_%', 'lev_maf_scale', 'pred_score', 'pred_action']
+2025-02-04 20:31:51,715 - __main__ - INFO - DataFrame shape: (14, 23)
+2025-02-04 20:31:51,718 - __main__ - INFO - CSV buffer created successfully
+2025-02-04 20:31:56,984 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:31:57,147 - __main__ - INFO - File uploaded: MAF_2024_TEST2.xlsx
+2025-02-04 20:31:57,148 - __main__ - INFO - DataFrame columns: ['id', 'scope_txt', 'tech_txt', 'fin_txt', 'maf_funding', 'cont_public', 'cont_private', 'cont_other', 'scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2', 'ADAPMIT', 'SECTOR1', 'SECTOR2', 'LANG', 'lev_total', 'lev_gt_0', 'lev_maf_%', 'lev_maf_scale', 'pred_score', 'pred_action']
+2025-02-04 20:31:57,148 - __main__ - INFO - DataFrame shape: (14, 23)
+2025-02-04 20:31:57,150 - __main__ - INFO - CSV buffer created successfully
+2025-02-04 20:33:58,763 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:34:06,910 - huggingface_hub._login - WARNING - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
+2025-02-04 20:34:07,067 - __main__ - INFO - File uploaded: MAF_2024_FULL.xlsx
+2025-02-04 20:34:07,068 - __main__ - INFO - Starting data processing
+2025-02-04 20:34:07,121 - modules.utils - INFO - data import successful
+2025-02-04 20:34:07,673 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:34:07,675 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab1
+2025-02-04 20:34:59,028 - sentence_transformers.SentenceTransformer - INFO - Use pytorch device_name: mps
+2025-02-04 20:34:59,028 - sentence_transformers.SentenceTransformer - INFO - Load pretrained SentenceTransformer: mtyrrell/classifier_SF_scope_lab2

modules/logging_config.py

@@ -0,0 +1,23 @@
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+
+def setup_logging():
+    # Set up logging
+    log_dir = 'logs'
+    os.makedirs(log_dir, exist_ok=True)
+    log_file = os.path.join(log_dir, 'app.log')
+
+    # Create a RotatingFileHandler
+    file_handler = RotatingFileHandler(log_file, maxBytes=1024 * 1024, backupCount=5)
+    file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+    # Configure the root logger
+    logging.basicConfig(level=logging.INFO,
+                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                        handlers=[file_handler, logging.StreamHandler()])
+
+if __name__ == "__main__":
+    setup_logging()
+    logger = logging.getLogger(__name__)
+    logger.info("Logging setup completed")

modules/utils.py

@@ -9,7 +9,9 @@ from transformers import pipeline
 from openpyxl import Workbook
 from openpyxl.styles import Font, NamedStyle, PatternFill
 from openpyxl.styles.differential import DifferentialStyle
+import logging
 
+logger = logging.getLogger(__name__)
 
 # Function for creating Upload template file
 def create_excel():
@@ -111,20 +113,18 @@ def predict_category(df, model_name, progress_bar, repo, profile, multilabel=Fal
 # Main function to process data
 def process_data(uploaded_file, sens_level):
     df = pd.read_excel(uploaded_file)
-    # Column renaming and initial processing
-    
+    logger.info(f"data import successful")
+    # Rename columns
     df.rename(columns={
         'id': 'id',
         'scope': 'scope_txt',
         'technology': 'tech_txt',
         'financial': 'fin_txt',
-        'barrier': 'bar_txt',
         'maf_funding_requested':'maf_funding',
         'contributions_public_sector':'cont_public',
         'contributions_private_sector':'cont_private',
         'contributions_other':'cont_other'}, inplace=True)
-
-
+    # clean the text fields
     df = df.filter(['id', 'scope_txt', 'tech_txt', 'fin_txt','maf_funding','cont_public','cont_private','cont_other'])
     df.fillna('', inplace=True)
     df[['scope_txt', 'tech_txt', 'fin_txt']] = df[['scope_txt', 'tech_txt', 'fin_txt']].applymap(clean_text)
@@ -209,7 +209,7 @@ def process_data(uploaded_file, sens_level):
 
     # Further data processing and actions
     sector_classes = ['Energy','Transport','Industries']
-    df['pred_score'] = df.apply(lambda x: round((x['fin_lab2']*2 + x['scope_lab1']*2 + x['scope_lab2']*2 + x['tech_lab1'] + x['tech_lab3']+ x['lev_gt_0']+df['lev_maf_scale'])/10*10,0), axis=1)
+    df['pred_score'] = df.apply(lambda x: round((x['fin_lab2']*2 + x['scope_lab1']*2 + x['scope_lab2']*2 + x['tech_lab1'] + x['tech_lab3']+ x['lev_gt_0']+x['lev_maf_scale'])/10*10,0), axis=1)
     df['pred_action'] = df.apply(lambda x: 'REJECT' if (x['pred_score'] <4 or x['LANG'] != 'en-US' or x['ADAPMIT'] == 'Adaptation' or not ((x['SECTOR1'] in sector_classes) or (x['SECTOR2'] in sector_classes))) else 'REVIEW', axis=1)
 
     return df