adapted logic for ADAPTMIT (consensus on tech + scope for ineligiibltiy)

app.py

@@ -26,15 +26,15 @@ from io import BytesIO
 logger = logging.getLogger(__name__)
 
 # Local
-# from dotenv import load_dotenv
-# load_dotenv()
+from dotenv import load_dotenv
+load_dotenv()
 
 
 # Main app logic
 def main():
     # Temporarily set authentication to True for testing
     if 'authenticated' not in st.session_state:
-        st.session_state['authenticated'] = False
+        st.session_state['authenticated'] = True
 
     if st.session_state['authenticated']:
         # Remove login success message for testing
@@ -183,15 +183,15 @@ def main():
 
 
     # Comment out for testing
-    else:
-        username = st.text_input("Username")
-        password = st.text_input("Password", type="password")
-        if st.button("Login"):
-            if validate_login(username, password):
-                st.session_state['authenticated'] = True
-                st.rerun()
-            else:
-                st.error("Incorrect username or password")
+    # else:
+    #     username = st.text_input("Username")
+    #     password = st.text_input("Password", type="password")
+    #     if st.button("Login"):
+    #         if validate_login(username, password):
+    #             st.session_state['authenticated'] = True
+    #             st.rerun()
+    #         else:
+    #             st.error("Incorrect username or password")
 
 
 

modules/utils.py

@@ -85,29 +85,36 @@ def extract_predicted_labels(output, ordinal_selection=1, threshold=0.5):
 def predict_category(df, model_name, progress_bar, repo, profile, multilabel=False):
     device = torch.device("cuda") if torch.cuda.is_available() else (torch.device("mps") if torch.backends.mps.is_built() else torch.device("cpu"))
     model_names_sf = ['scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2']
+
+    # Model configuration mapping
+    model_config = {
+        'ADAPMIT_TECH': {'col_name': 'tech_txt', 'top_k': 1},
+        'ADAPMIT_SCOPE': {'col_name': 'scope_txt', 'top_k': 1},
+        'LANG': {'col_name': 'scope_txt', 'top_k': 1},
+        'default': {'col_name': 'scope_txt', 'top_k': None}
+    }
+
     if model_name in model_names_sf:
         col_name = re.sub(r'_(.*)', r'_txt', model_name)
         model = SetFitModel.from_pretrained(profile+"/"+repo)
         model.to(device)
         # Get tokenizer from the model
         tokenizer = model.model_body.tokenizer
-    elif model_name == 'ADAPMIT_TECH_TEST':
-        col_name = 'tech_txt'
-        model = pipeline("text-classification", 
-                model=profile+"/"+repo, 
-                device=device,
-                return_all_scores=multilabel,
-                truncation=True,
-                max_length=512)
     else:
-        col_name = 'scope_txt'
-        model = pipeline("text-classification", 
-                        model=profile+"/"+repo, 
-                        device=device, 
-                        return_all_scores=multilabel,
+        # Get configuration for the model, falling back to default if not specified
+        config = model_config.get(model_name, model_config['default'])
+        col_name = config['col_name']
+        model = pipeline("text-classification",
+                        model=profile+"/"+repo,
+                        device=device,
+                        top_k=config['top_k'],
                         truncation=True,
                         max_length=512)
+        
+
+    
     predictions = []
+    # probabilities = []
     total = len(df)
     for i, text in enumerate(df[col_name]):
         try:
@@ -119,18 +126,19 @@ def predict_category(df, model_name, progress_bar, repo, profile, multilabel=Fal
                     predictions.append(0 if prediction == 'NEGATIVE' else 1)
             else:
                 prediction = model(text)
-                if model_name == 'ADAPMIT' or model_name == 'ADAPMIT_TECH_TEST':
-                    predictions.append(re.sub('Label$', '', prediction[0]['label']))
+                if model_name == 'ADAPMIT_SCOPE' or model_name == 'ADAPMIT_TECH':
+                    predictions.append(re.sub('Label$', '', prediction[0][0]['label']))
                 elif model_name == 'SECTOR':
                     predictions.append(extract_predicted_labels(prediction[0], threshold=0.5))
                 elif model_name == 'LANG':
-                    predictions.append(prediction[0]['label'])
+                    predictions.append(prediction[0][0]['label'])
         except Exception as e:
             logger.error(f"Error processing sample {df['id'][i]}: {str(e)}")
             st.error("Application Error. Please contact support.")
         # Update progress bar with each iteration
         progress = (i + 1) / total
-        progress_bar.progress(progress)
+        progress_bar.progress(progress) 
+
     return predictions
 
 
@@ -177,7 +185,9 @@ def process_data(uploaded_file, sens_level):
 
     # Define models and predictions
     model_names_sf = ['scope_lab1', 'scope_lab2', 'tech_lab1', 'tech_lab3', 'fin_lab2']
-    model_names = model_names_sf + ['ADAPMIT', 'SECTOR', 'LANG','ADAPMIT_TECH_TEST']
+    model_names = model_names_sf + ['ADAPMIT_SCOPE','ADAPMIT_TECH','SECTOR','LANG']
+    # model_names_sf = []
+    # model_names = ['ADAPMIT_SCOPE','ADAPMIT_TECH']
     total_predictions = len(model_names) * len(df)
     progress_count = 0
 
@@ -203,9 +213,10 @@ def process_data(uploaded_file, sens_level):
         # Load the model and run inference
         if model_name in model_names_sf:
             df[model_name] = predict_category(df, model_name, progress_bar, repo='classifier_SF_' + model_name, profile='mtyrrell')
-        elif model_name == 'ADAPMIT':
+        elif model_name == 'ADAPMIT_SCOPE':
             df[model_name] = predict_category(df, model_name, progress_bar, repo='ADAPMIT-multilabel-bge_f', profile='GIZ')
-            
+        elif model_name == 'ADAPMIT_TECH':
+            df[model_name]= predict_category(df, model_name, progress_bar, repo='ADAPMIT-multilabel-bge_f', profile='GIZ')
         elif model_name == 'SECTOR':
             sectors_dict = predict_category(df, model_name, progress_bar, repo='SECTOR-multilabel-bge_f', profile='GIZ', multilabel=True)
             df['SECTOR1'] = [item['SECTOR1'] for item in sectors_dict]
@@ -213,8 +224,7 @@ def process_data(uploaded_file, sens_level):
         elif model_name == 'LANG':
             df[model_name] = predict_category(df, model_name, progress_bar, repo='51-languages-classifier', profile='qanastek')
             # df[model_name] = predict_category(df, model_name, progress_bar, repo='xlm-roberta-base-language-detection', profile='papluca')
-        elif model_name == 'ADAPMIT_TECH_TEST':
-            df[model_name] = predict_category(df, model_name, progress_bar, repo='ADAPMIT-multilabel-bge_f', profile='GIZ')
+
         
         logger.info(f"Completed: {model_name}")
         model_progress.empty()
@@ -243,6 +253,21 @@ def process_data(uploaded_file, sens_level):
 
     st.write(f'Processing complete. Total time: {elapsed_time:.1f} seconds')
 
+
+    # df['ADAPMIT_SCOPE_SCORE'] = df['ADAPMIT_SCOPE'].apply(
+    #     lambda x: next((item['score'] for item in x if item['label'] == 'MitigationLabel'), 0)
+    # )
+    # df['ADAPMIT_TECH_SCORE'] = df['ADAPMIT_TECH'].apply(
+    #     lambda x: next((item['score'] for item in x if item['label'] == 'MitigationLabel'), 0)
+    # )
+    
+    # # Calculate average mitigation score
+    # df['ADAPMIT_SCORE'] = (df['ADAPMIT_SCOPE_SCORE'] + df['ADAPMIT_TECH_SCORE']) / 2
+
+    df['ADAPMIT'] = df.apply(lambda x: 'Adaptation' if x['ADAPMIT_SCOPE'] == 'Adaptation' and x['ADAPMIT_TECH'] == 'Adaptation' else 'Mitigation', axis=1)
+
+
+
     # Convert funding columns to numeric, replacing any non-numeric values with NaN
     df['maf_funding'] = pd.to_numeric(df['maf_funding'], errors='coerce')
     df['cont_public'] = pd.to_numeric(df['cont_public'], errors='coerce')
@@ -284,6 +309,14 @@ def process_data(uploaded_file, sens_level):
         else 'PRE-ASSESSMENT' if sens_level+1 <= x['pred_score'] <= sens_level+2 
         else 'FULL-ASSESSMENT' if x['pred_score'] > sens_level+2
         else 'ERROR', axis=1)
+    
+    # Reorder columns in final dataframe
+    column_order = ['id', 'scope_txt', 'tech_txt', 'fin_txt', 'maf_funding', 'cont_public', 
+                    'cont_private', 'cont_other', 'scope_lab1', 'scope_lab2', 'tech_lab1', 
+                    'tech_lab3', 'fin_lab2', 'ADAPMIT_SCOPE', 'ADAPMIT_TECH', 'ADAPMIT', 'SECTOR1', 
+                    'SECTOR2', 'LANG', 'lev_total', 'lev_gt_0', 'lev_maf_%', 'lev_maf_scale',
+                    'word_length_check', 'pred_score', 'pred_action']
+    df = df[column_order]
 
     return df