bug fixing

modules/org_count.py

@@ -0,0 +1,125 @@
+import pandas as pd
+from thefuzz import fuzz
+
+
+def standardize_organization_names(df):
+    """
+    Standardizes organization names in a DataFrame using exact matches, abbreviations, and fuzzy matching.
+    
+    Args:
+        df (pd.DataFrame): DataFrame containing an 'organization' column
+        
+    Returns:
+        pd.DataFrame: DataFrame with added 'org_renamed' and 'concept_count' columns
+    """
+    # Make a copy to avoid modifying the original DataFrame
+    df = df.copy()
+    
+    # Dictionary of organization variations and their standardized names
+    org_variations = {
+        'Adventist Development Relief Agency': ['adventist development'],
+        'Asian Development Bank': ['asian development bank'],
+        'Association of the Regional Mechanism for Emissions Reductions of Boyacá, Colombia (MRRE)': [' regional mechanism for emissions reductions of boyacá'],
+        'BioCarbon Partners (BCP)': ['biocarbon partners'],
+        'Biothermica Technologies Inc': ['biothermica tech'],
+        'Brazilian Tourist Board': ['brazilian tourist board'],
+        'Caribbean Community Climate Change Centre': ['caribbean community climate'],
+        'Caritas': ['caritas'],
+        'Climate Advocacy International (CAI)': ['climate advocacy int'],
+        'Deutsche Gesellschaft für Internationale Zusammenarbeit (GIZ)': ['deutsche gesellschaft für internationale'],
+        'Deutsche Sparkassenstiftung (DSIK)': ['deutsche sparkassenstiftung'],
+        'East African Centre of Excellence for Renewable Energy and Efficiency (EACREEE)': ['east african centre of excellence for renewable'],
+        'Eco-Ideal': ['eco-ideal'],
+        'Global Green Growth Institute (GGGI)': ['global green growth'],
+        'Inter-American Development Bank (IDB)': ['american development bank'],
+        'Iskandar Regional Development Authority (IRDA)': ['iskandar regional'],
+        'Islamic Development Bank': ['islamic development bank'],
+        'Malaysian Industry Government Group for High Technology (MIGHT)': ['government group for high technology'],
+        'Osh Technological University': ['osh technological university','ошский технологический университет'],
+        'Oxford Policy Management (OPM)': ['oxford policy management'],
+        'Pacific Rim Investment Management': ['pacific rim investment'],
+        'Palestinian Energy and Natural Resources Authority (PENRA)': ['palestinian energy and natural'],
+        'Secretariat of the Pacific Regional Environment Programme (SPREP)': ['secretariat of the pacific regional environment programme (sprep)'],
+        'Serviço Nacional de Aprendizagem Industrial (SENAI)': ['serviço nacional de aprendizagem'],
+        'Sumy City Council': ['sumy city council'],
+        'Uganda Development Bank Limited (UDBL)': ['uganda development bank'],
+        'UN-Habitat': ['united nations human settlement','un-habitat'],
+        'United Nations Children\'s Fund (UNICEF)': ['united nations children'],
+        'United Nations Conference on Trade and Development (UNCTAD)': ['united nations conference on trade'],
+        'United Nations Development Programme (UNDP)': ['united nations development program'],
+        'United Nations Economic and Social Commission (ECOSOC)': ['united nations economic and social'],
+        'United Nations Environment Programme (UNEP)': ['united nations environment'],
+        'United Nations Industrial Development Organization (UNIDO)': ['united nations industrial'],
+        'United Nations Office for Project Services (UNOPS)': ['united nations office for project'],
+        'World Food Programme (WFP)': ['world food program'],
+        'World Resources Institute (WRI)': ['world resources institute'],
+        'World Wide Fund for Nature (WWF)': ['world wildlife','world wide fund for nature'],
+    }
+    
+    # Process exact matches first
+    df['check_name'] = None
+    for standard_name, variations in org_variations.items():
+        mask = df['organization'].str.lower().apply(lambda x: any(var in str(x) for var in variations))
+        df.loc[mask, 'check_name'] = standard_name
+    
+    # Dictionary of organization abbreviations
+    org_abreviations = {
+        'Deutsche Gesellschaft für Internationale Zusammenarbeit (GIZ)': ['GIZ'],
+        'Deutsche Sparkassenstiftung (DSIK)': ['DSIK'],
+        'East African Centre of Excellence for Renewable Energy and Efficiency (EACREEE)': ['EACREEE'],
+        'Global Green Growth Institute (GGGI)': ['GGGI'],
+        'UN-Habitat': ['UN-Habitat'],
+        'United Nations Children\'s Fund (UNICEF)': ['UNICEF'],
+        'United Nations Conference on Trade and Development (UNCTAD)': ['UNCTAD'],
+        'United Nations Development Programme (UNDP)': ['UNDP'],
+        'United Nations Economic and Social Commission (ECOSOC)': ['ECOSOC'],
+        'United Nations Environment Programme (UNEP)': ['UNEP'],
+        'United Nations Industrial Development Organization (UNIDO)': ['UNIDO'],
+        'United Nations Office for Project Services (UNOPS)': ['UNOPS'],
+        'World Food Programme (WFP)': ['WFP'],
+        'World Resources Institute (WRI)': ['WRI'],
+        'World Wide Fund for Nature (WWF)': ['WWF']
+    }
+    
+    # Process abbreviations
+    df['check_abreviation'] = None
+    for standard_name, abreviations in org_abreviations.items():
+        for abreviation in abreviations:
+            mask = df['organization'].str.contains(abreviation, regex=False, na=False)
+            df.loc[mask, 'check_abreviation'] = standard_name
+    
+    df['org_renamed'] = df.apply(lambda row: row['check_abreviation'] if pd.isnull(row['check_name']) else row['check_name'], axis=1)
+    df.drop(columns=['check_name', 'check_abreviation'], inplace=True)
+    
+    # Process fuzzy matches
+    unmatched_mask = df['org_renamed'].isna()
+    threshold = 90
+    
+    for idx, row in df[unmatched_mask].iterrows():
+        org_name = str(row['organization']).lower()
+        best_match = None
+        highest_ratio = 0
+        
+        for standard_name, variations in org_variations.items():
+            all_forms = [standard_name.lower()] + variations
+            for variant in all_forms:
+                ratio = fuzz.ratio(org_name, variant)
+                if ratio > threshold and ratio > highest_ratio:
+                    highest_ratio = ratio
+                    best_match = standard_name
+        
+        if best_match:
+            df.loc[idx, 'org_renamed'] = best_match
+    
+    # Fill remaining empty values with original names
+    df.loc[df['org_renamed'].isna(), 'org_renamed'] = df.loc[df['org_renamed'].isna(), 'organization']
+    
+    # Add concept count
+    df['concept_count'] = df.groupby('org_renamed').cumcount() + 1
+    
+    return df
+
+# Example usage:
+if __name__ == "__main__":
+    df = pd.read_csv('sandbox/maf_orgs.csv')
+    df = standardize_organization_names(df)