clients.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jan  1 11:20:18 2024

@author: mohanadafiffy
"""
import os 
import streamlit as st
import pandas as pd
import requests
import os

host=os.getenv("backend")

CompanyBackendService=host+'/receive_companies/'
UserBackendService=host+'/receive_users/'
BothFeaturesService=host+'/receive_data/'
NGOEmailsService=host+'/receive_ngo_emails/'
IndustryEmailService=host+'/receive_industry_email/'

            
def add_https_to_urls(df, column_name):
    """
    Adds 'https://' to URLs in the specified column of a DataFrame if they don't already start with a valid protocol.
    Corrects URLs starting with 'http:/' or 'https:/'.
    Handles missing values, trims whitespace, and is case-insensitive.

    Parameters:
    df (pandas.DataFrame): The DataFrame containing the URLs.
    column_name (str): The name of the column with URLs.
    """
    # Define a helper function to add or correct protocols
    def correct_protocol(url):
        if pd.isna(url) or url.strip() == '':
            return url  # Return as is if the URL is NaN or empty
        url = url.strip()  # Trim whitespace
        lower_url = url.lower()
        if lower_url.startswith('http:/') and not lower_url.startswith('http://'):
            return 'http://' + url[6:]
        elif lower_url.startswith('https:/') and not lower_url.startswith('https://'):
            return 'https://' + url[7:]
        elif not lower_url.startswith(('http://', 'https://')):
            return 'https://' + url
        return url

    # Apply the helper function to the specified column
    df[column_name] = df[column_name].apply(correct_protocol)
    return df

def CompanySpecificClient(email_receiver):
    input_data_companies = None
    submitted_companies = False 
    uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="CompanyUploader")
    opt_out_scraping = st.checkbox("Opt out of scraping",key="CompanyScraper")
    with st.form(key='Comapny_form'):  
        if uploaded_file is not None:
    
            try:
                # Detect file type and read accordingly
                file_type = uploaded_file.name.split('.')[-1]
                if file_type == 'csv':
                    df = pd.read_csv(uploaded_file)
                elif file_type == 'xlsx':
                    df = pd.read_excel(uploaded_file)
                    # Check if 'Website' column exists
                if 'Website' not in df.columns:
                    all_columns = df.columns.tolist()
                    website_column  = st.selectbox("Select the column for Website:", all_columns,key="CompanyWebsite")
                else:
                    website_column  = 'Website'
                    # Check if 'Company Name for Emails' column exists
                if 'Company Name for Emails' not in df.columns:
                    all_columns = df.columns.tolist()
                    company_column= st.selectbox("Select the column for Company Name for Emails:", all_columns,key="CompanyName")
                else:
                    company_column = 'Company Name for Emails'
                    
                if opt_out_scraping:
                    if 'Company Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        description_column = st.selectbox("Select the column for Description:", all_columns,key="CompanyDescription")
                        df.rename(columns={description_column: 'scraped_content'}, inplace=True)
                    else:
                        df.rename(columns={'Company Description': 'scraped_content'}, inplace=True)     
                    
                input_data_companies = df  
                
            except Exception as E :
                st.error("An error occured while processing the file")
       
            # Fetch the filtered data
            
                
            
        prompt_notes= st.text_input("If applicable please mention the network name",key="CompanyPromptNotes")
        
        # If the button is clicked, it will return True for this run
        button_clicked = st.form_submit_button("Submit for processing")
        
        # 2. Update session state for the button
        if button_clicked:
            submitted_companies = True
            # Set the session state to the new value
            prompt_notes = prompt_notes 
# 3. Use the session state variable to determine if the button was previously clicked
    if submitted_companies and input_data_companies is not None:
        df = input_data_companies
        if not opt_out_scraping:
            df[website_column] = df[website_column].astype(str)
            df=df[[website_column,company_column]]
            df.columns = ["Website","Company Name for Emails"]
            df = df.drop_duplicates(subset="Website", keep='first')
            df = df.dropna().loc[~(df == '').all(axis=1)]
        else:
            df[website_column] = df[website_column].astype(str)
            df=df[[website_column,company_column,"scraped_content"]]
            df.columns = ["Website","Company Name for Emails","scraped_content"]
            df = df.drop_duplicates(subset="Website", keep='first')
            df = df.dropna().loc[~(df == '').all(axis=1)]
            
        df = df.dropna().loc[~(df == '').all(axis=1)]  
        df=add_https_to_urls(df, 'Website')
        st.write(df)    
        # Convert DataFrame to CSV for transmission
        csv = df.to_csv(index=False)
    
        # Construct the data to send
        data_to_send = {"prompt_notes": prompt_notes, "dataframe": csv,"email_receiver":email_receiver,"filename": uploaded_file.name}
    
        # Sending the POST request to FastAPI
        response = requests.post(CompanyBackendService, json=data_to_send)
    
        if response.status_code == 200:
            st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.")
        else:
            st.error("Data transmission failed. Please verify that your file contains the labels 'Company Website' and 'Company Name'. Additionally, ensure that your file is valid and contains records and try again , if the problem persists please contact us at [email protected]") 
    return None        
def UserSpecificClient(email_receiver):     
    input_data=None    
    submitted=None 
    column_selections = {}
    uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="UserUploader")
    opt_out_scraping = st.checkbox("Opt out of scraping",key="userSraping")
    with st.form(key='User_Form'):
        if uploaded_file is not None:
            try:
                # Detect file type and read accordingly
                file_type = uploaded_file.name.split('.')[-1]
                if file_type == 'csv':
                    try:
                        df = pd.read_csv(uploaded_file)
                    except:
                        df = pd.read_csv(uploaded_file, encoding='ISO-8859-1')
                # Check if 'Person Linkedin Url' column exists
                required_essential_columns = ['First Name','Company Name for Emails','Email']
                missing_essential_columns = [col for col in required_essential_columns if col not in df.columns]
                required_scraping_columns=['Title','Website','Last Name','Person Linkedin Url']
                missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns]
                for col in missing_essential_columns:
                    all_columns = df.columns.tolist()
                    selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col)
                    column_selections[col] = selected_column
                # Generate selectboxes for missing scraping columns if not opting out
                if not opt_out_scraping:
                    for col in missing_scraping_columns:
                        all_columns = df.columns.tolist()
                        selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col)
                        column_selections[col] = selected_column
                # Process the column renaming based on the selections
                for col, selected_column in column_selections.items():
                    df.rename(columns={selected_column: col}, inplace=True)
                    
                if opt_out_scraping:
                    if 'User Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        description_column = st.selectbox("Select the column for Description:", all_columns,key="userdescription")
                        df.rename(columns={description_column: 'Scrapped Profile'}, inplace=True)
                    else:
                        df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True)                    
                    # Check if "Person Linkedin Url" is in the DataFrame
                    if 'Person Linkedin Url' not in df.columns:
                        # Use the DataFrame index to generate a unique value for each row
                        # You can adjust this to create a more complex identifier
                        df['Person Linkedin Url'] = 'LI_' + df.index.astype(str)    
                input_data = df
    
            except Exception as E:
                st.write(E)
                st.error("An error occurred while processing the file")
        # If the button is clicked, it will return True for this run
        button_clicked = st.form_submit_button("Submit")

        # Update session state for the button
        if button_clicked:
            submitted = True

    # Use the session state variable to determine if the button was previously clicked
    if submitted and input_data is not None:
        df = input_data
        df = df.drop_duplicates(subset="Person Linkedin Url", keep='first')    
        if opt_out_scraping:
            df=df[['First Name','Company Name for Emails','Person Linkedin Url','Scrapped Profile','Email']]
        else:
            df=df[['First Name', 'Last Name', 'Title', 'Website','Company Name for Emails','Person Linkedin Url','Email']]
            df=add_https_to_urls(df, 'Website')
            
        # Convert DataFrame to CSV for transmission
        df = df.dropna().loc[~(df == '').all(axis=1)]  
        st.write(df)
        csv = df.to_csv(index=False)

        # Construct the data to send
        
        data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"email_template":"False","filename": uploaded_file.name}

        # Sending the POST request to FastAPI
        response = requests.post(UserBackendService, json=data_to_send)

        if response.status_code == 200:
            st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.")
        else:
            st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]")    
   
def bothFeaturesFunction(email_receiver):
    input_data=None    
    submitted=None 
    column_selections = {}
    uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="BothFeaturesUploader")
    opt_out_scraping = st.checkbox("Opt out of scraping",key="BothOptOut")
    with st.form(key='User_Form'):
        if uploaded_file is not None:
            try:
                # Detect file type and read accordingly
                file_type = uploaded_file.name.split('.')[-1]
                if file_type == 'csv':
                    try:
                        df = pd.read_csv(uploaded_file)
                    except:
                        df = pd.read_csv(uploaded_file, encoding='ISO-8859-1')
                # Check if 'Person Linkedin Url' column exists
                required_essential_columns = ['First Name','Company Name for Emails','Email']
                missing_essential_columns = [col for col in required_essential_columns if col not in df.columns]
                required_scraping_columns=['Title','Last Name','Person Linkedin Url','Website']
                missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns]
                for col in missing_essential_columns:
                    all_columns = df.columns.tolist()
                    selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col)
                    column_selections[col] = selected_column
                # Generate selectboxes for missing scraping columns if not opting out
                if not opt_out_scraping:
                    for col in missing_scraping_columns:
                        all_columns = df.columns.tolist()
                        selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col)
                        column_selections[col] = selected_column
                # Process the column renaming based on the selections
                for col, selected_column in column_selections.items():
                    df.rename(columns={selected_column: col}, inplace=True)
                    
                if opt_out_scraping:
                    if 'Company Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        description_column = st.selectbox("Select the column for Company Description:", all_columns,key="bothCompanyDescription")
                        df.rename(columns={description_column: 'scraped_content'}, inplace=True)
                    else:
                        df.rename(columns={'Company Description': 'scraped_content'}, inplace=True)    
                    if 'User Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        description_column = st.selectbox("Select the column for User Description:", all_columns,key="bothuserdescription")
                        df.rename(columns={description_column: 'Scrapped Profile'}, inplace=True)
                    else:
                        df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True)                    
                    # Check if "Person Linkedin Url" is in the DataFrame
                    if 'Person Linkedin Url' not in df.columns:
                        # Use the DataFrame index to generate a unique value for each row
                        # You can adjust this to create a more complex identifier
                        df['Person Linkedin Url'] = 'LI_' + df.index.astype(str)    
                input_data = df
    
            except Exception as E:
                st.write(E)
                st.error("An error occurred while processing the file")
        # If the button is clicked, it will return True for this run
        prompt_notes= st.text_input("If applicable please mention the network name",key="CompanyPromptNotes")
        button_clicked = st.form_submit_button("Submit")

        # Update session state for the button
        if button_clicked:
            submitted = True

    # Use the session state variable to determine if the button was previously clicked
    if submitted and input_data is not None:
        df = input_data
        df = df.drop_duplicates(subset="Person Linkedin Url", keep='first') 
        if opt_out_scraping:
            df=df[['First Name','Person Linkedin Url','Scrapped Profile',"Company Name for Emails","scraped_content","Email"]]
        else:
            df=df[['First Name', 'Last Name', 'Title', 'Person Linkedin Url',"Website","Company Name for Emails","Email"]]
            df=add_https_to_urls(df, 'Website')
            
        df = df.dropna().loc[~(df == '').all(axis=1)]  
            
        st.write(df)
        # Convert DataFrame to CSV for transmission
        csv = df.to_csv(index=False)

        # Construct the data to send
        data_to_send = {"prompt_notes": prompt_notes, "dataframe": csv,"email_receiver":email_receiver,"filename": uploaded_file.name}

        # Sending the POST request to FastAPI
        response = requests.post(BothFeaturesService, json=data_to_send)

        if response.status_code == 200:
            st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.")
        else:
            st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]")  

def BH_Ngo(email_receiver,calendly_link,sender_name):
    input_data=None    
    submitted=None 
    column_selections = {}
    uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="BothFeaturesUploader")
    opt_out_scraping = st.checkbox("Opt out of scraping",key="BothOptOut")
    with st.form(key='User_Form'):
        if uploaded_file is not None:
            try:
                # Detect file type and read accordingly
                file_type = uploaded_file.name.split('.')[-1]
                if file_type == 'csv':
                    try:
                        df = pd.read_csv(uploaded_file)
                    except:
                        df = pd.read_csv(uploaded_file, encoding='ISO-8859-1')
                # Check if 'Person Linkedin Url' column exists
                required_essential_columns = ['First Name','Company Name for Emails','Domain','Email']
                missing_essential_columns = [col for col in required_essential_columns if col not in df.columns]
                required_scraping_columns=['Title','Person Linkedin Url','Website']
                missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns]
                for col in missing_essential_columns:
                    all_columns = df.columns.tolist()
                    selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col)
                    column_selections[col] = selected_column
                # Generate selectboxes for missing scraping columns if not opting out
                if not opt_out_scraping:
                    for col in missing_scraping_columns:
                        all_columns = df.columns.tolist()
                        selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col)
                        column_selections[col] = selected_column
                # Process the column renaming based on the selections
                for col, selected_column in column_selections.items():
                    df.rename(columns={selected_column: col}, inplace=True)
                    
                if opt_out_scraping: 
                    if 'User Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        User_description_column = st.selectbox("Select the column for User Description:", all_columns,key="bothuserdescription")
                        df.rename(columns={User_description_column: 'Scrapped Profile'}, inplace=True)
                    else:
                        df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True)                    
                    # Check if "Person Linkedin Url" is in the DataFrame
                    if 'Person Linkedin Url' not in df.columns:
                        # Use the DataFrame index to generate a unique value for each row
                        # You can adjust this to create a more complex identifier
                        df['Person Linkedin Url'] = 'LI_' + df.index.astype(str)    
                input_data = df
    
            except Exception as E:
                st.write(E)
                st.error("An error occurred while processing the file")
        # If the button is clicked, it will return True for this run
        button_clicked = st.form_submit_button("Submit")

        # Update session state for the button
        if button_clicked:
            submitted = True

    # Use the session state variable to determine if the button was previously clicked
    if submitted and input_data is not None:
        df = input_data
        
        df = df.drop_duplicates(subset="Person Linkedin Url", keep='first')    
        if opt_out_scraping:
            df=df[['First Name','Person Linkedin Url','Scrapped Profile',"Company Name for Emails","Domain","Email"]]
            
        else:
            
            columns_to_select = ['First Name', 'Title', 'Person Linkedin Url', "Company Name for Emails", "Domain","Website","Email"]
            df=add_https_to_urls(df, 'Website')
            if 'Last Name' in df.columns:
                columns_to_select.insert(1, 'Last Name')  # Insert 'Last Name' at the correct position
            
            df = df[columns_to_select]

            
        # Convert DataFrame to CSV for transmission
        df = df.dropna().loc[~(df == '').all(axis=1)]  
        st.write(df)
        csv = df.to_csv(index=False)

        # Construct the data to send
        data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"calendly_link":calendly_link,"sender_name":sender_name}

        # Sending the POST request to FastAPI
        response = requests.post(NGOEmailsService, json=data_to_send)

        if response.status_code == 200:
            st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.")
        else:
            st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]")  
            
def BH_industry(email_receiver,calendly_link,sender_name):
    input_data_companies = None
    submitted_companies = False 
    uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="CompanyUploader")
    opt_out_scraping = st.checkbox("Opt out of scraping",key="CompanyScraper")
    with st.form(key='Comapny_form'):  
        if uploaded_file is not None:
    
            try:
                # Detect file type and read accordingly
                file_type = uploaded_file.name.split('.')[-1]
                if file_type == 'csv':
                    df = pd.read_csv(uploaded_file)
                elif file_type == 'xlsx':
                    df = pd.read_excel(uploaded_file)
                    # Check if 'Website' column exists
                if 'Website' not in df.columns:
                    all_columns = df.columns.tolist()
                    website_column  = st.selectbox("Select the column for Website:", all_columns,key="CompanyWebsite")
                else:
                    website_column  = 'Website'
                if 'First Name' not in df.columns:
                    all_columns = df.columns.tolist()
                    name_column  = st.selectbox("Select the column for first name:", all_columns,key="firstname")
                else:
                    name_column  = 'First Name'                    
                    # Check if 'Company Name for Emails' column exists
                if 'Company Name for Emails' not in df.columns:
                    all_columns = df.columns.tolist()
                    company_column= st.selectbox("Select the column for Company Name :", all_columns,key="CompanyName")
                else:
                    company_column = 'Company Name for Emails'
                    
                if 'Email' not in df.columns:
                    all_columns = df.columns.tolist()
                    Email_column= st.selectbox("Select the column for email:", all_columns,key="Companyemail")
                else:
                    Email_column = 'Email'                    
                if opt_out_scraping:
                    if 'Company Description' not in df.columns:
                        all_columns = df.columns.tolist()
                        description_column = st.selectbox("Select the column for Description:", all_columns,key="CompanyDescription")
                        df.rename(columns={description_column: 'scraped_content'}, inplace=True)
                    else:
                        df.rename(columns={'Company Description': 'scraped_content'}, inplace=True)     
                    
                input_data_companies = df  
                
            except Exception as E :
                st.error("An error occured while processing the file")
       
            # Fetch the filtered data
            
        
        # If the button is clicked, it will return True for this run
        button_clicked = st.form_submit_button("Submit for processing")
        
        # 2. Update session state for the button
        if button_clicked:
            submitted_companies = True
# 3. Use the session state variable to determine if the button was previously clicked
    if submitted_companies and input_data_companies is not None:
        df = input_data_companies
        if not opt_out_scraping:
            df[website_column] = df[website_column].astype(str)
            df=df[[website_column,company_column,name_column,Email_column]]
            df.columns = ["Website","Company Name for Emails","First Name","Email"]
            df = df.drop_duplicates(subset="Email", keep='first')
            df = df.dropna().loc[~(df == '').all(axis=1)]
        else:
            df[website_column] = df[website_column].astype(str)
            df=df[[website_column,company_column,"scraped_content",name_column,Email_column]]
            df.columns = ["Website","Company Name for Emails","scraped_content","First Name","Email"]
            df = df.drop_duplicates(subset="Email", keep='first')
            df = df.dropna().loc[~(df == '').all(axis=1)]
            
        df = df.dropna().loc[~(df == '').all(axis=1)] 
        df=add_https_to_urls(df, 'Website')
        st.write(df)    
        # Convert DataFrame to CSV for transmission
        csv = df.to_csv(index=False)
    
        # Construct the data to send
        data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"calendly_link":calendly_link,"sender_name":sender_name}
    
        # Sending the POST request to FastAPI
        response = requests.post(IndustryEmailService, json=data_to_send)
    
        if response.status_code == 200:
            st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.")
        else:
            st.error("Data transmission failed. Please verify that your file contains the labels 'Company Website' and 'Company Name'. Additionally, ensure that your file is valid and contains records and try again , if the problem persists please contact us at [email protected]") 
    return None