Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| """ | |
| Created on Mon Jan 1 11:20:18 2024 | |
| @author: mohanadafiffy | |
| """ | |
| import os | |
| import streamlit as st | |
| import pandas as pd | |
| import requests | |
| import os | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| host=os.getenv("backend") | |
| CompanyBackendService=host+'/receive_companies/' | |
| UserBackendService=host+'/receive_users/' | |
| BothFeaturesService=host+'/receive_data/' | |
| NGOEmailsService=host+'/receive_ngo_emails/' | |
| IndustryEmailService=host+'/receive_industry_email/' | |
| def add_https_to_urls(df, column_name): | |
| """ | |
| Adds 'https://' to URLs in the specified column of a DataFrame if they don't already start with a valid protocol. | |
| Corrects URLs starting with 'http:/' or 'https:/'. | |
| Handles missing values, trims whitespace, and is case-insensitive. | |
| Parameters: | |
| df (pandas.DataFrame): The DataFrame containing the URLs. | |
| column_name (str): The name of the column with URLs. | |
| """ | |
| # Define a helper function to add or correct protocols | |
| def correct_protocol(url): | |
| if pd.isna(url) or url.strip() == '': | |
| return url # Return as is if the URL is NaN or empty | |
| url = url.strip() # Trim whitespace | |
| lower_url = url.lower() | |
| if lower_url.startswith('http:/') and not lower_url.startswith('http://'): | |
| return 'http://' + url[6:] | |
| elif lower_url.startswith('https:/') and not lower_url.startswith('https://'): | |
| return 'https://' + url[7:] | |
| elif not lower_url.startswith(('http://', 'https://')): | |
| return 'https://' + url | |
| return url | |
| # Apply the helper function to the specified column | |
| df[column_name] = df[column_name].apply(correct_protocol) | |
| return df | |
| def CompanySpecificClient(email_receiver): | |
| input_data_companies = None | |
| submitted_companies = False | |
| uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="CompanyUploader") | |
| opt_out_scraping = st.checkbox("Opt out of scraping",key="CompanyScraper") | |
| with st.form(key='Comapny_form'): | |
| if uploaded_file is not None: | |
| try: | |
| # Detect file type and read accordingly | |
| file_type = uploaded_file.name.split('.')[-1] | |
| if file_type == 'csv': | |
| df = pd.read_csv(uploaded_file) | |
| elif file_type == 'xlsx': | |
| df = pd.read_excel(uploaded_file) | |
| # Check if 'Website' column exists | |
| if 'Website' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| website_column = st.selectbox("Select the column for Website:", all_columns,key="CompanyWebsite") | |
| else: | |
| website_column = 'Website' | |
| # Check if 'Company Name for Emails' column exists | |
| if 'Company Name for Emails' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| company_column= st.selectbox("Select the column for Company Name for Emails:", all_columns,key="CompanyName") | |
| else: | |
| company_column = 'Company Name for Emails' | |
| if 'Apollo description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| Apollo_description= st.selectbox("Select the column for Apollo description:", all_columns,key="ApolloDescription") | |
| else: | |
| Apollo_description = 'Apollo description' | |
| if opt_out_scraping: | |
| if 'Company Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| description_column = st.selectbox("Select the column for Description:", all_columns,key="CompanyDescription") | |
| df.rename(columns={description_column: 'scraped_content'}, inplace=True) | |
| else: | |
| df.rename(columns={'Company Description': 'scraped_content'}, inplace=True) | |
| input_data_companies = df | |
| except Exception as E : | |
| st.error("An error occured while processing the file") | |
| prompt_notes= st.text_input("If applicable please mention the network name",key="CompanyPromptNotes") | |
| # If the button is clicked, it will return True for this run | |
| button_clicked = st.form_submit_button("Submit for processing") | |
| # 2. Update session state for the button | |
| if button_clicked: | |
| submitted_companies = True | |
| # Set the session state to the new value | |
| prompt_notes = prompt_notes | |
| # 3. Use the session state variable to determine if the button was previously clicked | |
| if submitted_companies and input_data_companies is not None: | |
| df = input_data_companies | |
| if not opt_out_scraping: | |
| df[website_column] = df[website_column].astype(str) | |
| df=df[[website_column,company_column,Apollo_description]] | |
| df.columns = ["Website","Company Name for Emails","Apollo description"] | |
| df = df.drop_duplicates(subset="Website", keep='first') | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| else: | |
| df[website_column] = df[website_column].astype(str) | |
| df=df[[website_column,company_column,Apollo_description,"scraped_content"]] | |
| df.columns = ["Website","Company Name for Emails","Apollo description","scraped_content"] | |
| df = df.drop_duplicates(subset="Website", keep='first') | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| df=add_https_to_urls(df, 'Website') | |
| st.write(df) | |
| # Convert DataFrame to CSV for transmission | |
| csv = df.to_csv(index=False) | |
| # Construct the data to send | |
| data_to_send = {"prompt_notes": prompt_notes, "dataframe": csv,"email_receiver":email_receiver,"filename": uploaded_file.name} | |
| # Sending the POST request to FastAPI | |
| response = requests.post(CompanyBackendService, json=data_to_send) | |
| if response.status_code == 200: | |
| st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.") | |
| else: | |
| st.error("Data transmission failed. Please verify that your file contains the labels 'Company Website' and 'Company Name'. Additionally, ensure that your file is valid and contains records and try again , if the problem persists please contact us at [email protected]") | |
| return None | |
| def UserSpecificClient(email_receiver): | |
| input_data=None | |
| submitted=None | |
| column_selections = {} | |
| uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="UserUploader") | |
| opt_out_scraping = st.checkbox("Opt out of scraping",key="userSraping") | |
| with st.form(key='User_Form'): | |
| if uploaded_file is not None: | |
| try: | |
| # Detect file type and read accordingly | |
| file_type = uploaded_file.name.split('.')[-1] | |
| if file_type == 'csv': | |
| try: | |
| df = pd.read_csv(uploaded_file) | |
| except: | |
| df = pd.read_csv(uploaded_file, encoding='ISO-8859-1') | |
| # Check if 'Person Linkedin Url' column exists | |
| required_essential_columns = ['First Name','Company Name for Emails','Email'] | |
| missing_essential_columns = [col for col in required_essential_columns if col not in df.columns] | |
| required_scraping_columns=['Title','Website','Last Name','Person Linkedin Url'] | |
| missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns] | |
| for col in missing_essential_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col) | |
| column_selections[col] = selected_column | |
| # Generate selectboxes for missing scraping columns if not opting out | |
| if not opt_out_scraping: | |
| for col in missing_scraping_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col) | |
| column_selections[col] = selected_column | |
| # Process the column renaming based on the selections | |
| for col, selected_column in column_selections.items(): | |
| df.rename(columns={selected_column: col}, inplace=True) | |
| if opt_out_scraping: | |
| if 'User Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| description_column = st.selectbox("Select the column for Description:", all_columns,key="userdescription") | |
| df.rename(columns={description_column: 'Scrapped Profile'}, inplace=True) | |
| else: | |
| df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True) | |
| # Check if "Person Linkedin Url" is in the DataFrame | |
| if 'Person Linkedin Url' not in df.columns: | |
| # Use the DataFrame index to generate a unique value for each row | |
| # You can adjust this to create a more complex identifier | |
| df['Person Linkedin Url'] = 'LI_' + df.index.astype(str) | |
| input_data = df | |
| except Exception as E: | |
| st.write(E) | |
| st.error("An error occurred while processing the file") | |
| # If the button is clicked, it will return True for this run | |
| button_clicked = st.form_submit_button("Submit") | |
| # Update session state for the button | |
| if button_clicked: | |
| submitted = True | |
| # Use the session state variable to determine if the button was previously clicked | |
| if submitted and input_data is not None: | |
| df = input_data | |
| df = df.drop_duplicates(subset="Person Linkedin Url", keep='first') | |
| if opt_out_scraping: | |
| df=df[['First Name','Company Name for Emails','Person Linkedin Url','Scrapped Profile','Email']] | |
| else: | |
| df=df[['First Name', 'Last Name', 'Title', 'Website','Company Name for Emails','Person Linkedin Url','Email']] | |
| df=add_https_to_urls(df, 'Website') | |
| # Convert DataFrame to CSV for transmission | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| st.write(df) | |
| csv = df.to_csv(index=False) | |
| # Construct the data to send | |
| data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"email_template":"False","filename": uploaded_file.name} | |
| # Sending the POST request to FastAPI | |
| response = requests.post(UserBackendService, json=data_to_send) | |
| if response.status_code == 200: | |
| st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.") | |
| else: | |
| st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]") | |
| def bothFeaturesFunction(email_receiver): | |
| input_data=None | |
| submitted=None | |
| column_selections = {} | |
| uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="BothFeaturesUploader") | |
| opt_out_scraping = st.checkbox("Opt out of scraping",key="BothOptOut") | |
| with st.form(key='User_Form'): | |
| if uploaded_file is not None: | |
| try: | |
| # Detect file type and read accordingly | |
| file_type = uploaded_file.name.split('.')[-1] | |
| if file_type == 'csv': | |
| try: | |
| df = pd.read_csv(uploaded_file) | |
| except: | |
| df = pd.read_csv(uploaded_file, encoding='ISO-8859-1') | |
| # Check if 'Person Linkedin Url' column exists | |
| required_essential_columns = ['First Name','Company Name for Emails','Email'] | |
| missing_essential_columns = [col for col in required_essential_columns if col not in df.columns] | |
| required_scraping_columns=['Title','Last Name','Person Linkedin Url','Website'] | |
| missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns] | |
| for col in missing_essential_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col) | |
| column_selections[col] = selected_column | |
| # Generate selectboxes for missing scraping columns if not opting out | |
| if not opt_out_scraping: | |
| for col in missing_scraping_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col) | |
| column_selections[col] = selected_column | |
| # Process the column renaming based on the selections | |
| for col, selected_column in column_selections.items(): | |
| df.rename(columns={selected_column: col}, inplace=True) | |
| if opt_out_scraping: | |
| if 'Company Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| description_column = st.selectbox("Select the column for Company Description:", all_columns,key="bothCompanyDescription") | |
| df.rename(columns={description_column: 'scraped_content'}, inplace=True) | |
| else: | |
| df.rename(columns={'Company Description': 'scraped_content'}, inplace=True) | |
| if 'User Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| description_column = st.selectbox("Select the column for User Description:", all_columns,key="bothuserdescription") | |
| df.rename(columns={description_column: 'Scrapped Profile'}, inplace=True) | |
| else: | |
| df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True) | |
| # Check if "Person Linkedin Url" is in the DataFrame | |
| if 'Person Linkedin Url' not in df.columns: | |
| # Use the DataFrame index to generate a unique value for each row | |
| # You can adjust this to create a more complex identifier | |
| df['Person Linkedin Url'] = 'LI_' + df.index.astype(str) | |
| input_data = df | |
| except Exception as E: | |
| st.write(E) | |
| st.error("An error occurred while processing the file") | |
| # If the button is clicked, it will return True for this run | |
| prompt_notes= st.text_input("If applicable please mention the network name",key="CompanyPromptNotes") | |
| button_clicked = st.form_submit_button("Submit") | |
| # Update session state for the button | |
| if button_clicked: | |
| submitted = True | |
| # Use the session state variable to determine if the button was previously clicked | |
| if submitted and input_data is not None: | |
| df = input_data | |
| df = df.drop_duplicates(subset="Person Linkedin Url", keep='first') | |
| if opt_out_scraping: | |
| df=df[['First Name','Person Linkedin Url','Scrapped Profile',"Company Name for Emails","scraped_content","Email"]] | |
| else: | |
| df=df[['First Name', 'Last Name', 'Title', 'Person Linkedin Url',"Website","Company Name for Emails","Email"]] | |
| df=add_https_to_urls(df, 'Website') | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| st.write(df) | |
| # Convert DataFrame to CSV for transmission | |
| csv = df.to_csv(index=False) | |
| # Construct the data to send | |
| data_to_send = {"prompt_notes": prompt_notes, "dataframe": csv,"email_receiver":email_receiver,"filename": uploaded_file.name} | |
| # Sending the POST request to FastAPI | |
| response = requests.post(BothFeaturesService, json=data_to_send) | |
| if response.status_code == 200: | |
| st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.") | |
| else: | |
| st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]") | |
| def BH_Ngo(email_receiver,calendly_link,sender_name): | |
| input_data=None | |
| submitted=None | |
| column_selections = {} | |
| uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="BothFeaturesUploader") | |
| opt_out_scraping = st.checkbox("Opt out of scraping",key="BothOptOut") | |
| with st.form(key='User_Form'): | |
| if uploaded_file is not None: | |
| try: | |
| # Detect file type and read accordingly | |
| file_type = uploaded_file.name.split('.')[-1] | |
| if file_type == 'csv': | |
| try: | |
| df = pd.read_csv(uploaded_file) | |
| except: | |
| df = pd.read_csv(uploaded_file, encoding='ISO-8859-1') | |
| # Check if 'Person Linkedin Url' column exists | |
| required_essential_columns = ['First Name','Company Name for Emails','Domain','Email'] | |
| missing_essential_columns = [col for col in required_essential_columns if col not in df.columns] | |
| required_scraping_columns=['Title','Person Linkedin Url','Website'] | |
| missing_scraping_columns = [col for col in required_scraping_columns if col not in df.columns] | |
| for col in missing_essential_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns,key=col) | |
| column_selections[col] = selected_column | |
| # Generate selectboxes for missing scraping columns if not opting out | |
| if not opt_out_scraping: | |
| for col in missing_scraping_columns: | |
| all_columns = df.columns.tolist() | |
| selected_column = st.selectbox(f"Select the column for {col}:", all_columns, key=col) | |
| column_selections[col] = selected_column | |
| # Process the column renaming based on the selections | |
| for col, selected_column in column_selections.items(): | |
| df.rename(columns={selected_column: col}, inplace=True) | |
| if opt_out_scraping: | |
| if 'User Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| User_description_column = st.selectbox("Select the column for User Description:", all_columns,key="bothuserdescription") | |
| df.rename(columns={User_description_column: 'Scrapped Profile'}, inplace=True) | |
| else: | |
| df.rename(columns={'User Description': 'Scrapped Profile'}, inplace=True) | |
| # Check if "Person Linkedin Url" is in the DataFrame | |
| if 'Person Linkedin Url' not in df.columns: | |
| # Use the DataFrame index to generate a unique value for each row | |
| # You can adjust this to create a more complex identifier | |
| df['Person Linkedin Url'] = 'LI_' + df.index.astype(str) | |
| input_data = df | |
| except Exception as E: | |
| st.write(E) | |
| st.error("An error occurred while processing the file") | |
| # If the button is clicked, it will return True for this run | |
| button_clicked = st.form_submit_button("Submit") | |
| # Update session state for the button | |
| if button_clicked: | |
| submitted = True | |
| # Use the session state variable to determine if the button was previously clicked | |
| if submitted and input_data is not None: | |
| df = input_data | |
| df = df.drop_duplicates(subset="Person Linkedin Url", keep='first') | |
| if opt_out_scraping: | |
| df=df[['First Name','Person Linkedin Url','Scrapped Profile',"Company Name for Emails","Domain","Email"]] | |
| else: | |
| columns_to_select = ['First Name', 'Title', 'Person Linkedin Url', "Company Name for Emails", "Domain","Website","Email"] | |
| df=add_https_to_urls(df, 'Website') | |
| if 'Last Name' in df.columns: | |
| columns_to_select.insert(1, 'Last Name') # Insert 'Last Name' at the correct position | |
| df = df[columns_to_select] | |
| # Convert DataFrame to CSV for transmission | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| st.write(df) | |
| csv = df.to_csv(index=False) | |
| # Construct the data to send | |
| data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"calendly_link":calendly_link,"sender_name":sender_name} | |
| # Sending the POST request to FastAPI | |
| response = requests.post(NGOEmailsService, json=data_to_send) | |
| if response.status_code == 200: | |
| st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.") | |
| else: | |
| st.error("Data transmission failed. Please verify that your file contains the labels 'Company' and 'Person Linkedin Url'. Additionally, ensure that your file is valid and contains records and try again, if the problem persists please contact us at [email protected]") | |
| def BH_industry(email_receiver,calendly_link,sender_name): | |
| input_data_companies = None | |
| submitted_companies = False | |
| uploaded_file = st.file_uploader("Kindly upload a CSV file that includes the names and websites of the companies", type=["csv"],key="CompanyUploader") | |
| opt_out_scraping = st.checkbox("Opt out of scraping",key="CompanyScraper") | |
| with st.form(key='Comapny_form'): | |
| if uploaded_file is not None: | |
| try: | |
| # Detect file type and read accordingly | |
| file_type = uploaded_file.name.split('.')[-1] | |
| if file_type == 'csv': | |
| df = pd.read_csv(uploaded_file) | |
| elif file_type == 'xlsx': | |
| df = pd.read_excel(uploaded_file) | |
| # Check if 'Website' column exists | |
| if 'Website' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| website_column = st.selectbox("Select the column for Website:", all_columns,key="CompanyWebsite") | |
| else: | |
| website_column = 'Website' | |
| if 'First Name' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| name_column = st.selectbox("Select the column for first name:", all_columns,key="firstname") | |
| else: | |
| name_column = 'First Name' | |
| # Check if 'Company Name for Emails' column exists | |
| if 'Company Name for Emails' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| company_column= st.selectbox("Select the column for Company Name :", all_columns,key="CompanyName") | |
| else: | |
| company_column = 'Company Name for Emails' | |
| if 'Email' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| Email_column= st.selectbox("Select the column for email:", all_columns,key="Companyemail") | |
| else: | |
| Email_column = 'Email' | |
| if opt_out_scraping: | |
| if 'Company Description' not in df.columns: | |
| all_columns = df.columns.tolist() | |
| description_column = st.selectbox("Select the column for Description:", all_columns,key="CompanyDescription") | |
| df.rename(columns={description_column: 'scraped_content'}, inplace=True) | |
| else: | |
| df.rename(columns={'Company Description': 'scraped_content'}, inplace=True) | |
| input_data_companies = df | |
| except Exception as E : | |
| st.error("An error occured while processing the file") | |
| # Fetch the filtered data | |
| # If the button is clicked, it will return True for this run | |
| button_clicked = st.form_submit_button("Submit for processing") | |
| # 2. Update session state for the button | |
| if button_clicked: | |
| submitted_companies = True | |
| # 3. Use the session state variable to determine if the button was previously clicked | |
| if submitted_companies and input_data_companies is not None: | |
| df = input_data_companies | |
| if not opt_out_scraping: | |
| df[website_column] = df[website_column].astype(str) | |
| df=df[[website_column,company_column,name_column,Email_column]] | |
| df.columns = ["Website","Company Name for Emails","First Name","Email"] | |
| df = df.drop_duplicates(subset="Email", keep='first') | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| else: | |
| df[website_column] = df[website_column].astype(str) | |
| df=df[[website_column,company_column,"scraped_content",name_column,Email_column]] | |
| df.columns = ["Website","Company Name for Emails","scraped_content","First Name","Email"] | |
| df = df.drop_duplicates(subset="Email", keep='first') | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| df = df.dropna().loc[~(df == '').all(axis=1)] | |
| df=add_https_to_urls(df, 'Website') | |
| st.write(df) | |
| # Convert DataFrame to CSV for transmission | |
| csv = df.to_csv(index=False) | |
| # Construct the data to send | |
| data_to_send = {"dataframe": csv, "email_receiver": email_receiver,"calendly_link":calendly_link,"sender_name":sender_name} | |
| # Sending the POST request to FastAPI | |
| response = requests.post(IndustryEmailService, json=data_to_send) | |
| if response.status_code == 200: | |
| st.info(f"We're processing your request. You can close the app now. An email will be sent to {email_receiver} once the process is finished.") | |
| else: | |
| st.error("Data transmission failed. Please verify that your file contains the labels 'Company Website' and 'Company Name'. Additionally, ensure that your file is valid and contains records and try again , if the problem persists please contact us at [email protected]") | |
| return None |