Upload app.py

app.py

@@ -12,6 +12,7 @@ from langchain.chat_models import ChatOpenAI
 from langchain.chains.question_answering import load_qa_chain
 from selenium import webdriver
 from selenium.webdriver.common.by import By
+from selenium.common.exceptions import NoSuchElementException
 import warnings
 warnings.filterwarnings('ignore')
 
@@ -121,165 +122,238 @@ class resume_analyzer:
         return response
 
 
-class linkedin_scrap:
+class linkedin_scraper:
 
-    def linkedin_open_scrolldown(driver, user_job_title):
+    def webdriver_setup():
+            
+        options = webdriver.ChromeOptions()
+        options.add_argument('--headless')
+        options.add_argument('--no-sandbox')
+        options.add_argument('--disable-dev-shm-usage')
+
+        driver = webdriver.Chrome(options=options)
+        driver.maximize_window()
+        return driver
+
+
+    def get_userinput():
+
+        add_vertical_space(2)
+        with st.form(key='linkedin_scarp'):
+
+            add_vertical_space(1)
+            col1,col2 = st.columns([0.7,0.3], gap='medium')
+            with col1:
+                job_title = st.text_input(label='Job Title')
+                job_title = job_title.split()
+            with col2:
+                job_count = st.number_input(label='Job Count', min_value=1, value=1, step=1)
+
+            # Submit Button
+            add_vertical_space(1)
+            submit = st.form_submit_button(label='Submit')
+            add_vertical_space(1)
+        
+        return job_title, job_count, submit
+
+
+    def build_url(job_title):
 
         b = []
-        for i in user_job_title:
+        for i in job_title:
             x = i.split()
             y = '%20'.join(x)
             b.append(y)
-        job_title = '%2C%20'.join(b)
 
+        job_title = '%2C%20'.join(b)
         link = f"https://in.linkedin.com/jobs/search?keywords={job_title}&location=India&locationId=&geoId=102713980&f_TPR=r604800&position=1&pageNum=0"
 
+        return link
+    
+
+    def link_open_scrolldown(driver, link, job_count):
+
+        # Open the Link in LinkedIn
         driver.get(link)
         driver.implicitly_wait(10)
 
-        for i in range(0,3):
+        # Scroll Down the Page
+        for i in range(0,job_count):
             driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
-            time.sleep(5)
+            driver.implicitly_wait(5)
+
+            # Click on See More Jobs Button if Present
             try:
                 x = driver.find_element(by=By.CSS_SELECTOR, value="button[aria-label='See more jobs']").click()
-                time.sleep(3)
+                driver.implicitly_wait(5)
             except:
                 pass
 
 
-    def company_name(driver):
-
-        company = driver.find_elements(by=By.CSS_SELECTOR, value='h4[class="base-search-card__subtitle"]')
+    def job_title_filter(scrap_job_title, user_job_title_input):
+        
+        # User Job Title Convert Lower Case and Split into List
+        user_input = []
+        for i in [i.lower() for i in user_job_title_input]:
+            user_input.extend(i.split())
 
-        company_name = []
+        # scraped Job Title Convert Lower Case and Split into List
+        scrap_title = [i.lower() for i in scrap_job_title.split()]
 
-        for i in company:
-            company_name.append(i.text)
+        # Identify Same Words in Both Lists
+        matched_words = list(set(user_input).intersection(set(scrap_title)))
 
-        return company_name
+        # Return Job Title if there are more than 1 matched word else return NaN
+        return scrap_job_title if len(matched_words) > 1 else np.nan
 
 
-    def company_location(driver):
-        
-        location = driver.find_elements(by=By.CSS_SELECTOR, value='span[class="job-search-card__location"]')
+    def scrap_company_data(driver, job_title_input, job_count):
 
-        company_location = []
+        # scraping the Company Data
+        company = driver.find_elements(by=By.CSS_SELECTOR, value='h4[class="base-search-card__subtitle"]')
+        company_name = [i.text for i in company]
 
-        for i in location:
-            company_location.append(i.text)
-        
-        return company_location
-    
+        location = driver.find_elements(by=By.CSS_SELECTOR, value='span[class="job-search-card__location"]')
+        company_location = [i.text for i in location]
 
-    def job_title(driver):
-                
         title = driver.find_elements(by=By.CSS_SELECTOR, value='h3[class="base-search-card__title"]')
-
-        job_title = []
-        
-        for i in title:
-            job_title.append(i.text)
-        
-        return job_title
-
-
-    def job_url(driver):
+        job_title = [i.text for i in title]
 
         url = driver.find_elements(by=By.XPATH, value='//a[contains(@href, "/jobs/")]')
-        
-        url_list = [i.get_attribute('href') for i in url]
-        
-        job_url = []
-        
-        for url in url_list:
-                job_url.append(url)
-        
-        return job_url
-
-
-    def job_title_filter(x, user_job_title):
-
-        s = [i.lower() for i in user_job_title]
-        suggestion = []
-        for i in s:
-            suggestion.extend(i.split())
-
-        s = x.split()
-        a = [i.lower() for i in s]
-
-        intersection = list(set(suggestion).intersection(set(a)))
-        return x if len(intersection) > 1 else np.nan
+        website_url = [i.get_attribute('href') for i in url]
 
+        # combine the all data to single dataframe
+        df = pd.DataFrame(company_name, columns=['Company Name'])
+        df['Job Title'] = pd.DataFrame(job_title)
+        df['Location'] = pd.DataFrame(company_location)
+        df['Website URL'] = pd.DataFrame(website_url)
 
-    def get_description(driver, link):
+        # Return Job Title if there are more than 1 matched word else return NaN
+        df['Job Title'] = df['Job Title'].apply(lambda x: linkedin_scraper.job_title_filter(x, job_title_input))
 
-        driver.get(link)
-        time.sleep(3)
+        # Drop Null Values and Reset Index
+        df = df.dropna()
+        df.reset_index(drop=True, inplace=True)
 
-        driver.find_element(by=By.CSS_SELECTOR, 
-                            value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
-        time.sleep(2)
+        # Filter Job Title Based on User Input
+        df = df.iloc[:job_count, :]
 
-        description = driver.find_elements(by=By.CSS_SELECTOR, 
-                                           value='div[class="show-more-less-html__markup relative overflow-hidden"]')
-        driver.implicitly_wait(4)
+        return df 
         
-        for j in description:
-            return j.text
-
-
-    def data_scrap(driver, user_job_title):
 
-        # combine the all data to single dataframe
-        df = pd.DataFrame(linkedin_scrap.company_name(driver), columns=['Company Name'])
-        df['Job Title'] = pd.DataFrame(linkedin_scrap.job_title(driver))
-        df['Location'] = pd.DataFrame(linkedin_scrap.company_location(driver))
-        df['Website URL'] = pd.DataFrame(linkedin_scrap.job_url(driver))
-
-        # job title filter based on user input
-        df['Job Title'] = df['Job Title'].apply(lambda x: linkedin_scrap.job_title_filter(x, user_job_title))
-        df = df.dropna()
-        df.reset_index(drop=True, inplace=True)
-        df = df.iloc[:10, :]
-
-        # make a list after filter
+    def scrap_job_description(driver, df):
+        
+        # Get URL into List
         website_url = df['Website URL'].tolist()
-
-        # add job description in df
+        
+        # Scrap the Job Description
         job_description = []
-
         for i in range(0, len(website_url)):
-            link = website_url[i]
-            data = linkedin_scrap.get_description(driver, link)
-            if data is not None and len(data.strip()) > 0:
+            # Open the URL
+            driver.get(website_url[i])
+            driver.implicitly_wait(10)
+            time.sleep(1)
+
+            try:
+                # Click on Show More Button
+                driver.find_element(by=By.CSS_SELECTOR, value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
+                driver.implicitly_wait(10)
+                time.sleep(1)
+
+            except NoSuchElementException:
+                # Open the URL
+                driver.get(website_url[i])
+                driver.implicitly_wait(10)
+                time.sleep(1)
+
+                # Click on Show More Button
+                driver.find_element(by=By.CSS_SELECTOR, value='button[data-tracking-control-name="public_jobs_show-more-html-btn"]').click()
+                driver.implicitly_wait(10)
+                time.sleep(1)
+
+            # Get Job Description
+            description = driver.find_elements(by=By.CSS_SELECTOR, value='div[class="show-more-less-html__markup relative overflow-hidden"]')
+            driver.implicitly_wait(10)
+            data = [i.text for i in description][0]
+
+            if len(data.strip()) > 0:
                 job_description.append(data)
             else:
                 job_description.append('Description Not Available')
 
+
+        # Add Job Description in Dataframe
         df['Job Description'] = pd.DataFrame(job_description, columns=['Description'])
         df = df.dropna()
         df.reset_index(drop=True, inplace=True)
         return df
 
 
-    def main(user_job_title):
+    def display_data_userinterface(df_final):
 
-        options = webdriver.ChromeOptions()
-        options.add_argument('--headless')
-        options.add_argument('--no-sandbox')
-        options.add_argument('--disable-dev-shm-usage')
+        # Display the Data in User Interface
+        add_vertical_space(1)
+        for i in range(0, len(df_final)):
 
-        driver = webdriver.Chrome(options=options)
-        driver.maximize_window()
+            st.write(f"Company Name : {df_final.iloc[i,0]}")
+            st.write(f"Job Title    : {df_final.iloc[i,1]}")
+            st.write(f"Location     : {df_final.iloc[i,2]}")
+            st.write(f"Website URL  : {df_final.iloc[i,3]}")
+
+            with st.expander(label='Job Desription'):
+                st.write(df_final.iloc[i, 4])
+            add_vertical_space(3)
+
+
+    def main():
+        
+        # Initially set driver to None
+        # driver = None
+        
+        # try:
+            job_title_input, job_count, submit = linkedin_scraper.get_userinput()
+            add_vertical_space(2)
 
-        linkedin_scrap.linkedin_open_scrolldown(driver, user_job_title)
+            if submit:
+                if job_title_input != '':
+                    
+                    with st.spinner('Webdriver Setup Initializing...'):
+                        driver = linkedin_scraper.webdriver_setup()
+                                       
+                    with st.spinner('Build URL and Open Link...'):
+
+                        # build URL based on User Job Title Input
+                        link = linkedin_scraper.build_url(job_title_input)
+
+                        # Open the Link in LinkedIn and Scroll Down the Page
+                        linkedin_scraper.link_open_scrolldown(driver, link, job_count)
+
+                    with st.spinner('scraping Company Data...'):
+                        df = linkedin_scraper.scrap_company_data(driver, job_title_input, job_count)
+                    
+                    with st.spinner('Scraping Job Description Data...'):
+                        df_final = linkedin_scraper. scrap_job_description(driver, df)
+                    
+                    # Display the Data in User Interface
+                    linkedin_scraper.display_data_userinterface(df_final)
 
-        final_df = linkedin_scrap.data_scrap(driver, user_job_title)
-        driver.quit()
+                
+                # If User Click Submit Button and Job Title is Empty
+                elif job_title_input == '':
+                    st.markdown(f'<h5 style="text-align: center;color: orange;">Job Title is Empty</h5>', 
+                                unsafe_allow_html=True)
+
+        # except Exception as e:
+        #     add_vertical_space(2)
+        #     st.markdown(f'<h5 style="text-align: center;color: orange;">{e}</h5>', unsafe_allow_html=True)
+        
+        # finally:
+        #     if driver:
+        #         driver.quit()
 
-        return final_df
 
 
+# Streamlit Configuration Setup
 streamlit_config()
 add_vertical_space(1)
 
@@ -310,9 +384,8 @@ if option == 'Summary':
             st.write(result_summary)
 
     except Exception as e:
-        col1, col2 = st.columns(2)
-        with col1:
-            st.warning(e)
+        add_vertical_space(2)
+        st.markdown(f'<h5 style="text-align: center;color: orange;">{e}</h5>', unsafe_allow_html=True)
 
 
 elif option == 'Strength':
@@ -337,9 +410,8 @@ elif option == 'Strength':
             st.write(result_strength)
 
     except Exception as e:
-        col1, col2 = st.columns(2)
-        with col1:
-            st.warning(e)
+        add_vertical_space(2)
+        st.markdown(f'<h5 style="text-align: center;color: orange;">{e}</h5>', unsafe_allow_html=True)
 
 
 elif option == 'Weakness':
@@ -364,9 +436,8 @@ elif option == 'Weakness':
             st.write(result_weakness)
 
     except Exception as e:
-        col1, col2 = st.columns(2)
-        with col1:
-            st.warning(e)
+        add_vertical_space(2)
+        st.markdown(f'<h5 style="text-align: center;color: orange;">{e}</h5>', unsafe_allow_html=True)
 
 
 elif option == 'Job Titles':
@@ -390,47 +461,14 @@ elif option == 'Job Titles':
             st.write(result_suggestion)
 
     except Exception as e:
-        col1, col2 = st.columns(2)
-        with col1:
-            st.warning(e)
+        add_vertical_space(2)
+        st.markdown(f'<h5 style="text-align: center;color: orange;">{e}</h5>', unsafe_allow_html=True)
 
 
 elif option == 'Linkedin Jobs':
 
-    try:        
-        # get user input of job title
-        user_input_job_title = st.text_input(label='Enter Job Titles (with comma separated):')
-        submit = st.button('Submit')
-
-        if submit and len(user_input_job_title) > 0:
-
-            user_job_title = user_input_job_title.split(',')
-
-            df = linkedin_scrap.main(user_job_title)
-
-            l = len(df['Company Name'])
-            for i in range(0, l):
-                st.write(f"Company Name : {df.iloc[i,0]}")
-                st.write(f"Job Title    : {df.iloc[i,1]}")
-                st.write(f"Location     : {df.iloc[i,2]}")
-                st.write(f"Website URL  : {df.iloc[i,3]}")
-                with st.expander(label='Job Desription'):
-                    st.write(df.iloc[i, 4])
-                st.write('')
-                st.write('')
-
-        elif submit and len(user_input_job_title) == 0:
-            col1, col2 = st.columns(2)
-            with col1:
-                st.info('Please Enter the Job Titles')
-
-    except:
-        st.write('')
-        st.info("This feature is currently not working in the deployed Streamlit application due to a 'selenium.common.exceptions.WebDriverException' error.")
-        st.write('')
-
-        st.write(
-            "Please use the local Streamlit application for a smooth experience: [http://localhost:8501](http://localhost:8501)")
+    add_vertical_space(2)
+    linkedin_scraper.main()
 
 
 elif option == 'Exit':