Update app.py

app.py

@@ -6,6 +6,8 @@ from urllib.parse import urlparse
 import base64
 import requests
 import time
+from bs4 import BeautifulSoup
+import re
 
 def get_seo_powersuite_data(domains, api_key):
     url_domain_inlink_rank = "https://api.seopowersuite.com/backlinks/v1.0/get-domain-inlink-rank"
@@ -74,23 +76,65 @@ def remove_subdomain(domain):
         return '.'.join(parts[-2:])
     return domain
 
-def crawl_website(url, exclude_url_regex, page_count, concurrent_requests, depth_limit):
-    with tempfile.NamedTemporaryFile(delete=False, suffix='.jl') as temp_file:
-        custom_settings = {
-            "CLOSESPIDER_PAGECOUNT": page_count,
-            "CONCURRENT_REQUESTS_PER_DOMAIN": concurrent_requests,
-            "DEPTH_LIMIT": depth_limit
-        }
-        if exclude_url_regex:
-            adv.crawl(url, output_file=temp_file.name, follow_links=True, exclude_url_params=True, exclude_url_regex=exclude_url_regex, custom_settings=custom_settings)
-        else:
-            adv.crawl(url, output_file=temp_file.name, follow_links=True, exclude_url_params=True, custom_settings=custom_settings)
-        crawl_df = pd.read_json(temp_file.name, lines=True)
-    crawl_df['hostname'] = crawl_df['url'].apply(extract_hostname)
-    # Select only the desired columns
-    desired_columns = ['url', 'hostname', 'title', 'meta_desc', 'status', 'links_url', 'links_text', 'links_nofollow']
-    crawl_df = crawl_df[desired_columns]
-    return crawl_df
+def domain_matches_blacklist(domain, regex_patterns):
+    for pattern in regex_patterns:
+        if re.search(pattern, domain, re.IGNORECASE):
+            return 'Yes'
+    return 'No'
+
+def find_sitemap(url):
+    robots_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}/robots.txt"
+    try:
+        robots_response = requests.get(robots_url)
+        if robots_response.status_code == 200:
+            for line in robots_response.text.split("\n"):
+                if line.startswith("Sitemap:"):
+                    return line.split(":", 1)[1].strip()
+    except requests.exceptions.RequestException:
+        pass
+        
+    sitemap_urls = [
+        "/sitemap.xml", "/wp-sitemap.xml", "/?sitemap=1", "/sitemap_index/xml",
+        "/sitemap-index.xml", "/sitemap.php", "/sitemap.txt", "/sitemap.xml.gz",
+        "/sitemap/", "/sitemap/sitemap.xml", "/sitemapindex.xml", "/sitemap/index.xml", "/sitemap1.xml"
+    ]
+    
+    for sitemap_url in sitemap_urls:
+        try:
+            sitemap_response = requests.get(f"{urlparse(url).scheme}://{urlparse(url).netloc}{sitemap_url}")
+            if sitemap_response.status_code == 200:
+                return f"{urlparse(url).scheme}://{urlparse(url).netloc}{sitemap_url}"
+        except requests.exceptions.RequestException:
+            pass
+            
+    return None
+
+def crawl_posts(df, page_count):
+    crawl_results = []
+    for i, row in df.head(page_count).iterrows():
+        url = row['loc']
+        try:
+            response = requests.get(url)
+            if response.status_code == 200:
+                html = response.text
+                soup = BeautifulSoup(html, 'html.parser')
+                title = soup.title.text if soup.title else ''
+                meta_desc = soup.find('meta', attrs={'name': 'description'})['content'] if soup.find('meta', attrs={'name': 'description'}) else ''
+                links = []
+                for a in soup.find_all('a', href=True):
+                    link_url = a['href']
+                    link_text = a.text.strip()
+                    link_nofollow = 'nofollow' in a.get('rel', [])
+                    links.append({'url': link_url, 'text': link_text, 'nofollow': link_nofollow})
+                crawl_results.append({
+                    'url': url,
+                    'title': title,
+                    'meta_desc': meta_desc,
+                    'links': links
+                })
+        except requests.exceptions.RequestException:
+            pass
+    return pd.DataFrame(crawl_results)
 
 def download_csv(df, filename):
     csv = df.to_csv(index=False)
@@ -101,183 +145,169 @@ def download_csv(df, filename):
 def main():
     st.title("Website Crawler")
     
-    domains = st.text_area("Enter the website URLs (one per line):", value="")
-    page_count = st.number_input("Enter the maximum number of pages to crawl:", value=5000, min_value=1, step=1)
-    col1, col2 = st.columns(2)
-    with col1:
-        concurrent_requests = st.number_input("Enter the number of concurrent requests per domain:", value=8, min_value=1, step=1)
-    with col2:
-        depth_limit = st.number_input("Enter the depth limit for the crawl:", value=0, min_value=0, step=1)
-        
+    urls = st.text_area("Enter the website URLs (one per line):", value="")
+    page_count = st.number_input("Enter the number of pages to crawl:", value=1000, min_value=1, step=1)
+    
     col1, col2 = st.columns(2)
     with col1:
-        exclude_url_regex_input = st.text_area("Enter exclude URL regex patterns (one per line):", value="cdn\nwp-content")
+        domain_filter_regex_input = st.text_area("Filter out Unique Outbound Domains:", help="This uses a regex filter to find domains in the unique outbound domains list. Enter one regex per line.", value="instagram\nfacebook\ntwitter\nlinkedin\nsnapchat\ntiktok\nreddit\npinterest\namazon\ncdn\nyoutube\nyoutu.be")
     with col2:
-        domain_filter_regex_input = st.text_area("Filter our unique domains with regex (one per line):", value="instagram\nfacebook\ntwitter\nlinkedin\nsnapchat\ntiktok\nreddit\npinterest\namazon\ncdn\nyoutube\nyoutu.be")
-        
-    col1, col2 = st.columns([2, 5])
-    with col1:
-        use_seo_powersuite = st.checkbox("Use SEO PowerSuite")
-    with col2:
-        api_key = None
-        if use_seo_powersuite:
-            api_key = st.text_input("Enter the SEO PowerSuite API key:", type="password", value="")
+        domain_match_regex_input = st.text_area("Domain Blacklist:", help="This uses a regex filter to match domains in the Unique Outbound Domains to the blacklist entered. Enter one regex per line.", value="xyz\ncasino\ncbd\nessay")
+    use_seo_powersuite = st.checkbox("Use SEO PowerSuite")
+    api_key = None
+    if use_seo_powersuite:
+        api_key = st.text_input("Enter the SEO PowerSuite API key:", type="password")
     download_links = st.checkbox("Show Download Links")
-        
+    
     if st.button("Crawl"):
-        if domains:
-            domain_list = [domain.strip() for domain in domains.split('\n') if domain.strip()]
-            if domain_list:
-                exclude_url_regex_patterns = exclude_url_regex_input.split('\n')
-                exclude_url_regex = '|'.join(exclude_url_regex_patterns) if exclude_url_regex_patterns else None
+        if urls:
+            url_list = [url.strip() for url in urls.split('\n') if url.strip()]
+            if url_list:
+                all_link_df = pd.DataFrame()
+                all_unique_outbound_links_df = pd.DataFrame()
+                all_final_df = pd.DataFrame()
+                all_analysis_df = pd.DataFrame()
                 
-                all_crawl_results = []
-                all_link_df = []
-                all_unique_links_df = []
-                all_unique_outbound_links_df = []
-                all_domain_df = []
-                all_final_df = []
-                all_analysis_df = []
-                
-                for domain in domain_list:
-                    with st.spinner(f"Crawling {domain}..."):
-                        crawl_results = crawl_website(domain, exclude_url_regex, page_count, concurrent_requests, depth_limit)
-                        crawl_results.insert(0, 'Originating Domain', domain)
-                        all_crawl_results.append(crawl_results)
-                        
-                        if not crawl_results.empty:
-                            link_df = adv.crawlytics.links(crawl_results, internal_url_regex=extract_hostname(domain))
-                            link_df.insert(0, 'Originating Domain', domain)
-                            all_link_df.append(link_df)
-                            
-                            unique_links_df = link_df['link'].value_counts().reset_index()
-                            unique_links_df.columns = ['Link', 'Count']
-                            unique_links_df.insert(0, 'Originating Domain', domain)
-                            all_unique_links_df.append(unique_links_df)
-                            
-                            outbound_links_df = link_df[link_df['internal'] == False]
-                            unique_outbound_links_df = outbound_links_df['link'].value_counts().reset_index()
-                            unique_outbound_links_df.columns = ['Link', 'Count']
-                            unique_outbound_links_df.insert(0, 'Originating Domain', domain)
-                            all_unique_outbound_links_df.append(unique_outbound_links_df)
-                            
-                            outbound_links_df['link'] = outbound_links_df['link'].astype(str)
-                            domain_df = outbound_links_df['link'].apply(extract_hostname).value_counts().reset_index()
-                            domain_df.columns = ['Domain', 'Count']
-                            domain_df = domain_df[domain_df['Domain'] != '']
-                            peter_lowe_domains = get_peter_lowe_domains()
-                            domain_df['In Peter Lowe List'] = domain_df['Domain'].apply(lambda x: 'Yes' if remove_subdomain(x) in peter_lowe_domains else 'No')
-                            domain_df.insert(0, 'Originating Domain', domain)
-                            all_domain_df.append(domain_df)
-                            
-                            if not domain_df.empty:
-                                if domain_filter_regex_input:
-                                    domain_filter_regex_patterns = domain_filter_regex_input.split('\n')
-                                    domain_filter_regex = '|'.join(domain_filter_regex_patterns)
-                                    domain_df = domain_df[~domain_df['Domain'].str.contains(domain_filter_regex, case=False, regex=True)]
-                                    
-                            if use_seo_powersuite and api_key:
-                                seo_powersuite_df = get_seo_powersuite_data(domain_df['Domain'].tolist(), api_key)
-                                if seo_powersuite_df is not None:
-                                    domain_df = pd.merge(domain_df, seo_powersuite_df, left_on='Domain', right_on='target', how='left')
-                                    domain_df.drop('target', axis=1, inplace=True)
+                for url in url_list:
+                    with st.spinner(f"Finding sitemap for {url}..."):
+                        sitemap_url = find_sitemap(url)
+                        if sitemap_url:
+                            with st.spinner(f"Crawling {url}..."):
+                                sitemap_df = adv.sitemap_to_df(sitemap_url)
+                                crawl_results = crawl_posts(sitemap_df, page_count)
+                                
+                                if not crawl_results.empty:
+                                    link_df = pd.DataFrame(crawl_results['links'].explode().tolist())
+                                    link_df = link_df[~link_df['url'].str.startswith(('/','#'))]
+                                    link_df['internal'] = link_df['url'].apply(lambda x: extract_hostname(url) in extract_hostname(x))
+                                    link_df = link_df[link_df['internal'] == False]  # Filter out internal links
+                                    link_df.insert(0, 'Originating Domain', url)  # Add 'Originating Domain' column
+                                    link_df = link_df[['Originating Domain', 'url', 'text', 'nofollow']]  # Remove the 'internal' column
                                     
-                                    desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List', 'domain_inlink_rank', 'refdomains']
-                                    final_df = domain_df[desired_columns]
-                                    all_final_df.append(final_df)
+                                    outbound_links_df = link_df.copy()  # Create a copy of link_df for outbound links                            
+                                    unique_links_df = link_df['url'].value_counts().reset_index()
+                                    unique_links_df = unique_links_df[~unique_links_df['url'].str.startswith(('/','#'))]
+                                    unique_links_df.columns = ['Link', 'Count']
+                                    unique_links_df.insert(0, 'Originating Domain', url)
                                     
-                                    total_domains = len(final_df)
-                                    peter_lowe_percentage = round((final_df['In Peter Lowe List'] == 'No').sum() / total_domains * 100, 2)
-                                    avg_domain_inlink_rank = round(final_df['domain_inlink_rank'].mean(), 2)
-                                    avg_domain_inlink_rank_less_than_70 = round(final_df[final_df['domain_inlink_rank'] < 70]['domain_inlink_rank'].mean(), 2)
-                                    avg_refdomains = round(final_df['refdomains'].mean(), 2)
+                                    unique_outbound_links_df = outbound_links_df['url'].value_counts().reset_index()
+                                    unique_outbound_links_df = unique_outbound_links_df[~unique_outbound_links_df['url'].str.startswith(('/','#'))]
+                                    unique_outbound_links_df.columns = ['Link', 'Count']
+                                    unique_outbound_links_df.insert(0, 'Originating Domain', url)
                                     
-                                    analysis_data = {
-                                        'Originating Domain': [domain] * 4,
-                                        'Metric': [
-                                            'Percentage of domains not in Peter Lowe\'s list',
-                                            'Average domain inlink rank',
-                                            'Average domain inlink rank (< 70)',
-                                            'Average number of refdomains'
-                                        ],
-                                        'Value': [
-                                            f"{peter_lowe_percentage}%",
-                                            avg_domain_inlink_rank,
-                                            avg_domain_inlink_rank_less_than_70,
-                                            avg_refdomains
-                                        ]
-                                    }
+                                    outbound_links_df['url'] = outbound_links_df['url'].astype(str)
+                                    domain_df = outbound_links_df['url'].apply(extract_hostname).value_counts().reset_index()
+                                    domain_df.columns = ['Domain', 'Count']
+                                    domain_df = domain_df[domain_df['Domain'] != '']
+                                    peter_lowe_domains = get_peter_lowe_domains()
+                                    domain_df['In Peter Lowe List'] = domain_df['Domain'].apply(lambda x: 'Yes' if remove_subdomain(x) in peter_lowe_domains else 'No')
+                                    domain_df.insert(0, 'Originating Domain', url)
                                     
-                                    analysis_df = pd.DataFrame(analysis_data)
-                                    all_analysis_df.append(analysis_df)
+                                    # Determine the 'DoFollow' value for each domain
+                                    domain_df['DoFollow'] = domain_df['Domain'].apply(lambda x: any(outbound_links_df[(outbound_links_df['url'].str.contains(x)) & (outbound_links_df['nofollow'] == False)]))
                                     
-                            else:
-                                desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List']
-                                final_df = domain_df[desired_columns]
-                                all_final_df.append(final_df)
-                                
-                if all_crawl_results:
-                    st.subheader("Crawl Results")
-                    combined_crawl_results = pd.concat(all_crawl_results, ignore_index=True)
-                    if download_links:
-                        st.markdown(download_csv(combined_crawl_results, "Crawl Results"), unsafe_allow_html=True)
-                    else:
-                        st.write(combined_crawl_results)
-                        
-                if all_link_df:
-                    st.subheader("All Links")
-                    combined_link_df = pd.concat(all_link_df, ignore_index=True)
-                    if download_links:
-                        st.markdown(download_csv(combined_link_df, "All Links"), unsafe_allow_html=True)
-                    else:
-                        st.write(combined_link_df)
-                        
-                if all_unique_links_df:
-                    st.subheader("Unique Links")
-                    combined_unique_links_df = pd.concat(all_unique_links_df, ignore_index=True)
-                    if download_links:
-                        st.markdown(download_csv(combined_unique_links_df, "Unique Links"), unsafe_allow_html=True)
-                    else:
-                        st.write(combined_unique_links_df)
-                        
-                if all_unique_outbound_links_df:
-                    st.subheader("Unique Outbound Links")
-                    combined_unique_outbound_links_df = pd.concat(all_unique_outbound_links_df, ignore_index=True)
-                    if download_links:
-                        st.markdown(download_csv(combined_unique_outbound_links_df, "Unique Outbound Links"), unsafe_allow_html=True)
-                    else:
-                        st.write(combined_unique_outbound_links_df)
-                        
-                if all_final_df:
-                    st.subheader("Unique Outbound Domains")
-                    combined_final_df = pd.concat(all_final_df, ignore_index=True)
-                    if download_links:
-                        st.markdown(download_csv(combined_final_df, "Unique Outbound Domains"), unsafe_allow_html=True)
-                    else:
-                        st.write(combined_final_df)
-                        
-                if all_analysis_df:
-                    st.subheader("Analytics")
-                    combined_analysis_df = pd.concat(all_analysis_df)
-                    combined_analysis_df = combined_analysis_df.pivot(index='Originating Domain', columns='Metric', values='Value').reset_index()
-                    combined_analysis_df.columns.name = None
+                                    if not domain_df.empty:
+                                        if domain_filter_regex_input:
+                                            domain_filter_regex_patterns = domain_filter_regex_input.split('\n')
+                                            domain_filter_regex = '|'.join(domain_filter_regex_patterns)
+                                            domain_df = domain_df[~domain_df['Domain'].str.contains(domain_filter_regex, case=False, regex=True)]
+                                            
+                                        if not domain_df.empty:
+                                            if domain_match_regex_input:
+                                                domain_match_regex_patterns = domain_match_regex_input.split('\n')
+                                                domain_df['Blacklist'] = domain_df['Domain'].apply(lambda x: domain_matches_blacklist(x, domain_match_regex_patterns) == 'Yes')
+                                            else:
+                                                domain_df['Blacklist'] = False
+                                                
+                                            total_domains = len(domain_df)
+                                            peter_lowe_percentage = round((domain_df['In Peter Lowe List'] == 'No').sum() / total_domains * 100, 2)
+                                            blacklist_percentage = round((domain_df['Blacklist'] == True).sum() / total_domains * 100, 2)
+                                            
+                                            analysis_data = {
+                                                'Originating Domain': [url] * 2,
+                                                'Metric': ['Percentage of domains not in Peter Lowe\'s list', 'Percentage of domains in the Blacklist'],
+                                                'Value': [f"{peter_lowe_percentage}%", f"{blacklist_percentage}%"]
+                                            }
+                                            
+                                            analysis_df = pd.DataFrame(analysis_data)
+                                            
+                                            if use_seo_powersuite and api_key:
+                                                seo_powersuite_df = get_seo_powersuite_data(domain_df['Domain'].tolist(), api_key)
+                                                if seo_powersuite_df is not None:
+                                                    domain_df = pd.merge(domain_df, seo_powersuite_df, left_on='Domain', right_on='target', how='left')
+                                                    domain_df.drop('target', axis=1, inplace=True)
+                                                    
+                                                    avg_domain_inlink_rank = round(domain_df['domain_inlink_rank'].mean(), 2)
+                                                    avg_domain_inlink_rank_less_than_70 = round(domain_df[domain_df['domain_inlink_rank'] < 70]['domain_inlink_rank'].mean(), 2)
+                                                    avg_refdomains = round(domain_df['refdomains'].mean(), 2)
+                                                    
+                                                    additional_analysis_data = {
+                                                        'Originating Domain': [url] * 3,
+                                                        'Metric': [
+                                                            'Average domain inlink rank',
+                                                            'Average domain inlink rank (< 70)',
+                                                            'Average number of refdomains'
+                                                        ],
+                                                        'Value': [
+                                                            avg_domain_inlink_rank,
+                                                            avg_domain_inlink_rank_less_than_70,
+                                                            avg_refdomains
+                                                        ]
+                                                    }
+                                                    
+                                                    analysis_df = pd.concat([analysis_df, pd.DataFrame(additional_analysis_data)], ignore_index=True)
+                                                    
+                                                    desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List', 'DoFollow', 'Blacklist', 'domain_inlink_rank', 'refdomains']
+                                                    final_df = domain_df[desired_columns]
+                                            else:
+                                                desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List', 'DoFollow', 'Blacklist']
+                                                final_df = domain_df[desired_columns]
+                                        else:
+                                            st.warning(f"No unique outbound domains found for {url} after filtering.")
+                                    else:
+                                        st.warning(f"No unique outbound domains found for {url}.")
+                                        
+                                    all_link_df = pd.concat([all_link_df, link_df], ignore_index=True)
+                                    all_unique_outbound_links_df = pd.concat([all_unique_outbound_links_df, unique_outbound_links_df], ignore_index=True)
+                                    all_final_df = pd.concat([all_final_df, final_df], ignore_index=True)
+                                    all_analysis_df = pd.concat([all_analysis_df, analysis_df], ignore_index=True)
+                                else:
+                                    st.warning(f"No posts found in the sitemap for {url}.")
+                        else:
+                            st.warning(f"Sitemap not found for {url}.")
+                            
+                st.subheader("Outbound Links")
+                if download_links:
+                    st.markdown(download_csv(all_link_df, "Outbound Links"), unsafe_allow_html=True)
+                else:
+                    st.write(all_link_df)
+                    
+                st.subheader("Unique Outbound Links")
+                if download_links:
+                    st.markdown(download_csv(all_unique_outbound_links_df, "Unique Outbound Links"), unsafe_allow_html=True)
+                else:
+                    st.write(all_unique_outbound_links_df)
+                    
+                st.subheader("Unique Outbound Domains")
+                if download_links:
+                    st.markdown(download_csv(all_final_df, "Unique Outbound Domains"), unsafe_allow_html=True)
+                else:
+                    st.write(all_final_df)
+                    
+                st.subheader("Analytics")
+                all_analysis_df = all_analysis_df.pivot(index='Originating Domain', columns='Metric', values='Value').reset_index()
+                all_analysis_df.columns.name = None
+                if use_seo_powersuite and api_key:
                     numeric_columns = ['Average domain inlink rank', 'Average domain inlink rank (< 70)', 'Average number of refdomains']
-                    combined_analysis_df[numeric_columns] = combined_analysis_df[numeric_columns].astype(int)
-                    combined_analysis_df = combined_analysis_df[[
-                        'Originating Domain',
-                        'Percentage of domains not in Peter Lowe\'s list',
-                        'Average domain inlink rank',
-                        'Average domain inlink rank (< 70)',
-                        'Average number of refdomains'
-                    ]]
-                    if download_links:
-                        st.markdown(download_csv(combined_analysis_df, "Analytics"), unsafe_allow_html=True)
-                    else:
-                        st.table(combined_analysis_df)
+                    all_analysis_df[numeric_columns] = all_analysis_df[numeric_columns].astype(int)
+                if download_links:
+                    st.markdown(download_csv(all_analysis_df, "Analytics"), unsafe_allow_html=True)
+                else:
+                    st.table(all_analysis_df)
             else:
                 st.warning("Please enter at least one website URL.")
         else:
-            st.warning("Please enter the website URLs.")
+            st.warning("Please enter website URLs.")
             
 if __name__ == '__main__':
     main()