import advertools as adv import streamlit as st import tempfile import pandas as pd from urllib.parse import urlparse import base64 import requests import time from bs4 import BeautifulSoup import re import concurrent.futures def get_seo_powersuite_data(domains, api_key): url_domain_inlink_rank = "https://api.seopowersuite.com/backlinks/v1.0/get-domain-inlink-rank" url_refdomains_count = "https://api.seopowersuite.com/backlinks/v1.0/get-refdomains-count" headers = {"Content-Type": "application/json"} results = [] for i in range(0, len(domains), 100): batch_domains = domains[i:i+100] # Get domain inlink rank start_time = time.time() payload_domain_inlink_rank = {"target": list(batch_domains)} params_domain_inlink_rank = {"apikey": api_key, "output": "json"} response_domain_inlink_rank = requests.post(url_domain_inlink_rank, json=payload_domain_inlink_rank, headers=headers, params=params_domain_inlink_rank) duration = time.time() - start_time print(f"get-domain-inlink-rank API call for {len(batch_domains)} domains took {duration:.2f} seconds") if response_domain_inlink_rank.status_code == 200: data_domain_inlink_rank = response_domain_inlink_rank.json() domain_inlink_rank_dict = {page["url"]: page["domain_inlink_rank"] for page in data_domain_inlink_rank["pages"]} else: st.error(f"Error fetching domain inlink rank data from SEO PowerSuite API: {response_domain_inlink_rank.status_code}") st.error("Error Response:") st.write(response_domain_inlink_rank.text) return None # Get refdomains count start_time = time.time() payload_refdomains_count = {"target": list(batch_domains), "mode": "domain"} params_refdomains_count = {"apikey": api_key, "output": "json"} response_refdomains_count = requests.post(url_refdomains_count, json=payload_refdomains_count, headers=headers, params=params_refdomains_count) duration = time.time() - start_time print(f"get-refdomains-count API call for {len(batch_domains)} domains took {duration:.2f} seconds") if response_refdomains_count.status_code == 200: data_refdomains_count = response_refdomains_count.json() for metric in data_refdomains_count["metrics"]: result = { "target": metric["target"], "domain_inlink_rank": domain_inlink_rank_dict.get(metric["target"], None), "refdomains": metric["refdomains"] } results.append(result) else: st.error(f"Error fetching refdomains count data from SEO PowerSuite API: {response_refdomains_count.status_code}") st.error("Error Response:") st.write(response_refdomains_count.text) return None return pd.DataFrame(results) def get_peter_lowe_domains(): url = "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&mimetype=plaintext" response = requests.get(url) lines = response.text.split('\n') domains = [line.strip('|^') for line in lines if line.startswith('||')] return set(domains) def extract_hostname(url): return urlparse(url).netloc def remove_subdomain(domain): parts = domain.split('.') if len(parts) > 2: return '.'.join(parts[-2:]) return domain def domain_matches_blacklist(domain, regex_patterns): for pattern in regex_patterns: if re.search(pattern, domain, re.IGNORECASE): return 'Yes' return 'No' def find_sitemap(url): robots_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}/robots.txt" try: robots_response = requests.get(robots_url) if robots_response.status_code == 200: for line in robots_response.text.split("\n"): if line.startswith("Sitemap:"): sitemap_url = line.split(":", 1)[1].strip() if "post" in sitemap_url.lower() or "blog" in sitemap_url.lower(): return sitemap_url except requests.exceptions.RequestException: pass sitemap_urls = [ "/post-sitemap.xml", "/blog-sitemap.xml", "/sitemap-posts.xml", "/sitemap.xml", "/wp-sitemap.xml", "/?sitemap=1", "/sitemap_index/xml", "/sitemap-index.xml", "/sitemap.php", "/sitemap.txt", "/sitemap.xml.gz", "/sitemap/", "/sitemap/sitemap.xml", "/sitemapindex.xml", "/sitemap/index.xml", "/sitemap1.xml" ] for sitemap_url in sitemap_urls: try: sitemap_response = requests.get(f"{urlparse(url).scheme}://{urlparse(url).netloc}{sitemap_url}") if sitemap_response.status_code == 200: return f"{urlparse(url).scheme}://{urlparse(url).netloc}{sitemap_url}" except requests.exceptions.RequestException: pass return None def crawl_posts(df, page_count, url, concurrent_scrapes): crawl_results = [] crawl_status = st.empty() def crawl_page(row): page_url = row['loc'] try: response = requests.get(page_url) if response.status_code == 200: html = response.text soup = BeautifulSoup(html, 'html.parser') title = soup.title.text if soup.title else '' meta_desc = soup.find('meta', attrs={'name': 'description'})['content'] if soup.find('meta', attrs={'name': 'description'}) else '' links = [] for a in soup.find_all('a', href=True): link_url = a['href'] link_text = a.text.strip() link_nofollow = 'nofollow' in a.get('rel', []) links.append({'url': link_url, 'text': link_text, 'nofollow': link_nofollow}) return { 'url': page_url, # Use page_url instead of url 'title': title, 'meta_desc': meta_desc, 'links': links } except requests.exceptions.RequestException: return None with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for i in range(0, page_count, concurrent_scrapes): batch_df = df.iloc[i:i+concurrent_scrapes] batch_futures = [executor.submit(crawl_page, row) for _, row in batch_df.iterrows()] futures.extend(batch_futures) for i, future in enumerate(concurrent.futures.as_completed(futures)): result = future.result() if result is not None: crawl_results.append(result) crawl_status.text(f"Crawling {url} - Page {i+1}/{page_count}") crawl_status.empty() return pd.DataFrame(crawl_results) def download_csv(df, filename): csv = df.to_csv(index=False) b64 = base64.b64encode(csv.encode()).decode() href = f'Download {filename} CSV' return href def main(): st.title("Website Crawler") urls = st.text_area("Enter the website URLs (one per line):", value="") page_count = st.number_input("Enter the number of pages to crawl:", value=1000, min_value=1, step=1) concurrent_scrapes = st.number_input("Enter the number of concurrent scrapes:", value=20, min_value=1, step=1) col1, col2 = st.columns(2) with col1: domain_filter_regex_input = st.text_area("Filter out Unique Outbound Domains:", help="This uses a regex filter to find domains in the unique outbound domains list. Enter one regex per line.", value="instagram\nfacebook\ntwitter\nlinkedin\nsnapchat\ntiktok\nreddit\npinterest\namazon\ncdn\nyoutube\nyoutu.be") with col2: domain_match_regex_input = st.text_area("Domain Blacklist:", help="This uses a regex filter to match domains in the Unique Outbound Domains to the blacklist entered. Enter one regex per line.", value="xyz\ncasino\ncbd\nessay") use_seo_powersuite = st.checkbox("Use SEO PowerSuite") api_key = None if use_seo_powersuite: api_key = st.text_input("Enter the SEO PowerSuite API key:", type="password") download_links = st.checkbox("Show Download Links") if st.button("Crawl"): if urls: url_list = [url.strip() for url in urls.split('\n') if url.strip()] if url_list: all_link_df = pd.DataFrame() all_unique_outbound_links_df = pd.DataFrame() all_final_df = pd.DataFrame() all_analysis_df = pd.DataFrame() all_crawled_pages_df = pd.DataFrame() # for url in url_list: with st.spinner(f"Finding sitemap for {url}..."): sitemap_url = find_sitemap(url) if sitemap_url: with st.spinner(f"Crawling {url}..."): sitemap_df = adv.sitemap_to_df(sitemap_url) sitemap_df = sitemap_df.sort_values(by="lastmod", ascending=False) # Sort by lastmod in descending order crawl_results = crawl_posts(sitemap_df, page_count, url, concurrent_scrapes) if not crawl_results.empty: crawled_pages_df = pd.DataFrame({'Originating Domain': url, 'Crawled Page': crawl_results['url']}) all_crawled_pages_df = pd.concat([all_crawled_pages_df, crawled_pages_df], ignore_index=True) link_df = pd.DataFrame(crawl_results['links'].explode().tolist()) link_df = link_df[~link_df['url'].str.startswith(('/','#'))] link_df['internal'] = link_df['url'].apply(lambda x: extract_hostname(url) in extract_hostname(x)) link_df = link_df[link_df['internal'] == False] # Filter out internal links link_df.insert(0, 'Originating Domain', url) # Add 'Originating Domain' column link_df = link_df[['Originating Domain', 'url', 'text', 'nofollow']] # Remove the 'internal' column outbound_links_df = link_df.copy() # Create a copy of link_df for outbound links unique_links_df = link_df['url'].value_counts().reset_index() unique_links_df = unique_links_df[~unique_links_df['url'].str.startswith(('/','#'))] unique_links_df.columns = ['Link', 'Count'] unique_links_df.insert(0, 'Originating Domain', url) unique_outbound_links_df = outbound_links_df['url'].value_counts().reset_index() unique_outbound_links_df = unique_outbound_links_df[~unique_outbound_links_df['url'].str.startswith(('/','#'))] unique_outbound_links_df.columns = ['Link', 'Count'] unique_outbound_links_df.insert(0, 'Originating Domain', url) outbound_links_df['url'] = outbound_links_df['url'].astype(str) domain_df = outbound_links_df['url'].apply(extract_hostname).value_counts().reset_index() domain_df.columns = ['Domain', 'Count'] domain_df = domain_df[domain_df['Domain'] != ''] peter_lowe_domains = get_peter_lowe_domains() domain_df['In Peter Lowe List'] = domain_df['Domain'].apply(lambda x: 'Yes' if remove_subdomain(x) in peter_lowe_domains else 'No') domain_df.insert(0, 'Originating Domain', url) # Determine the 'DoFollow' value for each domain domain_df['DoFollow'] = domain_df['Domain'].apply(lambda x: any(outbound_links_df[(outbound_links_df['url'].str.contains(x)) & (outbound_links_df['nofollow'] == False)])) if not domain_df.empty: if domain_filter_regex_input: domain_filter_regex_patterns = domain_filter_regex_input.split('\n') domain_filter_regex = '|'.join(domain_filter_regex_patterns) domain_df = domain_df[~domain_df['Domain'].str.contains(domain_filter_regex, case=False, regex=True)] if not domain_df.empty: if domain_match_regex_input: domain_match_regex_patterns = domain_match_regex_input.split('\n') domain_df['Blacklist'] = domain_df['Domain'].apply(lambda x: domain_matches_blacklist(x, domain_match_regex_patterns) == 'Yes') else: domain_df['Blacklist'] = False total_domains = len(domain_df) peter_lowe_percentage = round((domain_df['In Peter Lowe List'] == 'No').sum() / total_domains * 100, 2) blacklist_percentage = round((domain_df['Blacklist'] == True).sum() / total_domains * 100, 2) analysis_data = { 'Originating Domain': [url] * 2, 'Metric': ['Percentage of domains not in Peter Lowe\'s list', 'Percentage of domains in the Blacklist'], 'Value': [f"{peter_lowe_percentage}%", f"{blacklist_percentage}%"] } analysis_df = pd.DataFrame(analysis_data) if use_seo_powersuite and api_key: seo_powersuite_df = get_seo_powersuite_data(domain_df['Domain'].tolist(), api_key) if seo_powersuite_df is not None: domain_df = pd.merge(domain_df, seo_powersuite_df, left_on='Domain', right_on='target', how='left') domain_df.drop('target', axis=1, inplace=True) avg_domain_inlink_rank = round(domain_df['domain_inlink_rank'].mean(), 2) avg_domain_inlink_rank_less_than_70 = round(domain_df[domain_df['domain_inlink_rank'] < 70]['domain_inlink_rank'].mean(), 2) avg_refdomains = round(domain_df['refdomains'].mean(), 2) additional_analysis_data = { 'Originating Domain': [url] * 3, 'Metric': [ 'Average domain inlink rank', 'Average domain inlink rank (< 70)', 'Average number of refdomains' ], 'Value': [ avg_domain_inlink_rank, avg_domain_inlink_rank_less_than_70, avg_refdomains ] } analysis_df = pd.concat([analysis_df, pd.DataFrame(additional_analysis_data)], ignore_index=True) desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List', 'DoFollow', 'Blacklist', 'domain_inlink_rank', 'refdomains'] final_df = domain_df[desired_columns] else: desired_columns = ['Originating Domain', 'Domain', 'Count', 'In Peter Lowe List', 'DoFollow', 'Blacklist'] final_df = domain_df[desired_columns] else: st.warning(f"No unique outbound domains found for {url} after filtering.") else: st.warning(f"No unique outbound domains found for {url}.") all_link_df = pd.concat([all_link_df, link_df], ignore_index=True) all_unique_outbound_links_df = pd.concat([all_unique_outbound_links_df, unique_outbound_links_df], ignore_index=True) all_final_df = pd.concat([all_final_df, final_df], ignore_index=True) all_analysis_df = pd.concat([all_analysis_df, analysis_df], ignore_index=True) else: st.warning(f"No posts found in the sitemap for {url}.") else: st.warning(f"Sitemap not found for {url}.") st.subheader("Crawled Pages") if download_links: st.markdown(download_csv(all_crawled_pages_df, "Crawled Pages"), unsafe_allow_html=True) else: st.write(all_crawled_pages_df) st.subheader("Outbound Links") if download_links: st.markdown(download_csv(all_link_df, "Outbound Links"), unsafe_allow_html=True) else: st.write(all_link_df) st.subheader("Unique Outbound Links") if download_links: st.markdown(download_csv(all_unique_outbound_links_df, "Unique Outbound Links"), unsafe_allow_html=True) else: st.write(all_unique_outbound_links_df) st.subheader("Unique Outbound Domains") if download_links: st.markdown(download_csv(all_final_df, "Unique Outbound Domains"), unsafe_allow_html=True) else: st.write(all_final_df) st.subheader("Analytics") all_analysis_df = all_analysis_df.pivot(index='Originating Domain', columns='Metric', values='Value').reset_index() all_analysis_df.columns.name = None if use_seo_powersuite and api_key: numeric_columns = ['Average domain inlink rank', 'Average domain inlink rank (< 70)', 'Average number of refdomains'] all_analysis_df[numeric_columns] = all_analysis_df[numeric_columns].astype(int) if download_links: st.markdown(download_csv(all_analysis_df, "Analytics"), unsafe_allow_html=True) else: st.table(all_analysis_df) else: st.warning("Please enter at least one website URL.") else: st.warning("Please enter website URLs.") if __name__ == '__main__': main()