Update app.py

app.py

@@ -8,6 +8,7 @@ import requests
 import time
 from bs4 import BeautifulSoup
 import re
+import concurrent.futures
 
 def get_seo_powersuite_data(domains, api_key):
     url_domain_inlink_rank = "https://api.seopowersuite.com/backlinks/v1.0/get-domain-inlink-rank"
@@ -89,11 +90,14 @@ def find_sitemap(url):
         if robots_response.status_code == 200:
             for line in robots_response.text.split("\n"):
                 if line.startswith("Sitemap:"):
-                    return line.split(":", 1)[1].strip()
+                    sitemap_url = line.split(":", 1)[1].strip()
+                    if "post" in sitemap_url.lower() or "blog" in sitemap_url.lower():
+                        return sitemap_url
     except requests.exceptions.RequestException:
         pass
         
     sitemap_urls = [
+        "/post-sitemap.xml", "/blog-sitemap.xml", "/sitemap-posts.xml",
         "/sitemap.xml", "/wp-sitemap.xml", "/?sitemap=1", "/sitemap_index/xml",
         "/sitemap-index.xml", "/sitemap.php", "/sitemap.txt", "/sitemap.xml.gz",
         "/sitemap/", "/sitemap/sitemap.xml", "/sitemapindex.xml", "/sitemap/index.xml", "/sitemap1.xml"
@@ -109,12 +113,14 @@ def find_sitemap(url):
             
     return None
 
-def crawl_posts(df, page_count):
+def crawl_posts(df, page_count, url, concurrent_scrapes):
     crawl_results = []
-    for i, row in df.head(page_count).iterrows():
-        url = row['loc']
+    crawl_status = st.empty()
+    
+    def crawl_page(row):
+        page_url = row['loc']
         try:
-            response = requests.get(url)
+            response = requests.get(page_url)
             if response.status_code == 200:
                 html = response.text
                 soup = BeautifulSoup(html, 'html.parser')
@@ -126,14 +132,29 @@ def crawl_posts(df, page_count):
                     link_text = a.text.strip()
                     link_nofollow = 'nofollow' in a.get('rel', [])
                     links.append({'url': link_url, 'text': link_text, 'nofollow': link_nofollow})
-                crawl_results.append({
-                    'url': url,
+                return {
+                    'url': page_url,  # Use page_url instead of url
                     'title': title,
                     'meta_desc': meta_desc,
                     'links': links
-                })
+                }
         except requests.exceptions.RequestException:
-            pass
+            return None
+        
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = []
+        for i in range(0, page_count, concurrent_scrapes):
+            batch_df = df.iloc[i:i+concurrent_scrapes]
+            batch_futures = [executor.submit(crawl_page, row) for _, row in batch_df.iterrows()]
+            futures.extend(batch_futures)
+            
+        for i, future in enumerate(concurrent.futures.as_completed(futures)):
+            result = future.result()
+            if result is not None:
+                crawl_results.append(result)
+            crawl_status.text(f"Crawling {url} - Page {i+1}/{page_count}")
+            
+    crawl_status.empty()
     return pd.DataFrame(crawl_results)
 
 def download_csv(df, filename):
@@ -146,7 +167,8 @@ def main():
     st.title("Website Crawler")
     
     urls = st.text_area("Enter the website URLs (one per line):", value="")
-    page_count = st.number_input("Enter the number of pages to crawl:", value=2000, min_value=1, step=1)
+    page_count = st.number_input("Enter the number of pages to crawl:", value=1000, min_value=1, step=1)
+    concurrent_scrapes = st.number_input("Enter the number of concurrent scrapes:", value=20, min_value=1, step=1)
     
     col1, col2 = st.columns(2)
     with col1:
@@ -167,16 +189,21 @@ def main():
                 all_unique_outbound_links_df = pd.DataFrame()
                 all_final_df = pd.DataFrame()
                 all_analysis_df = pd.DataFrame()
-                
+                all_crawled_pages_df = pd.DataFrame()
+                #
                 for url in url_list:
                     with st.spinner(f"Finding sitemap for {url}..."):
                         sitemap_url = find_sitemap(url)
                         if sitemap_url:
                             with st.spinner(f"Crawling {url}..."):
                                 sitemap_df = adv.sitemap_to_df(sitemap_url)
-                                crawl_results = crawl_posts(sitemap_df, page_count)
+                                sitemap_df = sitemap_df.sort_values(by="lastmod", ascending=False)  # Sort by lastmod in descending order
+                                crawl_results = crawl_posts(sitemap_df, page_count, url, concurrent_scrapes)
                                 
                                 if not crawl_results.empty:
+                                    crawled_pages_df = pd.DataFrame({'Originating Domain': url, 'Crawled Page': crawl_results['url']})
+                                    all_crawled_pages_df = pd.concat([all_crawled_pages_df, crawled_pages_df], ignore_index=True)
+                                    
                                     link_df = pd.DataFrame(crawl_results['links'].explode().tolist())
                                     link_df = link_df[~link_df['url'].str.startswith(('/','#'))]
                                     link_df['internal'] = link_df['url'].apply(lambda x: extract_hostname(url) in extract_hostname(x))
@@ -276,6 +303,12 @@ def main():
                         else:
                             st.warning(f"Sitemap not found for {url}.")
                             
+                st.subheader("Crawled Pages")
+                if download_links:
+                    st.markdown(download_csv(all_crawled_pages_df, "Crawled Pages"), unsafe_allow_html=True)
+                else:
+                    st.write(all_crawled_pages_df)
+                    
                 st.subheader("Outbound Links")
                 if download_links:
                     st.markdown(download_csv(all_link_df, "Outbound Links"), unsafe_allow_html=True)