Update app.py

app.py

@@ -25,6 +25,7 @@ from spacy.language import Language
 import google_auth_oauthlib.flow
 import googleapiclient.discovery
 import google.auth.transport.requests
+from async_timeout import timeout as async_timeout
 # -------------------- Logging Setup --------------------
 logging.basicConfig(
     filename='advanced_download_log.txt',
@@ -447,7 +448,60 @@ class DownloadManager:
             logger.error(f"Error downloading {file_url}: {e}")
             return None
 
-    async def deep_search(self, url, custom_ext_list=None, sublink_limit=100):
+    async def search_bing(self):
+        if not self.query:
+            return [], []
+            
+        search_query = self.query
+        if "filetype:pdf" not in search_query.lower():
+            search_query += " filetype:pdf"
+            
+        search_url = f"https://www.bing.com/search?q={search_query}&count={self.num_results}"
+        
+        try:
+            await self.page.goto(search_url, timeout=30000)
+            await self.page.wait_for_selector('li.b_algo', timeout=30000)
+            
+            results = []
+            elements = await self.page.query_selector_all('li.b_algo')
+            
+            for element in elements:
+                link = await element.query_selector('h2 a')
+                if link:
+                    url = await link.get_attribute('href')
+                    if url:
+                        results.append(url)
+                        
+            return results[:self.num_results]
+            
+        except Exception as e:
+            logger.error(f"Bing search error: {e}")
+            return []
+
+    async def get_sublinks(self, url, limit=100):
+        try:
+            await self.page.goto(url, timeout=30000)
+            content = await self.page.content()
+            soup = BeautifulSoup(content, 'html.parser')
+            
+            parsed_base = urlparse(url)
+            base_url = f"{parsed_base.scheme}://{parsed_base.netloc}"
+            
+            links = set()
+            for a in soup.find_all('a', href=True):
+                href = a['href'].strip()
+                if href.startswith('http'):
+                    links.add(href)
+                elif href.startswith('/'):
+                    links.add(f"{base_url}{href}")
+                    
+            return list(links)[:limit]
+            
+        except Exception as e:
+            logger.error(f"Error getting sublinks: {e}")
+            return []
+
+    async def deep_search(self, url, custom_ext_list=None, sublink_limit=100, timeout=30):
         if not custom_ext_list:
             custom_ext_list = []
             
@@ -464,7 +518,7 @@ class DownloadManager:
             
             # Get and search sublinks
             progress_text.text("Getting sublinks...")
-            sublinks = await self.get_sublinks(url, sublink_limit)
+            sublinks = await self.get_sublinks(url, limit=sublink_limit)
             total_links = len(sublinks)
             
             progress_text.text(f"Found {total_links} sublinks to process")
@@ -474,20 +528,39 @@ class DownloadManager:
                 return main_files
                 
             # Process sublinks
-            all_files = main_files
+            all_files = main_files.copy()
             
-            for i, sublink in enumerate(sublinks, 1):
-                progress = i/total_links
-                progress_text.text(f"Processing sublink {i}/{total_links}: {sublink}")
-                progress_bar.progress(progress)
-                
-                sub_files = await self.extract_downloadable_files(sublink, custom_ext_list)
+            # Create semaphore for concurrent processing
+            sem = asyncio.Semaphore(10)
+            
+            async def process_sublink(sublink, index):
+                async with sem:
+                    try:
+                        progress = index/total_links
+                        progress_text.text(f"Processing sublink {index}/{total_links}: {sublink}")
+                        progress_bar.progress(progress)
+                        
+                        # Set timeout for this sublink
+                        async with async_timeout.timeout(timeout):
+                            sub_files = await self.extract_downloadable_files(sublink, custom_ext_list)
+                            return sub_files
+                    except asyncio.TimeoutError:
+                        logger.warning(f"Timeout processing sublink: {sublink}")
+                        return []
+                    except Exception as e:
+                        logger.error(f"Error processing sublink {sublink}: {e}")
+                        return []
+            
+            # Process sublinks with concurrent tasks
+            tasks = [process_sublink(sublink, i+1) for i, sublink in enumerate(sublinks)]
+            sub_results = await asyncio.gather(*tasks)
+            
+            # Combine all results
+            for sub_files in sub_results:
                 all_files.extend(sub_files)
-                
-                # Update count in real-time
                 file_count_text.text(f"Found {len(all_files)} total files")
-                
-            # Make results unique
+            
+            # Make results unique based on URLs
             seen_urls = set()
             unique_files = []
             
@@ -501,41 +574,25 @@ class DownloadManager:
             file_count_text.text(f"Found {final_count} unique files")
             progress_bar.progress(1.0)
             
+            # Sort files by name for consistency
+            unique_files.sort(key=lambda x: x['filename'].lower())
+            
             return unique_files
             
         except Exception as e:
             logger.error(f"Deep search error: {e}")
             progress_text.text(f"Error during deep search: {str(e)}")
             return []
+            
         finally:
             # Clean up progress indicators after a delay
             await asyncio.sleep(2)
-            if not st.session_state.get('keep_progress', False):
+            try:
                 progress_text.empty()
                 progress_bar.empty()
-
-    async def get_sublinks(self, url, limit=100):
-        try:
-            await self.page.goto(url, timeout=30000)
-            content = await self.page.content()
-            soup = BeautifulSoup(content, 'html.parser')
-            
-            parsed_base = urlparse(url)
-            base_url = f"{parsed_base.scheme}://{parsed_base.netloc}"
-            
-            links = set()
-            for a in soup.find_all('a', href=True):
-                href = a['href'].strip()
-                if href.startswith('http'):
-                    links.add(href)
-                elif href.startswith('/'):
-                    links.add(f"{base_url}{href}")
-                    
-            return list(links)[:limit]
-            
-        except Exception as e:
-            logger.error(f"Error getting sublinks: {e}")
-            return []
+                file_count_text.empty()
+            except:
+                pass
 def main():
     if 'initialized' not in st.session_state:
         st.session_state.initialized = True
@@ -547,11 +604,7 @@ def main():
     st.title("Advanced File Downloader")
     
     # Sidebar settings
-    with st.sidebar:
-        st.header("Settings")
-        mode = st.radio("Select Mode", ["Manual URL", "Bing Search", "PDF Summarizer"])
-        
-        with st.expander("Advanced Options"):
+    with st.expander("Advanced Options"):
             custom_extensions = st.text_input(
                 "Custom File Extensions",
                 placeholder=".csv, .txt, .epub"
@@ -561,8 +614,17 @@ def main():
                 min_value=1,
                 max_value=10000,
                 value=100,
+                step=50,
                 help="Maximum number of sublinks to process from the main page"
             )
+            sublink_timeout = st.number_input(
+                "Search Timeout (seconds per sublink)",
+                min_value=1,
+                max_value=3000,
+                value=30,
+                step=5,
+                help="Maximum time to spend searching each sublink"
+            )
             use_proxy = st.checkbox("Use Proxy")
             proxy = st.text_input("Proxy URL", placeholder="http://proxy:port")