chore

Signed-off-by: Suvaditya Mukherjee <[email protected]>

app.py

@@ -189,7 +189,7 @@ if __name__ == "__main__":
                 "response_time": time.time() - start_time,
                 "gpu_memory_used": torch.cuda.memory_allocated() if torch.cuda.is_available() else 0,
                 "cpu_memory": psutil.Process().memory_info().rss,
-                "gpu_utilization": torch.cuda.utilization() if torch.cuda.is_available() else 0
+                # "gpu_utilization": torch.cuda.utilization() if torch.cuda.is_available() else 0
             }
         }
         wandb.log(metrics)

utils.py

@@ -138,6 +138,10 @@ def scrape_website(start_url, delay=1):
                 links.append(url)
         return links
     
+    headers = {
+        'User-Agent': 'Mozilla/5.0'
+    }
+    
     # Main scraping loop
     while queue:
         url = queue.popleft()
@@ -146,7 +150,7 @@ def scrape_website(start_url, delay=1):
             
         try:
             print(f"Scraping: {url}")
-            response = requests.get(url, timeout=10)
+            response = requests.get(url, headers=headers, timeout=10)
             response.raise_for_status()
             
             soup = BeautifulSoup(response.text, 'html.parser')