Update app.py
Browse files
app.py
CHANGED
|
@@ -40,7 +40,11 @@ def extract_links_and_text(base_url, max_depth=1, visited=None):
|
|
| 40 |
print(f"🔗 Crawling: {base_url}")
|
| 41 |
try:
|
| 42 |
headers = {
|
| 43 |
-
"User-Agent": "Mozilla/5.0 (
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
}
|
| 45 |
response = requests.get(base_url, headers=headers, timeout=10)
|
| 46 |
response.raise_for_status()
|
|
|
|
| 40 |
print(f"🔗 Crawling: {base_url}")
|
| 41 |
try:
|
| 42 |
headers = {
|
| 43 |
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",
|
| 44 |
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
| 45 |
+
"Accept-Language": "en-US,en;q=0.9",
|
| 46 |
+
"Referer": base_url,
|
| 47 |
+
"Connection": "keep-alive",
|
| 48 |
}
|
| 49 |
response = requests.get(base_url, headers=headers, timeout=10)
|
| 50 |
response.raise_for_status()
|