|
import os |
|
import pytest |
|
from unittest.mock import Mock, patch |
|
from bs4 import BeautifulSoup |
|
from scrapy.http import Response, Request |
|
from backend.app.crawler import WebsiteSpider, DomainCrawler |
|
|
|
|
|
@pytest.fixture |
|
def sample_html(): |
|
return """ |
|
<html> |
|
<head><title>Test Page</title></head> |
|
<body> |
|
<main> |
|
<h1>Main Content</h1> |
|
<p>This is the main content.</p> |
|
</main> |
|
</body> |
|
</html> |
|
""" |
|
|
|
|
|
@pytest.fixture |
|
def output_dir(tmp_path): |
|
"""Create a temporary directory for test outputs""" |
|
return str(tmp_path / "test_crawled_content") |
|
|
|
|
|
def test_website_spider_initialization(): |
|
"""Test WebsiteSpider initialization with correct parameters""" |
|
start_url = "https://example.com" |
|
output_dir = "test_output" |
|
|
|
spider = WebsiteSpider(start_url=start_url, output_dir=output_dir) |
|
|
|
assert spider.start_urls == [start_url] |
|
assert spider.allowed_domains == ["example.com"] |
|
assert spider.output_dir == output_dir |
|
assert len(spider.rules) == 1 |
|
|
|
|
|
def test_parse_item_with_main_content(sample_html, output_dir): |
|
"""Test parsing a page with main content section""" |
|
start_url = "https://example.com" |
|
spider = WebsiteSpider(start_url=start_url, output_dir=output_dir) |
|
|
|
|
|
mock_response = Mock(spec=Response) |
|
mock_response.url = "https://example.com/test" |
|
mock_response.body = sample_html.encode("utf-8") |
|
|
|
|
|
spider.parse_item(mock_response) |
|
|
|
|
|
files = os.listdir(output_dir) |
|
assert len(files) == 1 |
|
|
|
with open(os.path.join(output_dir, files[0]), "r", encoding="utf-8") as f: |
|
content = f.read() |
|
assert "Test Page" in content |
|
assert "Main Content" in content |
|
assert "This is the main content" in content |
|
assert "URL: https://example.com/test" in content |
|
|
|
|
|
def test_parse_item_without_main_content(output_dir): |
|
"""Test parsing a page without main content section""" |
|
html_without_main = """ |
|
<html> |
|
<head><title>No Main Page</title></head> |
|
<body> |
|
<div>Some body content</div> |
|
</body> |
|
</html> |
|
""" |
|
|
|
start_url = "https://example.com" |
|
spider = WebsiteSpider(start_url=start_url, output_dir=output_dir) |
|
|
|
mock_response = Mock(spec=Response) |
|
mock_response.url = "https://example.com/no-main" |
|
mock_response.body = html_without_main.encode("utf-8") |
|
|
|
spider.parse_item(mock_response) |
|
|
|
files = os.listdir(output_dir) |
|
assert len(files) == 1 |
|
|
|
with open(os.path.join(output_dir, files[0]), "r", encoding="utf-8") as f: |
|
content = f.read() |
|
assert "No Main Page" in content |
|
assert "Some body content" in content |
|
|
|
|
|
def test_domain_crawler_initialization(): |
|
"""Test DomainCrawler initialization""" |
|
start_url = "https://example.com" |
|
output_dir = "test_output" |
|
|
|
crawler = DomainCrawler(start_url=start_url, output_dir=output_dir) |
|
|
|
assert crawler.start_url == start_url |
|
assert crawler.domain == "example.com" |
|
assert crawler.output_dir == output_dir |
|
assert crawler.settings.get("BOT_NAME") == "website_crawler" |
|
assert crawler.settings.get("ROBOTSTXT_OBEY") is True |
|
assert crawler.settings.get("CONCURRENT_REQUESTS") == 16 |
|
assert crawler.settings.get("DOWNLOAD_DELAY") == 1 |
|
|
|
|
|
@patch("backend.app.crawler.CrawlerProcess") |
|
def test_domain_crawler_start(mock_crawler_process): |
|
"""Test starting the domain crawler""" |
|
start_url = "https://example.com" |
|
output_dir = "test_output" |
|
|
|
crawler = DomainCrawler(start_url=start_url, output_dir=output_dir) |
|
crawler.start() |
|
|
|
|
|
mock_crawler_process.assert_called_once_with(crawler.settings) |
|
mock_crawler_process.return_value.crawl.assert_called_once() |
|
mock_crawler_process.return_value.start.assert_called_once() |
|
|
|
|
|
def test_output_directory_creation(): |
|
"""Test that output directory is created if it doesn't exist""" |
|
start_url = "https://example.com" |
|
output_dir = "test_output_dir" |
|
|
|
|
|
if os.path.exists(output_dir): |
|
os.rmdir(output_dir) |
|
|
|
crawler = DomainCrawler(start_url=start_url, output_dir=output_dir) |
|
assert os.path.exists(output_dir) |
|
|
|
|
|
os.rmdir(output_dir) |
|
|
