Spaces:
Sleeping
Sleeping
File size: 1,051 Bytes
dbaa71b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
# Fetch full news article
from obsei.source.website_crawler_source import (
TrafilaturaCrawlerConfig,
TrafilaturaCrawlerSource,
)
def print_list(response_list):
for response in response_list:
print(response.__dict__)
# Single URL
source_config = TrafilaturaCrawlerConfig(urls=["https://obsei.github.io/obsei/"])
source = TrafilaturaCrawlerSource()
source_response_list = source.lookup(source_config)
print_list(source_response_list)
# RSS feed (Note it will take lot of time)
source_config = TrafilaturaCrawlerConfig(
urls=["https://news.google.com/rss/search?q=bitcoin&hl=en&gl=US&ceid=US:en"],
is_feed=True,
)
source = TrafilaturaCrawlerSource()
source_response_list = source.lookup(source_config)
print_list(source_response_list)
# Full website (Note it will take lot of time)
source_config = TrafilaturaCrawlerConfig(
urls=["https://haystack.deepset.ai/"],
is_sitemap=True,
)
source = TrafilaturaCrawlerSource()
source_response_list = source.lookup(source_config)
print_list(source_response_list)
|