File size: 5,314 Bytes
fe38f2a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 |
import asyncio
import re
import time
import aiohttp
from bs4 import BeautifulSoup
from helper.asyncioPoliciesFix import decorator_asyncio_fix
from helper.html_scraper import Scraper
from constants.base_url import LIMETORRENT
from constants.headers import HEADER_AIO
class Limetorrent:
def __init__(self):
self.BASE_URL = LIMETORRENT
self.LIMIT = None
@decorator_asyncio_fix
async def _individual_scrap(self, session, url, obj):
try:
async with session.get(url, headers=HEADER_AIO) as res:
html = await res.text(encoding="ISO-8859-1")
soup = BeautifulSoup(html, "html.parser")
try:
a_tag = soup.find_all("a", class_="csprite_dltorrent")
obj["torrent"] = a_tag[0]["href"]
obj["magnet"] = a_tag[-1]["href"]
obj["hash"] = re.search(
r"([{a-f\d,A-F\d}]{32,40})\b", obj["magnet"]
).group(0)
except:
...
except:
return None
async def _get_torrent(self, result, session, urls):
tasks = []
for idx, url in enumerate(urls):
for obj in result["data"]:
if obj["url"] == url:
task = asyncio.create_task(
self._individual_scrap(session, url, result["data"][idx])
)
tasks.append(task)
await asyncio.gather(*tasks)
return result
def _parser(self, htmls, idx=0):
try:
for html in htmls:
soup = BeautifulSoup(html, "html.parser")
list_of_urls = []
my_dict = {"data": []}
for tr in soup.find_all("tr")[idx:]:
td = tr.find_all("td")
if len(td) == 0:
continue
name = td[0].get_text(strip=True)
url = self.BASE_URL + td[0].find_all("a")[-1]["href"]
list_of_urls.append(url)
added_on_and_category = td[1].get_text(strip=True)
date = (added_on_and_category.split("-")[0]).strip()
category = (added_on_and_category.split("in")[-1]).strip()
size = td[2].text
seeders = td[3].text
leechers = td[4].text
my_dict["data"].append(
{
"name": name,
"size": size,
"date": date,
"category": category if category != date else None,
"seeders": seeders,
"leechers": leechers,
"url": url,
}
)
if len(my_dict["data"]) == self.LIMIT:
break
try:
div = soup.find("div", class_="search_stat")
current_page = int(div.find("span", class_="active").text)
total_page = int((div.find_all("a"))[-2].text)
if current_page > total_page:
total_page = current_page
my_dict["current_page"] = current_page
my_dict["total_pages"] = total_page
except:
...
return my_dict, list_of_urls
except:
return None, None
async def search(self, query, page, limit):
async with aiohttp.ClientSession() as session:
start_time = time.time()
self.LIMIT = limit
url = self.BASE_URL + "/search/all/{}//{}".format(query, page)
return await self.parser_result(start_time, url, session, idx=5)
async def parser_result(self, start_time, url, session, idx=0):
htmls = await Scraper().get_all_results(session, url)
result, urls = self._parser(htmls, idx)
if result is not None:
results = await self._get_torrent(result, session, urls)
results["time"] = time.time() - start_time
results["total"] = len(results["data"])
return results
return result
async def trending(self, category, page, limit):
async with aiohttp.ClientSession() as session:
start_time = time.time()
self.LIMIT = limit
url = self.BASE_URL + "/top100"
return await self.parser_result(start_time, url, session)
async def recent(self, category, page, limit):
async with aiohttp.ClientSession() as session:
start_time = time.time()
self.LIMIT = limit
if not category:
url = self.BASE_URL + "/latest100"
else:
category = (category).capitalize()
if category == "Apps":
category = "Applications"
elif category == "Tv":
category = "TV-shows"
url = self.BASE_URL + "/browse-torrents/{}/date/{}/".format(
category, page
)
return await self.parser_result(start_time, url, session) |