Spaces:
Running
Running
File size: 6,871 Bytes
cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 8f79bc5 cd23862 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 |
import json, environ, requests, os, subprocess
import asyncio, uuid, shutil, time
from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest, StreamingHttpResponse
from django_ratelimit.decorators import ratelimit
from django.views.decorators.csrf import csrf_exempt
from asgiref.sync import sync_to_async
from backend.module import web_scrap
from backend.module.utils import manage_image
from core.settings import BASE_DIR
from backend.module.utils import cloudflare_turnstile
from backend.models.model_1 import WebScrapeGetCoverCache, WebScrapeGetCache
from backend.models.model_cache import RequestWebScrapeGetCoverCache, RequestWebScrapeGetCache
from backend.module.utils import directory_info, date_utils
env = environ.Env()
STORAGE_DIR = os.path.join(BASE_DIR,"storage")
if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR)
@csrf_exempt
@ratelimit(key='ip', rate='30/m')
def get_list(request):
if request.method != "POST": return HttpResponseBadRequest('Allowed POST request only!', status=400)
token = request.META.get('HTTP_X_CLOUDFLARE_TURNSTILE_TOKEN')
if not cloudflare_turnstile.check(token): return HttpResponseBadRequest('Cloudflare turnstile token not existed or expired!', status=511)
payload = json.loads(request.body)
search = payload.get("search")
page = payload.get("page")
source = payload.get("source")
if search.get("text"): DATA = web_scrap.source_control[source].search.scrap(search=search,page=page)
else: DATA = web_scrap.source_control["colamanga"].get_list.scrap(page=page)
return JsonResponse({"data":DATA})
@csrf_exempt
@ratelimit(key='ip', rate='30/m')
def get(request):
if request.method != "POST": return HttpResponseBadRequest('Allowed POST request only!', status=400)
token = request.META.get('HTTP_X_CLOUDFLARE_TURNSTILE_TOKEN')
if not cloudflare_turnstile.check(token): return HttpResponseBadRequest('Cloudflare turnstile token not existed or expired!', status=511)
payload = json.loads(request.body)
id = payload.get("id")
source = payload.get("source")
file_path = ""
file_name = ""
chunk_size = 8192
try:
query_result = WebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
if (
query_result
and query_result.datetime >= date_utils.utc_time().add(-5,'hour').get()
and os.path.exists(query_result.file_path)
):
file_path = query_result.file_path
file_name = os.path.basename(file_path)
else:
request_query = RequestWebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
if not request_query:
RequestWebScrapeGetCache(
source=source,
comic_id=id,
).save()
timeout = date_utils.utc_time().add(30,'second').get()
while True:
if date_utils.utc_time().get() >= timeout: return HttpResponseBadRequest('Request timeout!', status=408)
count = RequestWebScrapeGetCache.objects.filter(source=source,comic_id=id).count()
if count: time.sleep(1)
else: break
query_result = WebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
if (query_result):
file_path = query_result.file_path
if not os.path.exists(file_path): return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
file_name = os.path.basename(file_path)
else:
return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
def file_iterator():
with open(file_path, 'r') as f:
while chunk := f.read(chunk_size):
yield chunk
response = StreamingHttpResponse(file_iterator(), content_type='application/json')
response['Content-Length'] = os.path.getsize(file_path)
response['Content-Disposition'] = f'attachment; filename="{file_name}"'
return response
except Exception as e:
print(e)
return HttpResponseBadRequest(str(e), status=500)
@ratelimit(key='ip', rate='60/m')
def get_cover(request,source,id,cover_id):
token = request.META.get('HTTP_X_CLOUDFLARE_TURNSTILE_TOKEN')
if not cloudflare_turnstile.check(token): return HttpResponseBadRequest('Cloudflare turnstile token not existed or expired!', status=511)
file_path = ""
file_name = ""
chunk_size = 8192
try:
query_result = WebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
if (
query_result
and os.path.exists(query_result.file_path)
and query_result.datetime >= date_utils.utc_time().add(-5,'hour').get()
):
file_path = query_result.file_path
file_name = os.path.basename(file_path)
else:
request_query = RequestWebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
if not request_query:
RequestWebScrapeGetCoverCache(
source=source,
comic_id=id,
cover_id=cover_id
).save()
timeout = date_utils.utc_time().add(30,'second').get()
while True:
if date_utils.utc_time().get() >= timeout: return HttpResponseBadRequest('Request timeout!', status=408)
count = RequestWebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).count()
if count: time.sleep(1)
else: break
query_result = WebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
if (query_result):
file_path = query_result.file_path
if not os.path.exists(file_path): return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
file_name = os.path.basename(file_path)
else:
return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
def file_iterator():
with open(file_path, 'rb') as f:
while chunk := f.read(chunk_size):
yield chunk
response = StreamingHttpResponse(file_iterator(), content_type='image/png')
response['Content-Length'] = os.path.getsize(file_path)
response['Content-Disposition'] = f'attachment; filename="{file_name}"'
return response
except Exception as e:
print(e)
return HttpResponseBadRequest(str(e), status=500)
|