from django_thread import Thread
from time import sleep
from backend.module.utils import date_utils
from django.db import connections
from backend.models.model_cache import SocketRequestChapterQueueCache
from backend.models.model_1 import ComicStorageCache
from core.settings import BASE_DIR
from backend.module import web_scrap
from backend.module.utils import manage_image
from backend.module.utils.directory_info import GetDirectorySize
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync
from django.db.models import Count
import requests, environ, os, subprocess, shutil, zipfile, uuid, sys
env = environ.Env()
# os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
STORAGE_DIR = os.path.join(BASE_DIR,"storage")
if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR)
COMIC_STORAGE_DIR = os.path.join(STORAGE_DIR,"comics")
if not os.path.exists(COMIC_STORAGE_DIR): os.makedirs(COMIC_STORAGE_DIR)
LOG_DIR = os.path.join(BASE_DIR, "log")
if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR)
MAX_STORAGE_SIZE = 20 * 1024**3
class Job(Thread):
def run(self):
while True:
input_dir = ""
managed_output_dir = ""
try:
query_result = SocketRequestChapterQueueCache.objects.order_by("datetime").first()
while True:
if (GetDirectorySize(COMIC_STORAGE_DIR) >= MAX_STORAGE_SIZE):
query_result_2 = ComicStorageCache.objects.order_by("datetime").first()
if (query_result_2):
file_path = query_result_2.file_path
if os.path.exists(file_path): os.remove(file_path)
ComicStorageCache.objects.filter(id=query_result_2.id).delete()
else: break
if (query_result):
socket_id = query_result.socket_id
channel_name = query_result.channel_name
source = query_result.source
comic_id = query_result.comic_id
chapter_id = query_result.chapter_id
chapter_idx = query_result.chapter_idx
options = query_result.options
if (options.get("translate").get("state")):
target_lang = options.get("translate").get("target")
else: target_lang = ""
stored = ComicStorageCache.objects.filter(
source=source,
comic_id=comic_id,
chapter_id=chapter_id,
chapter_idx=chapter_idx,
colorize= options.get("colorize"),
translate= options.get("translate").get("state"),
target_lang= target_lang,
).first()
if (stored):
SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete()
connections['cache'].close()
connections['DB1'].close()
else:
connections['cache'].close()
connections['DB1'].close()
if (options.get("colorize") or options.get("translate").get("state")):
script = []
input_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"temp")
merge_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"merged")
if (options.get("translate").get("state") and options.get("colorize")):
managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),f"{options.get("translate").get("target")}_translated_colorized")
script = [
"python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--ocr=mocr", "--no-text-lang-skip", "--det-auto-rotate", "--det-gamma-correct", "--colorize=mc2", "--translator=m2m100_big",
"-l", f"{options.get("translate").get("target")}", "-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}"
]
elif (options.get("translate").get("state") and not options.get("colorize")):
managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),f"{options.get("translate").get("target")}_translated")
script = [
"python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--ocr=mocr", "--no-text-lang-skip", "--det-auto-rotate", "--det-gamma-correct", "--translator=m2m100_big",
"-l", f"{options.get("translate").get("target")}", "-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}"
]
elif (options.get("colorize") and not options.get("translate").get("state")):
managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"colorized")
script = [
"python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--detector=none", "--translator=original", "--colorize=mc2", "--colorization-size=-1",
"-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}"
]
if target_lang == "ENG": script.append("--manga2eng")
if os.path.exists(input_dir): shutil.rmtree(input_dir)
if os.path.exists(merge_output_dir): shutil.rmtree(merge_output_dir)
if os.path.exists(managed_output_dir): shutil.rmtree(managed_output_dir)
job = web_scrap.source_control[source].get_chapter.scrap(comic_id=comic_id,chapter_id=chapter_id,output_dir=input_dir)
if job.get("status") == "success":
manage_image.merge_images_vertically(input_dir, merge_output_dir, max_height=1800)
shutil.rmtree(input_dir)
with open(os.path.join(LOG_DIR,"image_translator_output.log"), "w") as file:
result = subprocess.run(
script,
cwd=os.path.join(BASE_DIR, "backend", "module", "utils", "image_translator"),
check=True,
shell=False if sys.platform.startswith('linux') else True,
stdout=file,
stderr=file,
text=True,
)
if result.returncode != 0: raise Exception("Image Translator Execution error!")
os.makedirs(managed_output_dir,exist_ok=True)
shutil.rmtree(merge_output_dir)
with zipfile.ZipFile(managed_output_dir + '.zip', 'w') as zipf:
for foldername, subfolders, filenames in os.walk(managed_output_dir):
for filename in filenames:
if filename.endswith(('.png', '.jpg', '.jpeg')):
file_path = os.path.join(foldername, filename)
zipf.write(file_path, os.path.basename(file_path))
shutil.rmtree(managed_output_dir)
ComicStorageCache(
source = source,
comic_id = comic_id,
chapter_id = chapter_id,
chapter_idx = chapter_idx,
file_path = managed_output_dir + ".zip",
colorize = options.get("colorize"),
translate = options.get("translate").get("state"),
target_lang = target_lang,
).save()
query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first()
channel_name = query_result_3.channel_name if query_result_3 else ""
SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete()
if channel_name:
channel_layer = get_channel_layer()
async_to_sync(channel_layer.send)(channel_name, {
'type': 'event_send',
'data': {
"type": "chapter_ready_to_download",
"data": {
"source": source,
"comic_id": comic_id,
"chapter_id": chapter_id,
"chapter_idx": chapter_idx
}
}
})
connections['cache'].close()
connections['DB1'].close()
else:
connections['cache'].close()
connections['DB1'].close()
raise Exception("#1 Dowload chapter error!")
else:
input_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"original")
merge_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"origin-merged")
if os.path.exists(input_dir): shutil.rmtree(input_dir)
if os.path.exists(merge_output_dir): shutil.rmtree(merge_output_dir)
job = web_scrap.source_control["colamanga"].get_chapter.scrap(comic_id=comic_id,chapter_id=chapter_id,output_dir=input_dir)
if job.get("status") == "success":
manage_image.merge_images_vertically(input_dir, merge_output_dir, max_height=1800)
if os.path.exists(input_dir): shutil.rmtree(input_dir)
with zipfile.ZipFile(input_dir + '.zip', 'w') as zipf:
for foldername, subfolders, filenames in os.walk(merge_output_dir):
for filename in filenames:
if filename.endswith(('.png', '.jpg', '.jpeg')):
file_path = os.path.join(foldername, filename)
zipf.write(file_path, os.path.basename(file_path))
shutil.rmtree(merge_output_dir)
ComicStorageCache(
source = source,
comic_id = comic_id,
chapter_id = chapter_id,
chapter_idx = chapter_idx,
file_path = input_dir + '.zip',
colorize = False,
translate = False,
target_lang = "",
).save()
query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first()
channel_name = query_result_3.channel_name if query_result_3 else ""
SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete()
if channel_name:
channel_layer = get_channel_layer()
async_to_sync(channel_layer.send)(channel_name, {
'type': 'event_send',
'data': {
"type": "chapter_ready_to_download",
"data": {
"source": source,
"comic_id": comic_id,
"chapter_id": chapter_id,
"chapter_idx": chapter_idx
}
}
})
else:
connections['cache'].close()
connections['DB1'].close()
raise Exception("#2 Dowload chapter error!")
connections['cache'].close()
connections['DB1'].close()
else:
connections['cache'].close()
connections['DB1'].close()
sleep(5)
except Exception as e:
print("[Error] Chapter Queue Socket:", e)
if (input_dir):
if os.path.exists(input_dir): shutil.rmtree(input_dir)
if (managed_output_dir):
if os.path.exists(managed_output_dir): shutil.rmtree(managed_output_dir)
query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first()
channel_name = query_result_3.channel_name if query_result_3 else ""
SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete()
if (channel_name):
channel_layer = get_channel_layer()
async_to_sync(channel_layer.send)(channel_name, {
'type': 'event_send',
'data': {
"type": "chapter_ready_to_download",
"data": {"state":"error"}
}
})
connections['cache'].close()
connections['DB1'].close()
sleep(10)
thread = Job()
thread.daemon = True
thread.start()
class UpdateSocketQueue(Thread):
def run(self):
while True:
try:
queue = 0
MAX_QUEUE = SocketRequestChapterQueueCache.objects.count()
if (MAX_QUEUE):
query_result = list(set(SocketRequestChapterQueueCache.objects.order_by("datetime").values_list('socket_id', flat=True).distinct()))
for socket_id in query_result:
object = {}
query_result_2 = SocketRequestChapterQueueCache.objects.filter(socket_id=socket_id).order_by("datetime").values("source","comic_id","chapter_idx")
for item in query_result_2:
source = item.get("source")
comic_id = item.get("comic_id")
chapter_idx = item.get("chapter_idx")
object[f"{source}-{comic_id}-{chapter_idx}"] = queue
queue += 1
query_result_3 = SocketRequestChapterQueueCache.objects.filter(socket_id=socket_id).first()
if (query_result_3):
channel_layer = get_channel_layer()
async_to_sync(channel_layer.send)(query_result_3.channel_name, {
'type': 'event_send',
'data': {
"type": "chapter_queue_info",
"chapter_queue": {
"queue": object,
"max_queue": MAX_QUEUE,
}
}
})
else:
pass
except Exception as e:
print(e)
connections['cache'].close()
connections['DB1'].close()
sleep(10)
thread = UpdateSocketQueue()
thread.daemon = True
thread.start()