run.py to download video audio and process

colab_flask.py

@@ -2,7 +2,9 @@ import os
 import subprocess
 import time
 from datetime import datetime
-
+from functools import partial
+from concurrent.futures import ThreadPoolExecutor
+from collections import deque
 import cv2
 from flask import Flask, request, jsonify, send_file
 from flask_ngrok2 import run_with_ngrok
@@ -11,6 +13,8 @@ from tqdm import tqdm
 import numpy as np
 from ffmpy import FFmpeg
 
+from main import call_wav2lip, call_gfpgan, merge
+
 # !pip install flask flask-ngrok2 pyngrok
 
 app = Flask(__name__)
@@ -58,105 +62,33 @@ def wav2lip():
         output_mp4 = os.path.join(job_path, output_filename)
         call_gfpgan(wav2lip_mp4, audio_path, output_mp4)
 
+        output_filename = 'output.mp4'
+        output_mp4 = os.path.join(job_path, output_filename)
+        merge(job_path, audio_path, output_mp4)
+
         return jsonify({'url': f'/job/{job_id}/{output_filename}'})
 
     except Exception as e:
         return jsonify({'error': str(e)}), 500
 
 
-def call_wav2lip(video_path, audio_path, output_path):
-    checkpoint_path = os.path.join(root_dir, 'wav2lip/checkpoints/wav2lip.pth')
-    assert os.path.isfile(video_path), f'Video path {video_path} not exist.'
-    assert os.path.isfile(audio_path), f'Audio path {audio_path} not exist.'
-    assert os.path.isfile(checkpoint_path), f'Checkpoint file {checkpoint_path} not exist.'
-
-    # python inference.py \
-    # --checkpoint_path checkpoints/wav2lip.pth \
-    # --face {inputVideoPath} \
-    # --audio {inputAudioPath} \
-    # --outfile {lipSyncedOutputPath}
-    start = datetime.now()
-    cmd = [
-        "python",
-        "wav2lip/inference.py",
-        "--checkpoint_path", checkpoint_path,  #
-        # "--segmentation_path", "checkpoints/face_segmentation.pth",
-        "--face", video_path,
-        "--audio", audio_path,
-        "--outfile", output_path,
-    ]
-    print(f'Call subprocess: {cmd}')
-    proc = subprocess.Popen(cmd, shell=False)
-    proc.communicate()
-    duration = datetime.now() - start
-    print(f'wav2lip finished in {duration}')
-    return output_path
-
-
-def _get_frames(video_path):
-    folder_path = os.path.dirname(video_path)
-    origin_frames_folder = os.path.join(folder_path, 'frames')
-    os.makedirs(origin_frames_folder, exist_ok=True)
-
-    # get frames pics
-    vidcap = cv2.VideoCapture(video_path)
-    numberOfFrames = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
-    fps = vidcap.get(cv2.CAP_PROP_FPS)
-    print("FPS: ", fps, "Frames: ", numberOfFrames)
-
-    for frameNumber in tqdm(range(numberOfFrames)):
-        _, image = vidcap.read()
-        cv2.imwrite(os.path.join(origin_frames_folder, str(frameNumber).zfill(4) + '.jpg'), image)
-    return origin_frames_folder
-
-
-def call_gfpgan(wav2lip_mp4, audio_path, output_mp4):
-    assert os.path.isfile(wav2lip_mp4), f'Video path {wav2lip_mp4} not exist.'
-    origin_frames_folder = _get_frames(wav2lip_mp4)
-    folder_path = os.path.dirname(wav2lip_mp4)
-
-    # python inference_gfpgan.py
-    # -i "$unProcessedFramesFolderPath"
-    # -o "$outputPath"
-    # -v 1.3
-    # -s 2
-    # --only_center_face
-    # --bg_upsampler None
-    start = datetime.now()
-    cmd = [
-        "python",
-        "gfpgan/inference_gfpgan.py",
-        "-i", origin_frames_folder,
-        "-o", folder_path,
-        # "-v", str(1.4),
-        # "-s", str(2),
-        "--only_center_face",
-        "--bg_upsampler", 'realesrgan'
-    ]
-    print(cmd)
-    proc = subprocess.Popen(cmd, shell=True)
-    proc.communicate()
-    duration = datetime.now() - start
-    print(f'inference_gfpgan finished in {duration}')
-
-    start = datetime.now()
-    cmd = [
-        "python",
-        "merge.py",
-        "-j", folder_path,
-        "-a", audio_path,
-        "-o", output_mp4,
-    ]
-    proc = subprocess.Popen(cmd, shell=True)
-    proc.communicate()
-    duration = datetime.now() - start
-    print(f'Merge output in {duration}')
-    print(output_mp4)
-
-    # from google.colab import files
-    # files.download(finalProcessedOuputVideo)
-
-
 if __name__ == '__main__':
     run_with_ngrok(app, auth_token=auth_token)
     app.run()
+
+
+def test():
+    # request
+    import requests
+    ngrok_url = f"http://74c0-34-87-172-60.ngrok-free.app"
+    url = f"{ngrok_url}/wav2lip"
+    print(url)
+    video_path = '/Users/taoluo/Downloads/oIy5B4-vHVw.4.6588496370531551262.0.jpg'
+    audio_path = '/Users/taoluo/Downloads/test_audio.mp3'
+    files = {'video': ('video.jpg', open(video_path, 'rb')), 'audio': ('audio.mp3', open(audio_path, 'rb'))}
+    headers = {'ngrok-skip-browser-warning': 'true'}
+    response = requests.post(url, files=files, headers=headers)
+    # Print the response
+    print(response.json())
+    data = response.json()
+    print(ngrok_url + data['url'])

gfpgan/inference_gfpgan.py

@@ -117,7 +117,7 @@ def main():
     for img_path in tqdm(img_list):
         # read image
         img_name = os.path.basename(img_path)
-        print(f'Processing {img_name} ...')
+        # print(f'Processing {img_name} ...')
         basename, ext = os.path.splitext(img_name)
         input_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
 

main.py

@@ -1,27 +1,125 @@
 import os
-from argparse import Namespace
-from
-basePath = os.path.dirname(os.path.realpath(__file__))
-wav2lipFolderName = 'wav2lip'
-gfpganFolderName = 'gfpgan'
-wav2lipPath = os.path.join(basePath, wav2lipFolderName)
-gfpganPath = os.path.join(basePath, gfpganFolderName)
-
-outputPath = basePath + '/outputs'
-inputAudioPath = basePath + '/inputs/kim_audio.mp3'
-inputVideoPath = basePath + '/inputs/kimk_7s_raw.mp4'
-lipSyncedOutputPath = basePath + '/outputs/result.mp4'
-if not os.path.exists(outputPath):
-    os.makedirs(outputPath)
-
-if __name__ == '__main__':
-    args = Namespace(checkpoint_path='checkpoints/wav2lip.pth',
-                     face=inputVideoPath,
-                     audio=inputAudioPath,
-                     outfile='results/result_voice.mp4',
-                     static=False,
-                     fps=25.0, pads=[0, 10, 0, 0], face_det_batch_size=16, wav2lip_batch_size=128, resize_factor=1,
-                     crop=[0, -1, 0, -1], box=[-1, -1, -1, -1], rotate=False, nosmooth=False, img_size=96)
-
-    print(inputAudioPath, os.path.isdir(inputAudioPath))
-    print(inputVideoPath, os.path.isdir(inputVideoPath))
+import subprocess
+import cv2
+from datetime import datetime
+from functools import partial
+from concurrent.futures import ThreadPoolExecutor
+from collections import deque
+from tqdm import tqdm
+
+root_dir = '/content/wav2lip-gfpgan'
+
+
+def stream_command(
+        args,
+        *,
+        stdout_handler=print,
+        stderr_handler=print,
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        **kwargs,
+):
+    """Mimic subprocess.run, while processing the command output in real time."""
+    with subprocess.Popen(args, text=text, stdout=stdout, stderr=stderr, **kwargs) as process:
+        with ThreadPoolExecutor(2) as pool:  # two threads to handle the streams
+            exhaust = partial(pool.submit, partial(deque, maxlen=0))
+            exhaust(stdout_handler(line[:-1]) for line in process.stdout)
+            exhaust(stderr_handler(line[:-1]) for line in process.stderr)
+    retcode = process.poll()
+    if check and retcode:
+        raise subprocess.CalledProcessError(retcode, process.args)
+    return subprocess.CompletedProcess(process.args, retcode)
+
+
+def call_wav2lip(video_path, audio_path, output_path):
+    checkpoint_path = os.path.join(root_dir, 'wav2lip/checkpoints/wav2lip.pth')
+    assert os.path.isfile(video_path), f'Video path {video_path} not exist.'
+    assert os.path.isfile(audio_path), f'Audio path {audio_path} not exist.'
+    assert os.path.isfile(checkpoint_path), f'Checkpoint file {checkpoint_path} not exist.'
+
+    # python inference.py \
+    # --checkpoint_path checkpoints/wav2lip.pth \
+    # --face {inputVideoPath} \
+    # --audio {inputAudioPath} \
+    # --outfile {lipSyncedOutputPath}
+    start = datetime.now()
+    cmd = [
+        "python",
+        "wav2lip/inference.py",
+        "--checkpoint_path", checkpoint_path,  #
+        # "--segmentation_path", "checkpoints/face_segmentation.pth",
+        "--face", video_path,
+        "--audio", audio_path,
+        "--outfile", output_path,
+    ]
+
+    print(f'Call subprocess: {cmd}')
+    stream_command(cmd)
+    duration = datetime.now() - start
+    print(f'wav2lip finished in {duration}')
+    origin_frames_folder = _get_frames(output_path)
+    return output_path
+
+
+def _get_frames(video_path):
+    folder_path = os.path.dirname(video_path)
+    origin_frames_folder = os.path.join(folder_path, 'frames')
+    os.makedirs(origin_frames_folder, exist_ok=True)
+
+    # get frames pics
+    vidcap = cv2.VideoCapture(video_path)
+    numberOfFrames = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = vidcap.get(cv2.CAP_PROP_FPS)
+    print("FPS: ", fps, "Frames: ", numberOfFrames)
+
+    for frameNumber in tqdm(range(numberOfFrames)):
+        _, image = vidcap.read()
+        cv2.imwrite(os.path.join(origin_frames_folder, str(frameNumber).zfill(4) + '.jpg'), image)
+    return origin_frames_folder
+
+
+def call_gfpgan(wav2lip_mp4):
+    assert os.path.isfile(wav2lip_mp4), f'Video path {wav2lip_mp4} not exist.'
+    folder_path = os.path.dirname(wav2lip_mp4)
+    origin_frames_folder = os.path.join(folder_path, 'frames')
+
+    # python inference_gfpgan.py
+    # -i "$unProcessedFramesFolderPath"
+    # -o "$outputPath"
+    # -v 1.3
+    # -s 2
+    # --only_center_face
+    # --bg_upsampler None
+    start = datetime.now()
+    cmd = [
+        "python",
+        "gfpgan/inference_gfpgan.py",
+        "-i", origin_frames_folder,
+        "-o", folder_path,
+        # "-v", str(1.4),
+        # "-s", str(2),
+        "--only_center_face",
+        "--bg_upsampler", 'realesrgan'
+    ]
+    print(cmd)
+    stream_command(cmd)
+    duration = datetime.now() - start
+    print(f'inference_gfpgan finished in {duration}')
+
+
+def merge(folder_path, audio_path, output_mp4):
+    start = datetime.now()
+    cmd = [
+        "python",
+        "merge.py",
+        "-j", folder_path,
+        "-a", audio_path,
+        "-o", output_mp4,
+    ]
+    stream_command(cmd)
+    duration = datetime.now() - start
+    print(f'Merge output in {duration}')
+    print(output_mp4)
+

merge.py

@@ -73,5 +73,5 @@ if __name__ == '__main__':
     parser.add_argument('-a', '--audio', type=str, help='audio file path')
     parser.add_argument('-o', '--output', type=str, help='output file path')
     args = parser.parse_args()
-    
+
     cli(args)

run.py

@@ -0,0 +1,51 @@
+import os
+import sys
+from urllib.request import urlretrieve
+
+from main import call_wav2lip, call_gfpgan, merge
+
+root_dir = '/content/jobs'
+os.makedirs(root_dir,exist_ok=True)
+
+
+def main(job_id, video_url, audio_url):
+    job_path = os.path.join(root_dir, job_id)
+    os.makedirs(job_path, exist_ok=True)
+
+    if video_url.startswith('http'):
+        video_file = os.path.basename(video_url)
+        video_path = os.path.join(job_path, video_file)
+        urlretrieve(video_url, video_path)
+    else:
+        video_path = video_url
+
+    if audio_url.startswith('http'):
+        audio_file = os.path.basename(audio_url)
+        audio_path = os.path.join(job_path, audio_file)
+        urlretrieve(audio_url, audio_path)
+    else:
+        audio_path = audio_url
+
+    assert os.path.isfile(video_path), f'Video {video_path} not exist.'
+    assert os.path.isfile(audio_path), f'Audio {audio_path} not exist.'
+
+    wav2lip_mp4 = os.path.join(job_path, 'wav2lip.mp4')
+    call_wav2lip(video_path, audio_path, wav2lip_mp4)
+    call_gfpgan(wav2lip_mp4)
+
+    output_filename = 'output.mp4'
+    output_mp4 = os.path.join(job_path, output_filename)
+    merge(job_path, audio_path, output_mp4)
+    return output_mp4
+
+
+if __name__ == '__main__':
+    job_id = sys.argv[1]
+    video_url = sys.argv[2]
+    audio_url = sys.argv[3]
+
+    output_mp4 = main(job_id, video_url, audio_url)
+
+    from google.colab import files
+
+    files.download(output_mp4)