Update app.py

app.py

@@ -8,9 +8,13 @@ import whisper
 from llm.openai_api import openai_call
 from llm.yi_moe_api import yi_moe
 from utils.trans_utils import extract_timestamps
+import os
+from pytubefix import YouTube
+import ffmpeg
+from yidong import YiDong
 
 API_URL_TEMPLATE = "https://api-yidong.lingyiwanwu.com/v1/ops/api_key?user_email={user_email}&user_source=huggingface"
-model = whisper.load_model("tiny")
+model = whisper.load_model("tiny.en")
 audio_clipper = VideoClipper(model)
 
 def get_user_email(oauth_token: gr.OAuthToken | None) -> str | None:
@@ -27,6 +31,66 @@ def get_user_email(oauth_token: gr.OAuthToken | None) -> str | None:
     email = user_info.get("email")
     return call_api(email)
 
+
+
+def get_video_size(stream):
+
+    return stream.filesize / (1024 * 1024)
+
+def download_youtube_video(url):
+    try:
+        yt = YouTube(url, use_po_token=True)
+
+        video_streams = yt.streams.filter(type="video").order_by('resolution').desc()
+        audio_stream = yt.streams.filter(only_audio=True).first()
+
+        print("Available video streams:")
+        for i, stream in enumerate(video_streams):
+            size = get_video_size(stream)
+            stream_type = "Progressive" if stream.is_progressive else "Adaptive"
+            print(f"{i}. Resolution: {stream.resolution}, Size: {size:.2f} MB, Type: {stream_type}")
+
+        choice = int(input("Enter the number of the video stream to download: "))
+        selected_stream = video_streams[choice]
+
+        if not os.path.exists('videos'):
+            os.makedirs('videos')
+
+        print(f"Downloading video: {yt.title}")
+        video_file = selected_stream.download(output_path='videos', filename_prefix="video_")
+
+        if not selected_stream.is_progressive:
+            print("Downloading audio...")
+            audio_file = audio_stream.download(output_path='videos', filename_prefix="audio_")
+
+            print("Merging video and audio...")
+            output_file = os.path.join('videos', f"{yt.title}.mp4")
+            stream = ffmpeg.input(video_file)
+            audio = ffmpeg.input(audio_file)
+            stream = ffmpeg.output(stream, audio, output_file, vcodec='libx264', acodec='aac', strict='experimental')
+            ffmpeg.run(stream, overwrite_output=True)
+
+            os.remove(video_file)
+            os.remove(audio_file)
+        else:
+            output_file = video_file
+
+        
+        print(f"Downloaded: {yt.title} to 'videos' folder")
+        print(f"File path: {output_file}")
+        return output_file
+
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+        print("Please make sure you have the latest version of pytube and ffmpeg-python installed.")
+        print("You can update them by running:")
+        print("pip install --upgrade pytube ffmpeg-python")
+        print("Also, ensure that ffmpeg is installed on your system and available in your PATH.")
+
+def updata_video(url):
+    video_path = download_youtube_video(url)
+    return video_path
+
 def audio_recog(audio_input, output_dir):
     return audio_clipper.recog(audio_input, None, output_dir=output_dir)
 
@@ -62,31 +126,47 @@ def mix_recog(video_input, audio_input,output_dir,ASR="whisper"):
 
 def llm_inference(system_content, user_content, srt_text, model, apikey):
     SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot',"gpt-4o","22A"]
+    if model.startswith('qwen'):
+        return call_qwen_model(apikey, model, user_content+'\n'+srt_text, system_content)
     if model.startswith('gpt') or model.startswith('moonshot'):
         return openai_call(apikey, model, system_content = system_content, user_content = user_content+'\n'+srt_text)
-    elif model.startswith('22A'):
+    if model.startswith('22A'):
         return yi_moe(apikey, model, user_content+'\n'+srt_text, system_content)
+    elif model.startswith('g4f'):
+        model = "-".join(model.split('-')[1:])
+        return g4f_openai_call(model, system_content, user_content+'\n'+srt_text)
     else:
         logging.error("LLM name error, only {} are supported as LLM name prefix."
                         .format(SUPPORT_LLM_PREFIX))
 
-def AI_clip(LLM_res, dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir):
-    timestamp_list = extract_timestamps(LLM_res)
-    output_dir = output_dir.strip()
-    if not len(output_dir):
-        output_dir = None
-    else:
-        output_dir = os.path.abspath(output_dir)
-    if video_state is not None:
-        clip_video_file, message, clip_srt = audio_clipper.video_clip(
-            dest_text, start_ost, end_ost, video_state, 
-            dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list, add_sub=False)
-        return clip_video_file, None, message, clip_srt
-    if audio_state is not None:
-        (sr, res_audio), message, clip_srt = audio_clipper.clip(
-            dest_text, start_ost, end_ost, audio_state, 
-            dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list, add_sub=False)
-        return None, (sr, res_audio), message, clip_srt
+
+def clip_and_summary(LLM_res, dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir, apikey):
+    def AI_clip(LLM_res, dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir):
+        timestamp_list = extract_timestamps(LLM_res)
+        output_dir = output_dir.strip()
+        if not len(output_dir):
+            output_dir = None
+        else:
+            output_dir = os.path.abspath(output_dir)
+        if video_state is not None:
+            clip_video_file, message, clip_srt = audio_clipper.video_clip(
+                dest_text, start_ost, end_ost, video_state, 
+                dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list, add_sub=False)
+            return clip_video_file, None, message, clip_srt
+        if audio_state is not None:
+            (sr, res_audio), message, clip_srt = audio_clipper.clip(
+                dest_text, start_ost, end_ost, audio_state, 
+                dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list, add_sub=False)
+            return None, (sr, res_audio), message, clip_srt
+    def get_summarizes(api_key,input_file):
+        yd = YiDong(api_key = api_key)
+        rid = yd.add_resource(input_file)
+        t = yd.video_summary(rid)
+        data = t()
+        return data.video_summary.summary
+    clip_video_file, nouse, message, clip_srt = AI_clip(LLM_res, dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir)
+    summary = get_summarizes(apikey, clip_video_file)
+    return clip_video_file, nouse, message, clip_srt, summary
 
 with gr.Blocks() as clip_service:
     video_state, audio_state = gr.State(), gr.State()
@@ -98,6 +178,8 @@ with gr.Blocks() as clip_service:
         )
         clip_service.load(get_user_email, inputs=None, outputs=user_email_display)
         logging.info(f"The value of the current variable is: {user_email_display}")
+    youtube_url = gr.Textbox(label="🔗 Youtube视频链接｜Youtube Video URL")
+    download_button = gr.Button("📥 下载 | Download", variant="primary")    
     video_input = gr.Video(label="视频输入 | Video Input")
     audio_input = gr.Audio(label="音频输入 | Audio Input")
     with gr.Column():
@@ -147,8 +229,10 @@ with gr.Blocks() as clip_service:
         video_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩ 结束位置偏移 | End Offset (ms)",visible=False)
     video_output = gr.Video(label="裁剪结果 | Video Clipped")
     audio_output = gr.Audio(label="裁剪结果 | Audio Clipped")
-    clip_message = gr.Textbox(label="⚠️ 裁剪信息 | Clipping Log")
-    srt_clipped = gr.Textbox(label="📖 裁剪部分SRT字幕内容 | Clipped RST Subtitles")   
+    clip_message = gr.Textbox(label="⚠️ 裁剪信息 | Clipping Log",visible=False)
+    srt_clipped = gr.Textbox(label="📖 裁剪部分SRT字幕内容 | Clipped RST Subtitles",visible=False) 
+    summary = gr.Textbox(label="📖 视频摘要 | Video Summary")
+    download_button.click(updata_video, inputs=youtube_url, outputs=video_input)  
     recog_button.click(mix_recog, 
                     inputs=[video_input, 
                             audio_input, 
@@ -160,7 +244,7 @@ with gr.Blocks() as clip_service:
     llm_button.click(llm_inference,
                     inputs=[prompt_head, prompt_head2, video_srt_output, llm_model, apikey_input],
                     outputs=[llm_result])
-    llm_clip_button.click(AI_clip, 
+    llm_clip_button.click(clip_and_summary, 
                     inputs=[llm_result,
                             video_text_input, 
                             video_spk_input, 
@@ -169,8 +253,9 @@ with gr.Blocks() as clip_service:
                             video_state, 
                             audio_state, 
                             output_dir,
+                            user_email_display,
                             ],
-                    outputs=[video_output, audio_output, clip_message, srt_clipped])
+                    outputs=[video_output, audio_output, clip_message, srt_clipped,summary])