Spaces:
Sleeping
Sleeping
import gradio as gr | |
import pandas as pd | |
import requests | |
from bs4 import BeautifulSoup | |
from docx import Document | |
import os | |
from openai import OpenAI | |
from groq import Groq | |
import uuid | |
from gtts import gTTS | |
import math | |
from pydub import AudioSegment | |
from youtube_transcript_api import YouTubeTranscriptApi | |
from youtube_transcript_api._errors import NoTranscriptFound | |
import yt_dlp | |
from moviepy.editor import VideoFileClip | |
from pytube import YouTube | |
import os | |
import io | |
import time | |
import json | |
from urllib.parse import urlparse, parse_qs | |
from google.cloud import storage | |
from google.oauth2 import service_account | |
from googleapiclient.discovery import build | |
from googleapiclient.http import MediaFileUpload | |
from googleapiclient.http import MediaIoBaseDownload | |
from googleapiclient.http import MediaIoBaseUpload | |
from educational_material import EducationalMaterial | |
from storage_service import GoogleCloudStorage | |
import boto3 | |
from chatbot import Chatbot | |
is_env_local = os.getenv("IS_ENV_LOCAL", "false") == "true" | |
print(f"is_env_local: {is_env_local}") | |
print("===gr__version__===") | |
print(gr.__version__) | |
if is_env_local: | |
with open("local_config.json") as f: | |
config = json.load(f) | |
PASSWORD = config["PASSWORD"] | |
GCS_KEY = json.dumps(config["GOOGLE_APPLICATION_CREDENTIALS_JSON"]) | |
DRIVE_KEY = json.dumps(config["GOOGLE_APPLICATION_CREDENTIALS_JSON"]) | |
OPEN_AI_KEY = config["OPEN_AI_KEY"] | |
GROQ_API_KEY = config["GROQ_API_KEY"] | |
JUTOR_CHAT_KEY = config["JUTOR_CHAT_KEY"] | |
AWS_ACCESS_KEY = config["AWS_ACCESS_KEY"] | |
AWS_SECRET_KEY = config["AWS_SECRET_KEY"] | |
AWS_REGION_NAME = config["AWS_REGION_NAME"] | |
OUTPUT_PATH = config["OUTPUT_PATH"] | |
else: | |
PASSWORD = os.getenv("PASSWORD") | |
GCS_KEY = os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON") | |
DRIVE_KEY = os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON") | |
OPEN_AI_KEY = os.getenv("OPEN_AI_KEY") | |
GROQ_API_KEY = os.getenv("GROQ_API_KEY") | |
JUTOR_CHAT_KEY = os.getenv("JUTOR_CHAT_KEY") | |
AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY") | |
AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY") | |
AWS_REGION_NAME = 'us-west-2' | |
OUTPUT_PATH = 'videos' | |
TRANSCRIPTS = [] | |
CURRENT_INDEX = 0 | |
VIDEO_ID = "" | |
OPEN_AI_CLIENT = OpenAI(api_key=OPEN_AI_KEY) | |
GROQ_CLIENT = Groq(api_key=GROQ_API_KEY) | |
GCS_SERVICE = GoogleCloudStorage(GCS_KEY) | |
GCS_CLIENT = GCS_SERVICE.client | |
BEDROCK_CLIENT = boto3.client( | |
service_name="bedrock-runtime", | |
aws_access_key_id=AWS_ACCESS_KEY, | |
aws_secret_access_key=AWS_SECRET_KEY, | |
region_name=AWS_REGION_NAME, | |
) | |
# 驗證 password | |
def verify_password(password): | |
if password == PASSWORD: | |
return True | |
else: | |
raise gr.Error("密碼錯誤") | |
# ====gcs==== | |
def gcs_check_file_exists(gcs_client, bucket_name, file_name): | |
""" | |
检查 GCS 存储桶中是否存在指定的文件 | |
file_name 格式:{folder_name}/{file_name} | |
""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(file_name) | |
return blob.exists() | |
def upload_file_to_gcs(gcs_client, bucket_name, destination_blob_name, file_path): | |
"""上传文件到指定的 GCS 存储桶""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(destination_blob_name) | |
blob.upload_from_filename(file_path) | |
print(f"File {file_path} uploaded to {destination_blob_name} in GCS.") | |
def upload_file_to_gcs_with_json_string(gcs_client, bucket_name, destination_blob_name, json_string): | |
"""上传字符串到指定的 GCS 存储桶""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(destination_blob_name) | |
blob.upload_from_string(json_string) | |
print(f"JSON string uploaded to {destination_blob_name} in GCS.") | |
def download_blob_to_string(gcs_client, bucket_name, source_blob_name): | |
"""从 GCS 下载文件内容到字符串""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(source_blob_name) | |
return blob.download_as_text() | |
def make_blob_public(gcs_client, bucket_name, blob_name): | |
"""将指定的 GCS 对象设置为公共可读""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(blob_name) | |
blob.make_public() | |
print(f"Blob {blob_name} is now publicly accessible at {blob.public_url}") | |
def get_blob_public_url(gcs_client, bucket_name, blob_name): | |
"""获取指定 GCS 对象的公开 URL""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(blob_name) | |
return blob.public_url | |
def upload_img_and_get_public_url(gcs_client, bucket_name, file_name, file_path): | |
"""上传图片到 GCS 并获取其公开 URL""" | |
# 上传图片 | |
upload_file_to_gcs(gcs_client, bucket_name, file_name, file_path) | |
# 将上传的图片设置为公开 | |
make_blob_public(gcs_client, bucket_name, file_name) | |
# 获取图片的公开 URL | |
public_url = get_blob_public_url(gcs_client, bucket_name, file_name) | |
print(f"Public URL for the uploaded image: {public_url}") | |
return public_url | |
def copy_all_files_from_drive_to_gcs(drive_service, gcs_client, drive_folder_id, bucket_name, gcs_folder_name): | |
# Get all files from the folder | |
query = f"'{drive_folder_id}' in parents and trashed = false" | |
response = drive_service.files().list(q=query).execute() | |
files = response.get('files', []) | |
for file in files: | |
# Copy each file to GCS | |
file_id = file['id'] | |
file_name = file['name'] | |
gcs_destination_path = f"{gcs_folder_name}/{file_name}" | |
copy_file_from_drive_to_gcs(drive_service, gcs_client, file_id, bucket_name, gcs_destination_path) | |
def copy_file_from_drive_to_gcs(drive_service, gcs_client, file_id, bucket_name, gcs_destination_path): | |
# Download file content from Drive | |
request = drive_service.files().get_media(fileId=file_id) | |
fh = io.BytesIO() | |
downloader = MediaIoBaseDownload(fh, request) | |
done = False | |
while not done: | |
status, done = downloader.next_chunk() | |
fh.seek(0) | |
file_content = fh.getvalue() | |
# Upload file content to GCS | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(gcs_destination_path) | |
blob.upload_from_string(file_content) | |
print(f"File {file_id} copied to GCS at {gcs_destination_path}.") | |
def delete_blob(gcs_client, bucket_name, blob_name): | |
"""删除指定的 GCS 对象""" | |
bucket = gcs_client.bucket(bucket_name) | |
blob = bucket.blob(blob_name) | |
blob.delete() | |
print(f"Blob {blob_name} deleted from GCS.") | |
# # ====drive====初始化 | |
def init_drive_service(): | |
credentials_json_string = DRIVE_KEY | |
credentials_dict = json.loads(credentials_json_string) | |
SCOPES = ['https://www.googleapis.com/auth/drive'] | |
credentials = service_account.Credentials.from_service_account_info( | |
credentials_dict, scopes=SCOPES) | |
service = build('drive', 'v3', credentials=credentials) | |
return service | |
def create_folder_if_not_exists(service, folder_name, parent_id): | |
print("检查是否存在特定名称的文件夹,如果不存在则创建") | |
query = f"mimeType='application/vnd.google-apps.folder' and name='{folder_name}' and '{parent_id}' in parents and trashed=false" | |
response = service.files().list(q=query, spaces='drive', fields="files(id, name)").execute() | |
folders = response.get('files', []) | |
if not folders: | |
# 文件夹不存在,创建新文件夹 | |
file_metadata = { | |
'name': folder_name, | |
'mimeType': 'application/vnd.google-apps.folder', | |
'parents': [parent_id] | |
} | |
folder = service.files().create(body=file_metadata, fields='id').execute() | |
return folder.get('id') | |
else: | |
# 文件夹已存在 | |
return folders[0]['id'] | |
# 检查Google Drive上是否存在文件 | |
def check_file_exists(service, folder_name, file_name): | |
query = f"name = '{file_name}' and '{folder_name}' in parents and trashed = false" | |
response = service.files().list(q=query).execute() | |
files = response.get('files', []) | |
return len(files) > 0, files[0]['id'] if files else None | |
def upload_content_directly(service, file_name, folder_id, content): | |
""" | |
直接将内容上传到Google Drive中的新文件。 | |
""" | |
if not file_name: | |
raise ValueError("文件名不能为空") | |
if not folder_id: | |
raise ValueError("文件夹ID不能为空") | |
if content is None: # 允许空字符串上传,但不允许None | |
raise ValueError("内容不能为空") | |
file_metadata = {'name': file_name, 'parents': [folder_id]} | |
# 使用io.BytesIO为文本内容创建一个内存中的文件对象 | |
try: | |
with io.BytesIO(content.encode('utf-8')) as fh: | |
media = MediaIoBaseUpload(fh, mimetype='text/plain', resumable=True) | |
print("==content==") | |
print(content) | |
print("==content==") | |
print("==media==") | |
print(media) | |
print("==media==") | |
# 执行上传 | |
file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() | |
return file.get('id') | |
except Exception as e: | |
print(f"上传文件时发生错误: {e}") | |
raise # 重新抛出异常,调用者可以根据需要处理或忽略 | |
def upload_file_directly(service, file_name, folder_id, file_path): | |
# 上傳 .json to Google Drive | |
file_metadata = {'name': file_name, 'parents': [folder_id]} | |
media = MediaFileUpload(file_path, mimetype='application/json') | |
file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() | |
# return file.get('id') # 返回文件ID | |
return True | |
def upload_img_directly(service, file_name, folder_id, file_path): | |
file_metadata = {'name': file_name, 'parents': [folder_id]} | |
media = MediaFileUpload(file_path, mimetype='image/jpeg') | |
file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() | |
return file.get('id') # 返回文件ID | |
def download_file_as_string(service, file_id): | |
""" | |
从Google Drive下载文件并将其作为字符串返回。 | |
""" | |
request = service.files().get_media(fileId=file_id) | |
fh = io.BytesIO() | |
downloader = MediaIoBaseDownload(fh, request) | |
done = False | |
while done is False: | |
status, done = downloader.next_chunk() | |
fh.seek(0) | |
content = fh.read().decode('utf-8') | |
return content | |
def set_public_permission(service, file_id): | |
service.permissions().create( | |
fileId=file_id, | |
body={"type": "anyone", "role": "reader"}, | |
fields='id', | |
).execute() | |
def update_file_on_drive(service, file_id, file_content): | |
""" | |
更新Google Drive上的文件内容。 | |
参数: | |
- service: Google Drive API服务实例。 | |
- file_id: 要更新的文件的ID。 | |
- file_content: 新的文件内容,字符串格式。 | |
""" | |
# 将新的文件内容转换为字节流 | |
fh = io.BytesIO(file_content.encode('utf-8')) | |
media = MediaIoBaseUpload(fh, mimetype='application/json', resumable=True) | |
# 更新文件 | |
updated_file = service.files().update( | |
fileId=file_id, | |
media_body=media | |
).execute() | |
print(f"文件已更新,文件ID: {updated_file['id']}") | |
# ---- Text file ---- | |
def process_file(password, file): | |
verify_password(password) | |
# 读取文件 | |
if file.name.endswith('.csv'): | |
df = pd.read_csv(file) | |
text = df_to_text(df) | |
elif file.name.endswith('.xlsx'): | |
df = pd.read_excel(file) | |
text = df_to_text(df) | |
elif file.name.endswith('.docx'): | |
text = docx_to_text(file) | |
else: | |
raise ValueError("Unsupported file type") | |
df_string = df.to_string() | |
# 宜蘭:移除@XX@符号 to | | |
df_string = df_string.replace("@XX@", "|") | |
# 根据上传的文件内容生成问题 | |
questions = generate_questions(df_string) | |
summary = generate_summarise(df_string) | |
# 返回按钮文本和 DataFrame 字符串 | |
return questions[0] if len(questions) > 0 else "", \ | |
questions[1] if len(questions) > 1 else "", \ | |
questions[2] if len(questions) > 2 else "", \ | |
summary, \ | |
df_string | |
def df_to_text(df): | |
# 将 DataFrame 转换为纯文本 | |
return df.to_string() | |
def docx_to_text(file): | |
# 将 Word 文档转换为纯文本 | |
doc = Document(file) | |
return "\n".join([para.text for para in doc.paragraphs]) | |
# ---- YouTube link ---- | |
def format_seconds_to_time(seconds): | |
"""将秒数格式化为 时:分:秒 的形式""" | |
hours = int(seconds // 3600) | |
minutes = int((seconds % 3600) // 60) | |
seconds = int(seconds % 60) | |
return f"{hours:02}:{minutes:02}:{seconds:02}" | |
def extract_youtube_id(url): | |
parsed_url = urlparse(url) | |
if "youtube.com" in parsed_url.netloc: | |
# 对于标准链接,视频ID在查询参数'v'中 | |
query_params = parse_qs(parsed_url.query) | |
return query_params.get("v")[0] if "v" in query_params else None | |
elif "youtu.be" in parsed_url.netloc: | |
# 对于短链接,视频ID是路径的一部分 | |
return parsed_url.path.lstrip('/') | |
else: | |
return None | |
def get_transcript(video_id): | |
languages = ['zh-TW', 'zh-Hant', 'zh', 'en'] # 優先順序列表 | |
for language in languages: | |
try: | |
transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=[language]) | |
return transcript # 成功獲取字幕,直接返回結果 | |
except NoTranscriptFound: | |
continue # 當前語言的字幕沒有找到,繼續嘗試下一個語言 | |
return None # 所有嘗試都失敗,返回None | |
def generate_transcription(video_id): | |
youtube_url = f'https://www.youtube.com/watch?v={video_id}' | |
codec_name = "mp3" | |
outtmpl = f"{OUTPUT_PATH}/{video_id}.%(ext)s" | |
ydl_opts = { | |
'format': 'bestaudio/best', | |
'postprocessors': [{ | |
'key': 'FFmpegExtractAudio', | |
'preferredcodec': codec_name, | |
'preferredquality': '192' | |
}], | |
'outtmpl': outtmpl, | |
} | |
with yt_dlp.YoutubeDL(ydl_opts) as ydl: | |
ydl.download([youtube_url]) | |
audio_path = f"{OUTPUT_PATH}/{video_id}.{codec_name}" | |
full_audio = AudioSegment.from_mp3(audio_path) | |
max_part_duration = 10 * 60 * 1000 # 10 minutes | |
full_duration = len(full_audio) # in milliseconds | |
parts = math.ceil(full_duration / max_part_duration) | |
print(f"parts: {parts}") | |
transcription = [] | |
for i in range(parts): | |
print(f"== i: {i}==") | |
start_time = i * max_part_duration | |
end_time = min((i + 1) * max_part_duration, full_duration) | |
print(f"time: {start_time/1000} - {end_time/1000}") | |
chunk = full_audio[start_time:end_time] | |
chunk_path = f"{OUTPUT_PATH}/{video_id}_part_{i}.{codec_name}" | |
chunk.export(chunk_path, format=codec_name) | |
with open(chunk_path, "rb") as chunk_file: | |
response = OPEN_AI_CLIENT.audio.transcriptions.create( | |
model="whisper-1", | |
file=chunk_file, | |
response_format="verbose_json", | |
timestamp_granularities=["segment"], | |
prompt="Transcribe the following audio file. if chinese, please using 'language: zh-TW' ", | |
) | |
# Adjusting the timestamps for the chunk based on its position in the full audio | |
adjusted_segments = [{ | |
'text': segment['text'], | |
'start': math.ceil(segment['start'] + start_time / 1000.0), # Converting milliseconds to seconds | |
'end': math.ceil(segment['end'] + start_time / 1000.0), | |
'duration': math.ceil(segment['end'] - segment['start']) | |
} for segment in response.segments] | |
transcription.extend(adjusted_segments) | |
# Remove temporary chunk files after processing | |
os.remove(chunk_path) | |
return transcription | |
def process_transcript_and_screenshots(video_id): | |
print("====process_transcript_and_screenshots====") | |
# Drive | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
# 逐字稿文件名 | |
file_name = f'{video_id}_transcript.json' | |
# 检查逐字稿是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
# 从YouTube获取逐字稿并上传 | |
transcript = get_transcript(video_id) | |
if transcript: | |
print("成功獲取字幕") | |
else: | |
print("沒有找到字幕") | |
transcript_text = json.dumps(transcript, ensure_ascii=False, indent=2) | |
file_id = upload_content_directly(service, file_name, folder_id, transcript_text) | |
print("逐字稿已上传到Google Drive") | |
else: | |
# 逐字稿已存在,下载逐字稿内容 | |
print("逐字稿已存在于Google Drive中") | |
transcript_text = download_file_as_string(service, file_id) | |
transcript = json.loads(transcript_text) | |
# 处理逐字稿中的每个条目,检查并上传截图 | |
for entry in transcript: | |
if 'img_file_id' not in entry: | |
screenshot_path = screenshot_youtube_video(video_id, entry['start']) | |
img_file_id = upload_img_directly(service, f"{video_id}_{entry['start']}.jpg", folder_id, screenshot_path) | |
set_public_permission(service, img_file_id) | |
entry['img_file_id'] = img_file_id | |
print(f"截图已上传到Google Drive: {img_file_id}") | |
# 更新逐字稿文件 | |
updated_transcript_text = json.dumps(transcript, ensure_ascii=False, indent=2) | |
update_file_on_drive(service, file_id, updated_transcript_text) | |
print("逐字稿已更新,包括截图链接") | |
return transcript | |
def process_transcript_and_screenshots_on_gcs(video_id): | |
print("====process_transcript_and_screenshots_on_gcs====") | |
# GCS | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
# 逐字稿文件名 | |
transcript_file_name = f'{video_id}_transcript.json' | |
transcript_blob_name = f"{video_id}/{transcript_file_name}" | |
# 检查逐字稿是否存在 | |
is_transcript_exists = GCS_SERVICE.check_file_exists(bucket_name, transcript_blob_name) | |
if not is_transcript_exists: | |
# 从YouTube获取逐字稿并上传 | |
try: | |
transcript = get_transcript(video_id) | |
except: | |
# call open ai whisper | |
print("===call open ai whisper===") | |
transcript = generate_transcription(video_id) | |
if transcript: | |
print("成功獲取字幕") | |
else: | |
print("沒有找到字幕") | |
transcript = generate_transcription(video_id) | |
transcript_text = json.dumps(transcript, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, transcript_blob_name, transcript_text) | |
else: | |
# 逐字稿已存在,下载逐字稿内容 | |
print("逐字稿已存在于GCS中") | |
transcript_text = download_blob_to_string(gcs_client, bucket_name, transcript_blob_name) | |
transcript = json.loads(transcript_text) | |
# print("===確認其他衍生文件===") | |
# source = "gcs" | |
# get_questions(video_id, transcript_text, source) | |
# get_video_id_summary(video_id, transcript_text, source) | |
# get_mind_map(video_id, transcript_text, source) | |
# print("===確認其他衍生文件 end ===") | |
# 處理截圖 | |
for entry in transcript: | |
if 'img_file_id' not in entry: | |
# 檢查 OUTPUT_PATH 是否存在 video_id.mp4 | |
video_path = f'{OUTPUT_PATH}/{video_id}.mp4' | |
if not os.path.exists(video_path): | |
# try 5 times 如果都失敗就 raise | |
for i in range(5): | |
try: | |
download_youtube_video(video_id) | |
break | |
except Exception as e: | |
if i == 4: | |
raise gr.Error(f"下载视频失败: {str(e)}") | |
time.sleep(5) | |
# 截图 | |
screenshot_path = screenshot_youtube_video(video_id, entry['start']) | |
screenshot_blob_name = f"{video_id}/{video_id}_{entry['start']}.jpg" | |
img_file_id = upload_img_and_get_public_url(gcs_client, bucket_name, screenshot_blob_name, screenshot_path) | |
entry['img_file_id'] = img_file_id | |
print(f"截图已上传到GCS: {img_file_id}") | |
# 更新逐字稿文件 | |
print("===更新逐字稿文件===") | |
print(transcript) | |
print("===更新逐字稿文件===") | |
updated_transcript_text = json.dumps(transcript, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, transcript_blob_name, updated_transcript_text) | |
print("逐字稿已更新,包括截图链接") | |
updated_transcript_json = json.loads(updated_transcript_text) | |
return updated_transcript_json | |
def process_youtube_link(password, link): | |
verify_password(password) | |
# 使用 YouTube API 获取逐字稿 | |
# 假设您已经获取了 YouTube 视频的逐字稿并存储在变量 `transcript` 中 | |
video_id = extract_youtube_id(link) | |
global VIDEO_ID | |
VIDEO_ID = video_id | |
try: | |
# transcript = process_transcript_and_screenshots(video_id) | |
transcript = process_transcript_and_screenshots_on_gcs(video_id) | |
except Exception as e: | |
error_msg = f" {video_id} 逐字稿錯誤: {str(e)}" | |
print("===process_youtube_link error===") | |
print(error_msg) | |
raise gr.Error(error_msg) | |
formatted_transcript = [] | |
formatted_simple_transcript =[] | |
screenshot_paths = [] | |
for entry in transcript: | |
start_time = format_seconds_to_time(entry['start']) | |
end_time = format_seconds_to_time(entry['start'] + entry['duration']) | |
embed_url = get_embedded_youtube_link(video_id, entry['start']) | |
img_file_id = entry['img_file_id'] | |
# img_file_id ="" | |
# 先取消 Google Drive 的图片 | |
# screenshot_path = f"https://lh3.googleusercontent.com/d/{img_file_id}=s4000" | |
screenshot_path = img_file_id | |
line = { | |
"start_time": start_time, | |
"end_time": end_time, | |
"text": entry['text'], | |
"embed_url": embed_url, | |
"screenshot_path": screenshot_path | |
} | |
formatted_transcript.append(line) | |
# formatted_simple_transcript 只要 start_time, end_time, text | |
simple_line = { | |
"start_time": start_time, | |
"end_time": end_time, | |
"text": entry['text'] | |
} | |
formatted_simple_transcript.append(simple_line) | |
screenshot_paths.append(screenshot_path) | |
global TRANSCRIPTS | |
TRANSCRIPTS = formatted_transcript | |
# 基于逐字稿生成其他所需的输出 | |
source = "gcs" | |
questions = get_questions(video_id, formatted_simple_transcript, source) | |
formatted_transcript_json = json.dumps(formatted_transcript, ensure_ascii=False, indent=2) | |
summary_json = get_video_id_summary(video_id, formatted_simple_transcript, source) | |
summary = summary_json["summary"] | |
key_moments_json = get_key_moments(video_id, formatted_simple_transcript, formatted_transcript, source) | |
key_moments = key_moments_json["key_moments"] | |
key_moments_html = get_key_moments_html(key_moments) | |
html_content = format_transcript_to_html(formatted_transcript) | |
simple_html_content = format_simple_transcript_to_html(formatted_simple_transcript) | |
first_image = formatted_transcript[0]['screenshot_path'] | |
# first_image = "https://www.nameslook.com/names/dfsadf-nameslook.png" | |
first_text = formatted_transcript[0]['text'] | |
mind_map_json = get_mind_map(video_id, formatted_simple_transcript, source) | |
mind_map = mind_map_json["mind_map"] | |
mind_map_html = get_mind_map_html(mind_map) | |
reading_passage_json = get_reading_passage(video_id, formatted_simple_transcript, source) | |
reading_passage = reading_passage_json["reading_passage"] | |
meta_data = get_meta_data(video_id) | |
subject = meta_data["subject"] | |
grade = meta_data["grade"] | |
# 确保返回与 UI 组件预期匹配的输出 | |
return video_id, \ | |
questions[0] if len(questions) > 0 else "", \ | |
questions[1] if len(questions) > 1 else "", \ | |
questions[2] if len(questions) > 2 else "", \ | |
formatted_transcript_json, \ | |
summary, \ | |
key_moments_html, \ | |
mind_map, \ | |
mind_map_html, \ | |
html_content, \ | |
simple_html_content, \ | |
first_image, \ | |
first_text, \ | |
reading_passage, \ | |
subject, \ | |
grade | |
def format_transcript_to_html(formatted_transcript): | |
html_content = "" | |
for entry in formatted_transcript: | |
html_content += f"<h3>{entry['start_time']} - {entry['end_time']}</h3>" | |
html_content += f"<p>{entry['text']}</p>" | |
html_content += f"<img src='{entry['screenshot_path']}' width='500px' />" | |
return html_content | |
def format_simple_transcript_to_html(formatted_transcript): | |
html_content = "" | |
for entry in formatted_transcript: | |
html_content += f"<h3>{entry['start_time']} - {entry['end_time']}</h3>" | |
html_content += f"<p>{entry['text']}</p>" | |
return html_content | |
def get_embedded_youtube_link(video_id, start_time): | |
int_start_time = int(start_time) | |
embed_url = f"https://www.youtube.com/embed/{video_id}?start={int_start_time}&autoplay=1" | |
return embed_url | |
def download_youtube_video(youtube_id, output_path=OUTPUT_PATH): | |
# Construct the full YouTube URL | |
youtube_url = f'https://www.youtube.com/watch?v={youtube_id}' | |
# Create the output directory if it doesn't exist | |
if not os.path.exists(output_path): | |
os.makedirs(output_path) | |
# Download the video | |
yt = YouTube(youtube_url) | |
video_stream = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first() | |
video_stream.download(output_path=output_path, filename=youtube_id+".mp4") | |
print(f"Video downloaded successfully: {output_path}/{youtube_id}.mp4") | |
def screenshot_youtube_video(youtube_id, snapshot_sec): | |
video_path = f'{OUTPUT_PATH}/{youtube_id}.mp4' | |
file_name = f"{youtube_id}_{snapshot_sec}.jpg" | |
with VideoFileClip(video_path) as video: | |
screenshot_path = f'{OUTPUT_PATH}/{file_name}' | |
video.save_frame(screenshot_path, snapshot_sec) | |
return screenshot_path | |
# ---- Web ---- | |
def process_web_link(link): | |
# 抓取和解析网页内容 | |
response = requests.get(link) | |
soup = BeautifulSoup(response.content, 'html.parser') | |
return soup.get_text() | |
# ---- LLM Generator ---- | |
def get_reading_passage(video_id, df_string, source): | |
if source == "gcs": | |
print("===get_reading_passage on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_reading_passage.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查 reading_passage 是否存在 | |
is_file_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_file_exists: | |
reading_passage = generate_reading_passage(df_string) | |
reading_passage_json = {"reading_passage": str(reading_passage)} | |
reading_passage_text = json.dumps(reading_passage_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, reading_passage_text) | |
print("reading_passage已上传到GCS") | |
else: | |
# reading_passage已存在,下载内容 | |
print("reading_passage已存在于GCS中") | |
reading_passage_text = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
reading_passage_json = json.loads(reading_passage_text) | |
elif source == "drive": | |
print("===get_reading_passage on drive===") | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
file_name = f'{video_id}_reading_passage.json' | |
# 检查 reading_passage 是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
reading_passage = generate_reading_passage(df_string) | |
reading_passage_json = {"reading_passage": str(reading_passage)} | |
reading_passage_text = json.dumps(reading_passage_json, ensure_ascii=False, indent=2) | |
upload_content_directly(service, file_name, folder_id, reading_passage_text) | |
print("reading_passage已上傳到Google Drive") | |
else: | |
# reading_passage已存在,下载内容 | |
print("reading_passage已存在于Google Drive中") | |
reading_passage_text = download_file_as_string(service, file_id) | |
return reading_passage_json | |
def generate_reading_passage(df_string): | |
# 使用 OpenAI 生成基于上传数据的问题 | |
sys_content = "你是一個擅長資料分析跟影片教學的老師,user 為學生,請精讀資料文本,自行判斷資料的種類,使用 zh-TW" | |
user_content = f""" | |
請根據 {df_string} | |
文本自行判斷資料的種類 | |
幫我組合成 Reading Passage | |
並潤稿讓文句通順 | |
請一定要使用繁體中文 zh-TW,並用台灣人的口語 | |
產生的結果不要前後文解釋,也不要敘述這篇文章怎麼產生的 | |
只需要專注提供 Reading Passage,字數在 500 字以內 | |
""" | |
messages = [ | |
{"role": "system", "content": sys_content}, | |
{"role": "user", "content": user_content} | |
] | |
request_payload = { | |
"model": "gpt-4-1106-preview", | |
"messages": messages, | |
"max_tokens": 4000, | |
} | |
response = OPEN_AI_CLIENT.chat.completions.create(**request_payload) | |
reading_passage = response.choices[0].message.content.strip() | |
print("=====reading_passage=====") | |
print(reading_passage) | |
print("=====reading_passage=====") | |
return reading_passage | |
def text_to_speech(video_id, text): | |
tts = gTTS(text, lang='en') | |
filename = f'{video_id}_reading_passage.mp3' | |
tts.save(filename) | |
return filename | |
def get_mind_map(video_id, df_string, source): | |
if source == "gcs": | |
print("===get_mind_map on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_mind_map.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查檔案是否存在 | |
is_file_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_file_exists: | |
mind_map = generate_mind_map(df_string) | |
mind_map_json = {"mind_map": str(mind_map)} | |
mind_map_text = json.dumps(mind_map_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, mind_map_text) | |
print("mind_map已上傳到GCS") | |
else: | |
# mindmap已存在,下载内容 | |
print("mind_map已存在于GCS中") | |
mind_map_text = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
mind_map_json = json.loads(mind_map_text) | |
elif source == "drive": | |
print("===get_mind_map on drive===") | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
file_name = f'{video_id}_mind_map.json' | |
# 检查檔案是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
mind_map = generate_mind_map(df_string) | |
mind_map_json = {"mind_map": str(mind_map)} | |
mind_map_text = json.dumps(mind_map_json, ensure_ascii=False, indent=2) | |
upload_content_directly(service, file_name, folder_id, mind_map_text) | |
print("mind_map已上傳到Google Drive") | |
else: | |
# mindmap已存在,下载内容 | |
print("mind_map已存在于Google Drive中") | |
mind_map_text = download_file_as_string(service, file_id) | |
mind_map_json = json.loads(mind_map_text) | |
return mind_map_json | |
def generate_mind_map(df_string): | |
# 使用 OpenAI 生成基于上传数据的问题 | |
sys_content = "你是一個擅長資料分析跟影片教學的老師,user 為學生,請精讀資料文本,自行判斷資料的種類,使用 zh-TW" | |
user_content = f""" | |
請根據 {df_string} 文本建立 markdown 心智圖 | |
注意:不需要前後文敘述,直接給出 markdown 文本即可 | |
這對我很重要 | |
""" | |
messages = [ | |
{"role": "system", "content": sys_content}, | |
{"role": "user", "content": user_content} | |
] | |
request_payload = { | |
"model": "gpt-4-1106-preview", | |
"messages": messages, | |
"max_tokens": 4000, | |
} | |
response = OPEN_AI_CLIENT.chat.completions.create(**request_payload) | |
mind_map = response.choices[0].message.content.strip() | |
print("=====mind_map=====") | |
print(mind_map) | |
print("=====mind_map=====") | |
return mind_map | |
def get_mind_map_html(mind_map): | |
mind_map_markdown = mind_map.replace("```markdown", "").replace("```", "") | |
mind_map_html = f""" | |
<div class="markmap"> | |
<script type="text/template"> | |
{mind_map_markdown} | |
</script> | |
</div> | |
""" | |
return mind_map_html | |
def get_video_id_summary(video_id, df_string, source): | |
if source == "gcs": | |
print("===get_video_id_summary on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_summary.json' | |
summary_file_blob_name = f"{video_id}/{file_name}" | |
# 检查 summary_file 是否存在 | |
is_summary_file_exists = GCS_SERVICE.check_file_exists(bucket_name, summary_file_blob_name) | |
if not is_summary_file_exists: | |
summary = generate_summarise(df_string) | |
summary_json = {"summary": str(summary)} | |
summary_text = json.dumps(summary_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, summary_file_blob_name, summary_text) | |
print("summary已上传到GCS") | |
else: | |
# summary已存在,下载内容 | |
print("summary已存在于GCS中") | |
summary_text = download_blob_to_string(gcs_client, bucket_name, summary_file_blob_name) | |
summary_json = json.loads(summary_text) | |
elif source == "drive": | |
print("===get_video_id_summary===") | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
file_name = f'{video_id}_summary.json' | |
# 检查逐字稿是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
summary = generate_summarise(df_string) | |
summary_json = {"summary": str(summary)} | |
summary_text = json.dumps(summary_json, ensure_ascii=False, indent=2) | |
try: | |
upload_content_directly(service, file_name, folder_id, summary_text) | |
print("summary已上傳到Google Drive") | |
except Exception as e: | |
error_msg = f" {video_id} 摘要錯誤: {str(e)}" | |
print("===get_video_id_summary error===") | |
print(error_msg) | |
print("===get_video_id_summary error===") | |
else: | |
# 逐字稿已存在,下载逐字稿内容 | |
print("summary已存在Google Drive中") | |
summary_text = download_file_as_string(service, file_id) | |
summary_json = json.loads(summary_text) | |
return summary_json | |
def generate_summarise(df_string): | |
# 使用 OpenAI 生成基于上传数据的问题 | |
sys_content = "你是一個擅長資料分析跟影片教學的老師,user 為學生,請精讀資料文本,自行判斷資料的種類,使用 zh-TW" | |
user_content = f""" | |
請根據 {df_string},判斷這份文本 | |
如果是資料類型,請提估欄位敘述、資料樣態與資料分析,告訴學生這張表的意義,以及可能的結論與對應方式 | |
如果是影片類型,請提估影片內容,告訴學生這部影片的意義, | |
整體摘要在一百字以內 | |
小範圍切出不同段落的相對應時間軸的重點摘要,最多不超過五段 | |
注意不要遺漏任何一段時間軸的內容 | |
格式為 【start - end】: 摘要 | |
以及可能的結論與結尾延伸小問題提供學生作反思 | |
整體格式為: | |
🗂️ 1. 內容類型:? | |
📚 2. 整體摘要 | |
🔖 3. 重點概念 | |
🔑 4. 關鍵時刻 | |
💡 5. 為什麼我們要學這個? | |
❓ 6. 延伸小問題 | |
""" | |
# 🗂️ 1. 內容類型:? | |
# 📚 2. 整體摘要 | |
# 🔖 3. 條列式重點 | |
# 🔑 4. 關鍵時刻(段落摘要) | |
# 💡 5. 結論反思(為什麼我們要學這個?) | |
# ❓ 6. 延伸小問題 | |
messages = [ | |
{"role": "system", "content": sys_content}, | |
{"role": "user", "content": user_content} | |
] | |
request_payload = { | |
"model": "gpt-4-turbo-preview", | |
"messages": messages, | |
"max_tokens": 4000, | |
} | |
response = OPEN_AI_CLIENT.chat.completions.create(**request_payload) | |
df_summarise = response.choices[0].message.content.strip() | |
print("=====df_summarise=====") | |
print(df_summarise) | |
print("=====df_summarise=====") | |
return df_summarise | |
def get_questions(video_id, df_string, source="gcs"): | |
if source == "gcs": | |
# 去 gcs 確認是有有 video_id_questions.json | |
print("===get_questions on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_questions.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查檔案是否存在 | |
is_questions_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_questions_exists: | |
questions = generate_questions(df_string) | |
questions_text = json.dumps(questions, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, questions_text) | |
print("questions已上傳到GCS") | |
else: | |
# 逐字稿已存在,下载逐字稿内容 | |
print("questions已存在于GCS中") | |
questions_text = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
questions = json.loads(questions_text) | |
elif source == "drive": | |
# 去 g drive 確認是有有 video_id_questions.json | |
print("===get_questions===") | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
file_name = f'{video_id}_questions.json' | |
# 检查檔案是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
questions = generate_questions(df_string) | |
questions_text = json.dumps(questions, ensure_ascii=False, indent=2) | |
upload_content_directly(service, file_name, folder_id, questions_text) | |
print("questions已上傳到Google Drive") | |
else: | |
# 逐字稿已存在,下载逐字稿内容 | |
print("questions已存在于Google Drive中") | |
questions_text = download_file_as_string(service, file_id) | |
questions = json.loads(questions_text) | |
q1 = questions[0] if len(questions) > 0 else "" | |
q2 = questions[1] if len(questions) > 1 else "" | |
q3 = questions[2] if len(questions) > 2 else "" | |
print("=====get_questions=====") | |
print(f"q1: {q1}") | |
print(f"q2: {q2}") | |
print(f"q3: {q3}") | |
print("=====get_questions=====") | |
return q1, q2, q3 | |
def generate_questions(df_string): | |
# 使用 OpenAI 生成基于上传数据的问题 | |
sys_content = "你是一個擅長資料分析跟影片教學的老師,user 為學生,請精讀資料文本,自行判斷資料的種類,並用既有資料為本質猜測用戶可能會問的問題,使用 zh-TW" | |
user_content = f"請根據 {df_string} 生成三個問題,並用 JSON 格式返回 questions:[q1的敘述text, q2的敘述text, q3的敘述text]" | |
messages = [ | |
{"role": "system", "content": sys_content}, | |
{"role": "user", "content": user_content} | |
] | |
response_format = { "type": "json_object" } | |
print("=====messages=====") | |
print(messages) | |
print("=====messages=====") | |
request_payload = { | |
"model": "gpt-4-1106-preview", | |
"messages": messages, | |
"max_tokens": 4000, | |
"response_format": response_format | |
} | |
response = OPEN_AI_CLIENT.chat.completions.create(**request_payload) | |
questions = json.loads(response.choices[0].message.content)["questions"] | |
print("=====json_response=====") | |
print(questions) | |
print("=====json_response=====") | |
return questions | |
def change_questions(password, df_string): | |
verify_password(password) | |
questions = generate_questions(df_string) | |
q1 = questions[0] if len(questions) > 0 else "" | |
q2 = questions[1] if len(questions) > 1 else "" | |
q3 = questions[2] if len(questions) > 2 else "" | |
print("=====get_questions=====") | |
print(f"q1: {q1}") | |
print(f"q2: {q2}") | |
print(f"q3: {q3}") | |
print("=====get_questions=====") | |
return q1, q2, q3 | |
# 「關鍵時刻」另外獨立成一個 tab,時間戳記和文字的下方附上對應的截圖,重點摘要的「關鍵時刻」加上截圖資訊 | |
def get_key_moments(video_id, formatted_simple_transcript, formatted_transcript, source): | |
if source == "gcs": | |
print("===get_key_moments on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_key_moments.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查檔案是否存在 | |
is_key_moments_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_key_moments_exists: | |
key_moments = generate_key_moments(formatted_simple_transcript, formatted_transcript) | |
key_moments_json = {"key_moments": key_moments} | |
key_moments_text = json.dumps(key_moments_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, key_moments_text) | |
print("key_moments已上傳到GCS") | |
else: | |
# key_moments已存在,下载内容 | |
print("key_moments已存在于GCS中") | |
key_moments_text = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
key_moments_json = json.loads(key_moments_text) | |
elif source == "drive": | |
print("===get_key_moments on drive===") | |
service = init_drive_service() | |
parent_folder_id = '1GgI4YVs0KckwStVQkLa1NZ8IpaEMurkL' | |
folder_id = create_folder_if_not_exists(service, video_id, parent_folder_id) | |
file_name = f'{video_id}_key_moments.json' | |
# 检查檔案是否存在 | |
exists, file_id = check_file_exists(service, folder_id, file_name) | |
if not exists: | |
key_moments = generate_key_moments(formatted_simple_transcript, formatted_transcript) | |
key_moments_json = {"key_moments": key_moments} | |
key_moments_text = json.dumps(key_moments_json, ensure_ascii=False, indent=2) | |
upload_content_directly(service, file_name, folder_id, key_moments_text) | |
print("key_moments已上傳到Google Drive") | |
else: | |
# key_moments已存在,下载内容 | |
print("key_moments已存在于Google Drive中") | |
key_moments_text = download_file_as_string(service, file_id) | |
key_moments_json = json.loads(key_moments_text) | |
return key_moments_json | |
def generate_key_moments(formatted_simple_transcript, formatted_transcript): | |
# 使用 OpenAI 生成基于上传数据的问题 | |
sys_content = "你是一個擅長資料分析跟影片教學的老師,user 為學生,請精讀資料文本,自行判斷資料的種類,使用 zh-TW" | |
user_content = f""" | |
請根據 {formatted_simple_transcript} 文本,提取出重點摘要,並給出對應的時間軸 | |
重點摘要的「關鍵時刻」加上截圖資訊 | |
1. 小範圍切出不同段落的相對應時間軸的重點摘要, | |
2. 每一小段最多不超過 1/5 的總內容(例如五分鐘的影片就一段不超過一分鐘,10分鐘就一段最多兩分鐘) | |
3. 注意不要遺漏任何一段時間軸的內容 從零秒開始 | |
4. 如果頭尾的情節不是重點,就併入到附近的段落,特別是打招呼或是介紹人物就是不重要的情節 | |
以這種方式分析整個文本,從零秒開始分析,直到結束。這很重要 | |
並用 JSON 格式返回 key_moments:[{{ | |
"start": "00:00", | |
"end": "00:00", | |
"text": "逐字稿的重點摘要", | |
"transcript": "逐字稿的集合(要有合理的標點符號)", | |
"images": 截圖的連結們 list | |
}}] | |
""" | |
messages = [ | |
{"role": "system", "content": sys_content}, | |
{"role": "user", "content": user_content} | |
] | |
response_format = { "type": "json_object" } | |
request_payload = { | |
"model": "gpt-4-1106-preview", | |
"messages": messages, | |
"max_tokens": 4000, | |
"response_format": response_format | |
} | |
response = OPEN_AI_CLIENT.chat.completions.create(**request_payload) | |
key_moments = json.loads(response.choices[0].message.content)["key_moments"] | |
print("=====key_moments=====") | |
print(key_moments) | |
print("=====key_moments=====") | |
image_links = {entry['start_time']: entry['screenshot_path'] for entry in formatted_transcript} | |
for moment in key_moments: | |
start_time = moment['start'] | |
end_time = moment['end'] | |
moment_images = [image_links[time] for time in image_links if start_time <= time <= end_time] | |
moment['images'] = moment_images | |
return key_moments | |
def get_key_moments_html(key_moments): | |
css = """ | |
<style> | |
#gallery-main { | |
display: flex; | |
align-items: center; | |
margin-bottom: 20px; | |
} | |
#gallery { | |
position: relative; | |
width: 50%; | |
flex: 1; | |
} | |
#text-content { | |
flex: 2; | |
margin-left: 20px; | |
} | |
#gallery #gallery-container{ | |
position: relative; | |
width: 100%; | |
height: 0px; | |
padding-bottom: 56.7%; /* 16/9 ratio */ | |
background-color: blue; | |
} | |
#gallery #gallery-container #gallery-content{ | |
position: absolute; | |
top: 0px; | |
right: 0px; | |
bottom: 0px; | |
left: 0px; | |
height: 100%; | |
display: flex; | |
scroll-snap-type: x mandatory; | |
overflow-x: scroll; | |
scroll-behavior: smooth; | |
} | |
#gallery #gallery-container #gallery-content .gallery__item{ | |
width: 100%; | |
height: 100%; | |
flex-shrink: 0; | |
scroll-snap-align: start; | |
scroll-snap-stop: always; | |
position: relative; | |
} | |
#gallery #gallery-container #gallery-content .gallery__item img{ | |
display: block; | |
width: 100%; | |
height: 100%; | |
object-fit: contain; | |
background-color: white; | |
} | |
.click-zone{ | |
position: absolute; | |
width: 20%; | |
height: 100%; | |
z-index: 3; | |
} | |
.click-zone.click-zone-prev{ | |
left: 0px; | |
} | |
.click-zone.click-zone-next{ | |
right: 0px; | |
} | |
#gallery:not(:hover) .arrow{ | |
opacity: 0.8; | |
} | |
.arrow{ | |
text-align: center; | |
z-index: 3; | |
position: absolute; | |
display: block; | |
width: 25px; | |
height: 25px; | |
line-height: 25px; | |
background-color: black; | |
border-radius: 50%; | |
text-decoration: none; | |
color: black; | |
opacity: 0.8; | |
transition: opacity 200ms ease; | |
} | |
.arrow:hover{ | |
opacity: 1; | |
} | |
.arrow span{ | |
position: relative; | |
top: 2px; | |
} | |
.arrow.arrow-prev{ | |
top: 50%; | |
left: 5px; | |
} | |
.arrow.arrow-next{ | |
top: 50%; | |
right: 5px; | |
} | |
.arrow.arrow-disabled{ | |
opacity:0.8; | |
} | |
#text-content { | |
padding: 0px 36px; | |
} | |
#text-content p { | |
margin-top: 10px; | |
} | |
body{ | |
font-family: sans-serif; | |
margin: 0px; | |
padding: 0px; | |
} | |
main{ | |
padding: 0px; | |
margin: 0px; | |
max-width: 900px; | |
margin: auto; | |
} | |
.hidden{ | |
border: 0; | |
clip: rect(0 0 0 0); | |
height: 1px; | |
margin: -1px; | |
overflow: hidden; | |
padding: 0; | |
position: absolute; | |
width: 1px; | |
} | |
</style> | |
""" | |
key_moments_html = css | |
for i, moment in enumerate(key_moments): | |
images = moment['images'] | |
image_elements = "" | |
for j, image in enumerate(images): | |
current_id = f"img_{i}_{j}" | |
prev_id = f"img_{i}_{j-1}" if j-1 >= 0 else f"img_{i}_{len(images)-1}" | |
next_id = f"img_{i}_{j+1}" if j+1 < len(images) else f"img_{i}_0" | |
image_elements += f""" | |
<div id="{current_id}" class="gallery__item"> | |
<a href="#{prev_id}" class="click-zone click-zone-prev"> | |
<div class="arrow arrow-disabled arrow-prev"> < </div> | |
</a> | |
<a href="#{next_id}" class="click-zone click-zone-next"> | |
<div class="arrow arrow-next"> > </div> | |
</a> | |
<img src="{image}"> | |
</div> | |
""" | |
gallery_content = f""" | |
<div id="gallery-content"> | |
{image_elements} | |
</div> | |
""" | |
key_moments_html += f""" | |
<div class="gallery-container" id="gallery-main"> | |
<div id="gallery"><!-- gallery start --> | |
<div id="gallery-container"> | |
{gallery_content} | |
</div> | |
</div> | |
<div id="text-content"> | |
<h3>{moment['start']} - {moment['end']}</h3> | |
<p><strong>摘要: {moment['text']} </strong></p> | |
<p>內容: {moment['transcript']}</p> | |
</div> | |
</div> | |
""" | |
return key_moments_html | |
# ---- LLM CRUD ---- | |
def enable_edit_mode(): | |
return gr.update(interactive=True) | |
def delete_LLM_content(video_id, kind): | |
print(f"===delete_{kind}===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_{kind}.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查 reading_passage 是否存在 | |
is_file_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if is_file_exists: | |
delete_blob(gcs_client, bucket_name, blob_name) | |
print("reading_passage已从GCS中删除") | |
return gr.update(value="", interactive=False) | |
def update_LLM_content(video_id, new_content, kind): | |
print(f"===upfdate kind on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_{kind}.json' | |
blob_name = f"{video_id}/{file_name}" | |
if kind == "reading_passage": | |
reading_passage_json = {"reading_passage": str(new_content)} | |
reading_passage_text = json.dumps(reading_passage_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, reading_passage_text) | |
elif kind == "summary": | |
summary_json = {"summary": str(new_content)} | |
summary_text = json.dumps(summary_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, summary_text) | |
elif kind == "mind_map": | |
mind_map_json = {"mind_map": str(new_content)} | |
mind_map_text = json.dumps(mind_map_json, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, mind_map_text) | |
print(f"{kind} 已更新到GCS") | |
return gr.update(value=new_content, interactive=False) | |
def create_LLM_content(video_id, df_string, kind): | |
print(f"===create_{kind}===") | |
if kind == "reading_passage": | |
content = generate_reading_passage(df_string) | |
elif kind == "summary": | |
content = generate_summarise(df_string) | |
elif kind == "mind_map": | |
content = generate_mind_map(df_string) | |
update_LLM_content(video_id, content, kind) | |
return gr.update(value=content, interactive=False) | |
# AI 生成教學素材 | |
def get_meta_data(video_id, source="gcs"): | |
if source == "gcs": | |
print("===get_meta_data on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_meta_data.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查檔案是否存在 | |
is_file_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_file_exists: | |
meta_data_json = { | |
"subject": "", | |
"grade": "", | |
} | |
print("meta_data empty return") | |
else: | |
# meta_data已存在,下载内容 | |
print("meta_data已存在于GCS中") | |
meta_data_text = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
meta_data_json = json.loads(meta_data_text) | |
# meta_data_json grade 數字轉換成文字 | |
grade = meta_data_json["grade"] | |
case = { | |
1: "一年級", | |
2: "二年級", | |
3: "三年級", | |
4: "四年級", | |
5: "五年級", | |
6: "六年級", | |
7: "七年級", | |
8: "八年級", | |
9: "九年級", | |
10: "十年級", | |
11: "十一年級", | |
12: "十二年級", | |
} | |
grade_text = case.get(grade, "") | |
meta_data_json["grade"] = grade_text | |
return meta_data_json | |
def get_ai_content(password, video_id, df_string, topic, grade, level, specific_feature, content_type, source="gcs"): | |
verify_password(password) | |
if source == "gcs": | |
print("===get_ai_content on gcs===") | |
gcs_client = GCS_CLIENT | |
bucket_name = 'video_ai_assistant' | |
file_name = f'{video_id}_ai_content_list.json' | |
blob_name = f"{video_id}/{file_name}" | |
# 检查檔案是否存在 | |
is_file_exists = GCS_SERVICE.check_file_exists(bucket_name, blob_name) | |
if not is_file_exists: | |
# 先建立一個 ai_content_list.json | |
ai_content_list = [] | |
ai_content_text = json.dumps(ai_content_list, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, ai_content_text) | |
print("ai_content_list [] 已上傳到GCS") | |
# 此時 ai_content_list 已存在 | |
ai_content_list_string = download_blob_to_string(gcs_client, bucket_name, blob_name) | |
ai_content_list = json.loads(ai_content_list_string) | |
# by key 找到 ai_content (topic, grade, level, specific_feature, content_type) | |
target_kvs = { | |
"video_id": video_id, | |
"level": level, | |
"specific_feature": specific_feature, | |
"content_type": content_type | |
} | |
ai_content_json = [ | |
item for item in ai_content_list | |
if all(item[k] == v for k, v in target_kvs.items()) | |
] | |
if len(ai_content_json) == 0: | |
ai_content, prompt = generate_ai_content(password, df_string, topic, grade, level, specific_feature, content_type) | |
ai_content_json = { | |
"video_id": video_id, | |
"content": str(ai_content), | |
"prompt": prompt, | |
"level": level, | |
"specific_feature": specific_feature, | |
"content_type": content_type | |
} | |
ai_content_list.append(ai_content_json) | |
ai_content_text = json.dumps(ai_content_list, ensure_ascii=False, indent=2) | |
upload_file_to_gcs_with_json_string(gcs_client, bucket_name, blob_name, ai_content_text) | |
print("ai_content已上傳到GCS") | |
else: | |
ai_content_json = ai_content_json[-1] | |
ai_content = ai_content_json["content"] | |
prompt = ai_content_json["prompt"] | |
return ai_content, ai_content, prompt, prompt | |
def generate_ai_content(password, df_string, topic, grade, level, specific_feature, content_type): | |
verify_password(password) | |
material = EducationalMaterial(df_string, topic, grade, level, specific_feature, content_type) | |
prompt = material.generate_content_prompt() | |
user_content = material.build_user_content() | |
messages = material.build_messages(user_content) | |
ai_model_name = "gpt-4-1106-preview" | |
request_payload = { | |
"model": ai_model_name, | |
"messages": messages, | |
"max_tokens": 4000 # 举例,实际上您可能需要更详细的配置 | |
} | |
ai_content = material.send_ai_request(OPEN_AI_CLIENT, request_payload) | |
return ai_content, prompt | |
def generate_exam_fine_tune_result(password, exam_result_prompt , df_string_output, exam_result, exam_result_fine_tune_prompt): | |
verify_password(password) | |
material = EducationalMaterial(df_string_output, "", "", "", "", "") | |
user_content = material.build_fine_tune_user_content(exam_result_prompt, exam_result, exam_result_fine_tune_prompt) | |
messages = material.build_messages(user_content) | |
ai_model_name = "gpt-4-1106-preview" | |
request_payload = { | |
"model": ai_model_name, | |
"messages": messages, | |
"max_tokens": 4000 # 举例,实际上您可能需要更详细的配置 | |
} | |
ai_content = material.send_ai_request(OPEN_AI_CLIENT, request_payload) | |
return ai_content | |
def return_original_exam_result(exam_result_original): | |
return exam_result_original | |
def create_word(content): | |
unique_filename = str(uuid.uuid4()) | |
word_file_path = f"/tmp/{unique_filename}.docx" | |
doc = Document() | |
doc.add_paragraph(content) | |
doc.save(word_file_path) | |
return word_file_path | |
def download_exam_result(content): | |
word_path = create_word(content) | |
return word_path | |
# ---- Chatbot ---- | |
def chat_with_ai(ai_name, password, video_id, trascript, user_message, chat_history, content_subject, content_grade, socratic_mode=False): | |
verify_password(password) | |
if chat_history is not None and len(chat_history) > 10: | |
error_msg = "此次對話超過上限" | |
raise gr.Error(error_msg) | |
if ai_name == "jutor": | |
ai_client = "" | |
elif ai_name == "claude3": | |
ai_client = BEDROCK_CLIENT | |
elif ai_name == "groq": | |
ai_client = GROQ_CLIENT | |
chatbot_config = { | |
"video_id": video_id, | |
"trascript": trascript, | |
"content_subject": content_subject, | |
"content_grade": content_grade, | |
"jutor_chat_key": JUTOR_CHAT_KEY, | |
"ai_name": ai_name, | |
"ai_client": ai_client | |
} | |
chatbot = Chatbot(chatbot_config) | |
response_completion = chatbot.chat(user_message, chat_history, socratic_mode, ai_name) | |
try: | |
# 更新聊天历史 | |
new_chat_history = (user_message, response_completion) | |
if chat_history is None: | |
chat_history = [new_chat_history] | |
else: | |
chat_history.append(new_chat_history) | |
# 返回聊天历史和空字符串清空输入框 | |
return "", chat_history | |
except Exception as e: | |
# 处理错误情况 | |
print(f"Error: {e}") | |
return "请求失败,请稍后再试!", chat_history | |
def chat_with_opan_ai_assistant(password, youtube_id, thread_id, trascript, user_message, chat_history, content_subject, content_grade, socratic_mode=False): | |
verify_password(password) | |
# 先計算 user_message 是否超過 500 個字 | |
if len(user_message) > 1500: | |
error_msg = "你的訊息太長了,請縮短訊息長度至五百字以內" | |
raise gr.Error(error_msg) | |
# 如果 chat_history 超過 10 則訊息,直接 return "對話超過上限" | |
if chat_history is not None and len(chat_history) > 10: | |
error_msg = "此次對話超過上限" | |
raise gr.Error(error_msg) | |
try: | |
assistant_id = "asst_kmvZLNkDUYaNkMNtZEAYxyPq" | |
client = OPEN_AI_CLIENT | |
# 直接安排逐字稿資料 in instructions | |
trascript_json = json.loads(trascript) | |
# 移除 embed_url, screenshot_path | |
for entry in trascript_json: | |
entry.pop('embed_url', None) | |
entry.pop('screenshot_path', None) | |
trascript_text = json.dumps(trascript_json, ensure_ascii=False, indent=2) | |
instructions = f""" | |
科目:{content_subject} | |
年級:{content_grade} | |
逐字稿資料:{trascript_text} | |
------------------------------------- | |
你是一個專業的{content_subject}老師, user 為{content_grade}的學生 | |
socratic_mode = {socratic_mode} | |
if socratic_mode is True, | |
- 請用蘇格拉底式的提問方式,引導學生思考,並且給予學生一些提示 | |
- 一次只問一個問題,字數在100字以內 | |
- 不要直接給予答案,讓學生自己思考 | |
- 但可以給予一些提示跟引導,例如給予影片的時間軸,讓學生自己去找答案 | |
if socratic_mode is False, | |
- 直接回答學生問題,字數在100字以內 | |
rule: | |
- 請一定要用繁體中文回答 zh-TW,並用台灣人的口語表達,回答時不用特別說明這是台灣人的語氣,也不用說這是「台語的說法」 | |
- 不用提到「逐字稿」這個詞,用「內容」代替 | |
- 如果學生問了一些問題你無法判斷,請告訴學生你無法判斷,並建議學生可以問其他問題 | |
- 或者你可以反問學生一些問題,幫助學生更好的理解資料,字數在100字以內 | |
- 如果學生的問題與資料文本無關,請告訴學生你「無法回答超出影片範圍的問題」,並告訴他可以怎麼問什麼樣的問題(一個就好) | |
- 只要是參考逐字稿資料,請在回答的最後標註【參考資料:(分):(秒)】 | |
- 回答範圍一定要在逐字稿資料內,不要引用其他資料,請嚴格執行 | |
- 並在重複問句後給予學生鼓勵,讓學生有學習的動力 | |
- 請用 {content_grade} 的學生能懂的方式回答 | |
""" | |
# 创建线程 | |
if not thread_id: | |
thread = client.beta.threads.create() | |
thread_id = thread.id | |
else: | |
thread = client.beta.threads.retrieve(thread_id) | |
# 向线程添加用户的消息 | |
client.beta.threads.messages.create( | |
thread_id=thread.id, | |
role="user", | |
content=user_message + "/n (請一定要用繁體中文回答 zh-TW,並用台灣人的禮貌口語表達,回答時不要特別說明這是台灣人的語氣,不用提到「逐字稿」這個詞,用「內容」代替),回答時請用數學符號代替文字(Latex 用 $ 字號 render)" | |
) | |
# 运行助手,生成响应 | |
run = client.beta.threads.runs.create( | |
thread_id=thread.id, | |
assistant_id=assistant_id, | |
instructions=instructions, | |
) | |
# 等待助手响应,设定最大等待时间为 30 秒 | |
run_status = poll_run_status(run.id, thread.id, timeout=30) | |
# 获取助手的响应消息 | |
if run_status == "completed": | |
messages = client.beta.threads.messages.list(thread_id=thread.id) | |
# [MessageContentText(text=Text(annotations=[], value='您好!有什麼我可以幫助您的嗎?如果有任何問題或需要指導,請隨時告訴我!'), type='text')] | |
response_text = messages.data[0].content[0].text.value | |
else: | |
response_text = "學習精靈有點累,請稍後再試!" | |
# 更新聊天历史 | |
new_chat_history = (user_message, response_text) | |
if chat_history is None: | |
chat_history = [new_chat_history] | |
else: | |
chat_history.append(new_chat_history) | |
except Exception as e: | |
print(f"Error: {e}") | |
raise gr.Error(f"Error: {e}") | |
# 返回聊天历史和空字符串清空输入框 | |
return "", chat_history, thread.id | |
def process_open_ai_audio_to_chatbot(password, audio_url): | |
verify_password(password) | |
if audio_url: | |
with open(audio_url, "rb") as audio_file: | |
file_size = os.path.getsize(audio_url) | |
if file_size > 2000000: | |
raise gr.Error("檔案大小超過,請不要超過 60秒") | |
else: | |
response = OPEN_AI_CLIENT.audio.transcriptions.create( | |
model="whisper-1", | |
file=audio_file, | |
response_format="text" | |
) | |
# response 拆解 dict | |
print("=== response ===") | |
print(response) | |
print("=== response ===") | |
else: | |
response = "" | |
return response | |
def poll_run_status(run_id, thread_id, timeout=600, poll_interval=5): | |
""" | |
Polls the status of a Run and handles different statuses appropriately. | |
:param run_id: The ID of the Run to poll. | |
:param thread_id: The ID of the Thread associated with the Run. | |
:param timeout: Maximum time to wait for the Run to complete, in seconds. | |
:param poll_interval: Time to wait between each poll, in seconds. | |
""" | |
client = OPEN_AI_CLIENT | |
start_time = time.time() | |
while time.time() - start_time < timeout: | |
run = client.beta.threads.runs.retrieve(thread_id=thread_id, run_id=run_id) | |
if run.status in ["completed", "cancelled", "failed"]: | |
print(f"Run completed with status: {run.status}") | |
break | |
elif run.status == "requires_action": | |
print("Run requires action. Performing required action...") | |
# Here, you would perform the required action, e.g., running functions | |
# and then submitting the outputs. This is simplified for this example. | |
# After performing the required action, you'd complete the action: | |
# OPEN_AI_CLIENT.beta.threads.runs.complete_required_action(...) | |
elif run.status == "expired": | |
print("Run expired. Exiting...") | |
break | |
else: | |
print(f"Run status is {run.status}. Waiting for updates...") | |
time.sleep(poll_interval) | |
else: | |
print("Timeout reached. Run did not complete in the expected time.") | |
# Once the Run is completed, handle the result accordingly | |
if run.status == "completed": | |
# Retrieve and handle messages or run steps as needed | |
messages = client.beta.threads.messages.list(thread_id=thread_id) | |
for message in messages.data: | |
if message.role == "assistant": | |
print(f"Assistant response: {message.content}") | |
elif run.status in ["cancelled", "failed"]: | |
# Handle cancellation or failure | |
print(f"Run ended with status: {run.status}") | |
elif run.status == "expired": | |
# Handle expired run | |
print("Run expired without completion.") | |
return run.status | |
# --- Slide mode --- | |
def update_slide(direction): | |
global TRANSCRIPTS | |
global CURRENT_INDEX | |
print("=== 更新投影片 ===") | |
print(f"CURRENT_INDEX: {CURRENT_INDEX}") | |
# print(f"TRANSCRIPTS: {TRANSCRIPTS}") | |
CURRENT_INDEX += direction | |
if CURRENT_INDEX < 0: | |
CURRENT_INDEX = 0 # 防止索引小于0 | |
elif CURRENT_INDEX >= len(TRANSCRIPTS): | |
CURRENT_INDEX = len(TRANSCRIPTS) - 1 # 防止索引超出范围 | |
# 获取当前条目的文本和截图 URL | |
current_transcript = TRANSCRIPTS[CURRENT_INDEX] | |
slide_image = current_transcript["screenshot_path"] | |
slide_text = current_transcript["text"] | |
return slide_image, slide_text | |
def prev_slide(): | |
return update_slide(-1) | |
def next_slide(): | |
return update_slide(1) | |
def init_params(text, request: gr.Request): | |
if request: | |
print("Request headers dictionary:", request.headers) | |
print("IP address:", request.client.host) | |
print("Query parameters:", dict(request.query_params)) | |
# url = request.url | |
print("Request URL:", request.url) | |
youtube_link = "" | |
password_text = "" | |
admin = gr.update(visible=True) | |
reading_passage_admin = gr.update(visible=True) | |
summary_admin = gr.update(visible=True) | |
see_detail = gr.update(visible=True) | |
# if youtube_link in query_params | |
if "youtube_id" in request.query_params: | |
youtube_id = request.query_params["youtube_id"] | |
youtube_link = f"https://www.youtube.com/watch?v={youtube_id}" | |
print(f"youtube_link: {youtube_link}") | |
# check if origin is from junyiacademy | |
origin = request.headers.get("origin", "") | |
if "junyiacademy" in origin: | |
password_text = "6161" | |
admin = gr.update(visible=False) | |
reading_passage_admin = gr.update(visible=False) | |
summary_admin = gr.update(visible=False) | |
see_detail = gr.update(visible=False) | |
return admin, reading_passage_admin, summary_admin, see_detail, password_text, youtube_link | |
HEAD = """ | |
<meta charset="UTF-8"> | |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
<style> | |
svg.markmap {{ | |
width: 100%; | |
height: 100vh; | |
}} | |
</style> | |
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script> | |
<script> | |
const mind_map_tab_button = document.querySelector("#mind_map_tab-button"); | |
if (mind_map_tab_button) { | |
mind_map_tab_button.addEventListener('click', function() { | |
const mind_map_markdown = document.querySelector("#mind_map_markdown > label > textarea"); | |
if (mind_map_markdown) { | |
// 当按钮被点击时,打印当前的textarea的值 | |
console.log('Value changed to: ' + mind_map_markdown.value); | |
markmap.autoLoader.renderAll(); | |
} | |
}); | |
} | |
</script> | |
<script> | |
function changeImage(direction, count, galleryIndex) { | |
// Find the current visible image by iterating over possible indices | |
var currentImage = null; | |
var currentIndex = -1; | |
for (var i = 0; i < count; i++) { | |
var img = document.querySelector('.slide-image-' + galleryIndex + '-' + i); | |
if (img && img.style.display !== 'none') { | |
currentImage = img; | |
currentIndex = i; | |
break; | |
} | |
} | |
// If no current image is visible, show the first one and return | |
if (currentImage === null) { | |
document.querySelector('.slide-image-' + galleryIndex + '-0').style.display = 'block'; | |
console.error('No current image found for galleryIndex ' + galleryIndex + ', defaulting to first image.'); | |
return; | |
} | |
// Hide the current image | |
currentImage.style.display = 'none'; | |
// Calculate the index of the next image to show | |
var newIndex = (currentIndex + direction + count) % count; | |
// Select the next image and show it | |
var nextImage = document.querySelector('.slide-image-' + galleryIndex + '-' + newIndex); | |
if (nextImage) { | |
nextImage.style.display = 'block'; | |
} else { | |
console.error('No image found for galleryIndex ' + galleryIndex + ' and newIndex ' + newIndex); | |
} | |
} | |
</script> | |
""" | |
with gr.Blocks(theme=gr.themes.Base(primary_hue=gr.themes.colors.orange, secondary_hue=gr.themes.colors.amber, text_size = gr.themes.sizes.text_lg), head=HEAD) as demo: | |
with gr.Row() as admin: | |
password = gr.Textbox(label="Password", type="password", elem_id="password_input", visible=True) | |
file_upload = gr.File(label="Upload your CSV or Word file", visible=False) | |
youtube_link = gr.Textbox(label="Enter YouTube Link", elem_id="youtube_link_input", visible=True) | |
video_id = gr.Textbox(label="video_id", visible=False) | |
web_link = gr.Textbox(label="Enter Web Page Link", visible=False) | |
user_data = gr.Textbox(label="User Data", elem_id="user_data_input", visible=True) | |
youtube_link_btn = gr.Button("Submit_YouTube_Link", elem_id="youtube_link_btn", visible=True) | |
with gr.Tab("AI小精靈"): | |
with gr.Row(): | |
with gr.Tab("飛特"): | |
bot_avatar = "https://junyi-avatar.s3.ap-northeast-1.amazonaws.com/live/%20%20foxcat-star-18.png?v=20231113095823614" | |
user_avatar = "https://junyitopicimg.s3.amazonaws.com/s4byy--icon.jpe?v=20200513013523726" | |
latex_delimiters = [{"left": "$", "right": "$", "display": False}] | |
chatbot = gr.Chatbot(avatar_images=[bot_avatar, user_avatar], label="OPEN AI", show_share_button=False, likeable=True, show_label=False, latex_delimiters=latex_delimiters) | |
thread_id = gr.Textbox(label="thread_id", visible=False) | |
socratic_mode_btn = gr.Checkbox(label="蘇格拉底家教助理模式", value=True, visible=False) | |
with gr.Row(): | |
with gr.Accordion("你也有類似的問題想問嗎?", open=False) as ask_questions_accordion: | |
btn_1 = gr.Button("問題一") | |
btn_2 = gr.Button("問題一") | |
btn_3 = gr.Button("問題一") | |
gr.Markdown("### 重新生成問題") | |
btn_create_question = gr.Button("生成其他問題", variant="primary") | |
openai_chatbot_audio_input = gr.Audio(sources=["microphone"], type="filepath") | |
with gr.Row(): | |
msg = gr.Textbox(label="訊息",scale=3) | |
send_button = gr.Button("送出", variant="primary", scale=1) | |
# with gr.Tab("GROQ"): | |
# groq_ai_name = gr.Textbox(label="AI 助理名稱", value="groq", visible=False) | |
# groq_chatbot = gr.Chatbot(avatar_images=[bot_avatar, user_avatar], label="groq mode chatbot", show_share_button=False, likeable=True) | |
# groq_msg = gr.Textbox(label="Message") | |
# groq_send_button = gr.Button("Send", variant="primary") | |
# with gr.Tab("JUTOR"): | |
# jutor_ai_name = gr.Textbox(label="AI 助理名稱", value="jutor", visible=False) | |
# jutor_chatbot = gr.Chatbot(avatar_images=[bot_avatar, user_avatar], label="jutor mode chatbot", show_share_button=False, likeable=True) | |
# jutor_msg = gr.Textbox(label="Message") | |
# jutor_send_button = gr.Button("Send", variant="primary") | |
# with gr.Tab("CLAUDE"): | |
# claude_ai_name = gr.Textbox(label="AI 助理名稱", value="claude3", visible=False) | |
# claude_chatbot = gr.Chatbot(avatar_images=[bot_avatar, user_avatar], label="claude mode chatbot", show_share_button=False, likeable=True) | |
# claude_msg = gr.Textbox(label="Message") | |
# claude_send_button = gr.Button("Send", variant="primary") | |
with gr.Tab("其他精靈"): | |
ai_name = gr.Dropdown(label="選擇 AI 助理", choices=["jutor", "claude3", "groq"], value="jutor") | |
ai_chatbot = gr.Chatbot(avatar_images=[bot_avatar, user_avatar], label="ai_chatbot", show_share_button=False, likeable=True, show_label=False) | |
ai_msg = gr.Textbox(label="Message") | |
ai_send_button = gr.Button("Send", variant="primary") | |
with gr.Tab("文章模式"): | |
with gr.Row() as reading_passage_admin: | |
reading_passage_kind = gr.Textbox(value="reading_passage", show_label=False) | |
reading_passage_edit_button = gr.Button("編輯", size="sm", variant="primary") | |
reading_passage_update_button = gr.Button("更新", size="sm", variant="primary") | |
reading_passage_delete_button = gr.Button("刪除", size="sm", variant="primary") | |
reading_passage_create_button = gr.Button("建立", size="sm", variant="primary") | |
with gr.Row(): | |
reading_passage = gr.Textbox(label="Reading Passage", lines=40, show_label=False) | |
reading_passage_speak_button = gr.Button("Speak", visible=False) | |
reading_passage_audio_output = gr.Audio(label="Audio Output", visible=False) | |
with gr.Tab("重點摘要"): | |
with gr.Row() as summary_admmin: | |
summary_kind = gr.Textbox(value="summary", show_label=False) | |
summary_edit_button = gr.Button("編輯", size="sm", variant="primary") | |
summary_update_button = gr.Button("更新", size="sm", variant="primary") | |
summary_delete_button = gr.Button("刪除", size="sm", variant="primary") | |
summary_create_button = gr.Button("建立", size="sm", variant="primary") | |
with gr.Row(): | |
df_summarise = gr.Textbox(container=True, show_copy_button=True, lines=40, show_label=False) | |
with gr.Tab("關鍵時刻"): | |
with gr.Row(): | |
key_moments_html = gr.HTML(value="") | |
with gr.Tab("教學備課"): | |
with gr.Row(): | |
content_subject = gr.Dropdown(label="選擇主題", choices=["數學", "自然", "國文", "英文", "社會","物理", "化學", "生物", "地理", "歷史", "公民"], value="", visible=False) | |
content_grade = gr.Dropdown(label="選擇年級", choices=["一年級", "二年級", "三年級", "四年級", "五年級", "六年級", "七年級", "八年級", "九年級", "十年級", "十一年級", "十二年級"], value="", visible=False) | |
content_level = gr.Dropdown(label="差異化教學", choices=["基礎", "中級", "進階"], value="基礎") | |
with gr.Row(): | |
with gr.Tab("學習單"): | |
with gr.Row(): | |
with gr.Column(scale=1): | |
with gr.Row(): | |
worksheet_content_type_name = gr.Textbox(value="worksheet", visible=False) | |
worksheet_algorithm = gr.Dropdown(label="選擇教學策略或理論", choices=["Bloom認知階層理論", "Polya數學解題法", "CRA教學法"], value="Bloom認知階層理論", visible=False) | |
worksheet_content_btn = gr.Button("生成學習單 📄", variant="primary") | |
with gr.Accordion("微調", open=False): | |
worksheet_exam_result_fine_tune_prompt = gr.Textbox(label="根據結果,輸入你想更改的想法") | |
worksheet_exam_result_fine_tune_btn = gr.Button("微調結果", variant="primary") | |
worksheet_exam_result_retrun_original = gr.Button("返回原始結果") | |
with gr.Accordion("prompt", open=False): | |
worksheet_prompt = gr.Textbox(label="worksheet_prompt", show_copy_button=True, lines=40) | |
with gr.Column(scale=2): | |
# 生成對應不同模式的結果 | |
worksheet_exam_result_prompt = gr.Textbox(visible=False) | |
worksheet_exam_result_original = gr.Textbox(visible=False) | |
worksheet_exam_result = gr.Textbox(label="初次生成結果", show_copy_button=True, interactive=True, lines=40) | |
worksheet_download_exam_result_button = gr.Button("轉成 word,完成後請點擊右下角 download 按鈕", variant="primary") | |
worksheet_exam_result_word_link = gr.File(label="Download Word") | |
with gr.Tab("課程計畫"): | |
with gr.Row(): | |
with gr.Column(scale=1): | |
with gr.Row(): | |
lesson_plan_content_type_name = gr.Textbox(value="lesson_plan", visible=False) | |
lesson_plan_time = gr.Slider(label="選擇課程時間(分鐘)", minimum=10, maximum=120, step=5, value=40) | |
lesson_plan_btn = gr.Button("生成課程計畫 📕", variant="primary") | |
with gr.Accordion("微調", open=False): | |
lesson_plan_exam_result_fine_tune_prompt = gr.Textbox(label="根據結果,輸入你想更改的想法") | |
lesson_plan_exam_result_fine_tune_btn = gr.Button("微調結果", variant="primary") | |
lesson_plan_exam_result_retrun_original = gr.Button("返回原始結果") | |
with gr.Accordion("prompt", open=False): | |
lesson_plan_prompt = gr.Textbox(label="worksheet_prompt", show_copy_button=True, lines=40) | |
with gr.Column(scale=2): | |
# 生成對應不同模式的結果 | |
lesson_plan_exam_result_prompt = gr.Textbox(visible=False) | |
lesson_plan_exam_result_original = gr.Textbox(visible=False) | |
lesson_plan_exam_result = gr.Textbox(label="初次生成結果", show_copy_button=True, interactive=True, lines=40) | |
lesson_plan_download_exam_result_button = gr.Button("轉成 word,完成後請點擊右下角 download 按鈕", variant="primary") | |
lesson_plan_exam_result_word_link = gr.File(label="Download Word") | |
with gr.Tab("出場券"): | |
with gr.Row(): | |
with gr.Column(scale=1): | |
with gr.Row(): | |
exit_ticket_content_type_name = gr.Textbox(value="exit_ticket", visible=False) | |
exit_ticket_time = gr.Slider(label="選擇出場券時間(分鐘)", minimum=5, maximum=10, step=1, value=8) | |
exit_ticket_btn = gr.Button("生成出場券 🎟️", variant="primary") | |
with gr.Accordion("微調", open=False): | |
exit_ticket_exam_result_fine_tune_prompt = gr.Textbox(label="根據結果,輸入你想更改的想法") | |
exit_ticket_exam_result_fine_tune_btn = gr.Button("微調結果", variant="primary") | |
exit_ticket_exam_result_retrun_original = gr.Button("返回原始結果") | |
with gr.Accordion("prompt", open=False): | |
exit_ticket_prompt = gr.Textbox(label="worksheet_prompt", show_copy_button=True, lines=40) | |
with gr.Column(scale=2): | |
# 生成對應不同模式的結果 | |
exit_ticket_exam_result_prompt = gr.Textbox(visible=False) | |
exit_ticket_exam_result_original = gr.Textbox(visible=False) | |
exit_ticket_exam_result = gr.Textbox(label="初次生成結果", show_copy_button=True, interactive=True, lines=40) | |
exit_ticket_download_exam_result_button = gr.Button("轉成 word,完成後請點擊右下角 download 按鈕", variant="primary") | |
exit_ticket_exam_result_word_link = gr.File(label="Download Word") | |
# with gr.Tab("素養導向閱讀題組"): | |
# literacy_oriented_reading_content = gr.Textbox(label="輸入閱讀材料") | |
# literacy_oriented_reading_content_btn = gr.Button("生成閱讀理解題") | |
# with gr.Tab("自我評估"): | |
# self_assessment_content = gr.Textbox(label="輸入自評問卷或檢查表") | |
# self_assessment_content_btn = gr.Button("生成自評問卷") | |
# with gr.Tab("自我反思評量"): | |
# self_reflection_content = gr.Textbox(label="輸入自我反思活動") | |
# self_reflection_content_btn = gr.Button("生成自我反思活動") | |
# with gr.Tab("後設認知"): | |
# metacognition_content = gr.Textbox(label="輸入後設認知相關問題") | |
# metacognition_content_btn = gr.Button("生成後設認知問題") | |
with gr.Accordion("See Details", open=False) as see_details: | |
with gr.Tab("本文"): | |
df_string_output = gr.Textbox(lines=40, label="Data Text") | |
with gr.Tab("逐字稿"): | |
simple_html_content = gr.HTML(label="Simple Transcript") | |
with gr.Tab("圖文"): | |
transcript_html = gr.HTML(label="YouTube Transcript and Video") | |
with gr.Tab("投影片"): | |
slide_image = gr.Image() | |
slide_text = gr.Textbox() | |
with gr.Row(): | |
prev_button = gr.Button("Previous") | |
next_button = gr.Button("Next") | |
prev_button.click(fn=prev_slide, inputs=[], outputs=[slide_image, slide_text]) | |
next_button.click(fn=next_slide, inputs=[], outputs=[slide_image, slide_text]) | |
with gr.Tab("markdown"): | |
gr.Markdown("## 請複製以下 markdown 並貼到你的心智圖工具中,建議使用:https://markmap.js.org/repl") | |
mind_map = gr.Textbox(container=True, show_copy_button=True, lines=40, elem_id="mind_map_markdown") | |
with gr.Tab("心智圖",elem_id="mind_map_tab"): | |
mind_map_html = gr.HTML() | |
# --- Event --- | |
# OPENAI 模式 | |
send_button.click( | |
chat_with_opan_ai_assistant, | |
inputs=[password, video_id, thread_id, df_string_output, msg, chatbot, content_subject, content_grade, socratic_mode_btn], | |
outputs=[msg, chatbot, thread_id] | |
) | |
openai_chatbot_audio_input.change( | |
process_open_ai_audio_to_chatbot, | |
inputs=[password, openai_chatbot_audio_input], | |
outputs=[msg] | |
) | |
# # GROQ 模式 | |
# groq_send_button.click( | |
# chat_with_ai, | |
# inputs=[groq_ai_name, password, video_id, df_string_output, groq_msg, groq_chatbot, content_subject, content_grade, socratic_mode_btn], | |
# outputs=[groq_msg, groq_chatbot] | |
# ) | |
# # JUTOR API 模式 | |
# jutor_send_button.click( | |
# chat_with_ai, | |
# inputs=[jutor_ai_name, password, video_id, df_string_output, jutor_msg, jutor_chatbot, content_subject, content_grade, socratic_mode_btn], | |
# outputs=[jutor_msg, jutor_chatbot] | |
# ) | |
# # CLAUDE 模式 | |
# claude_send_button.click( | |
# chat_with_ai, | |
# inputs=[claude_ai_name, password, video_id, df_string_output, claude_msg, claude_chatbot, content_subject, content_grade, socratic_mode_btn], | |
# outputs=[claude_msg, claude_chatbot] | |
# ) | |
# ai_chatbot 模式 | |
ai_send_button.click( | |
chat_with_ai, | |
inputs=[ai_name, password, video_id, df_string_output, ai_msg, ai_chatbot, content_subject, content_grade, socratic_mode_btn], | |
outputs=[ai_msg, ai_chatbot] | |
) | |
# 连接按钮点击事件 | |
btn_1_chat_with_opan_ai_assistant_input =[password, video_id, thread_id, df_string_output, btn_1, chatbot, content_subject, content_grade, socratic_mode_btn] | |
btn_2_chat_with_opan_ai_assistant_input =[password, video_id, thread_id, df_string_output, btn_2, chatbot, content_subject, content_grade, socratic_mode_btn] | |
btn_3_chat_with_opan_ai_assistant_input =[password, video_id, thread_id, df_string_output, btn_3, chatbot, content_subject, content_grade, socratic_mode_btn] | |
btn_1.click( | |
chat_with_opan_ai_assistant, | |
inputs=btn_1_chat_with_opan_ai_assistant_input, | |
outputs=[msg, chatbot, thread_id] | |
) | |
btn_2.click( | |
chat_with_opan_ai_assistant, | |
inputs=btn_2_chat_with_opan_ai_assistant_input, | |
outputs=[msg, chatbot, thread_id] | |
) | |
btn_3.click( | |
chat_with_opan_ai_assistant, | |
inputs=btn_3_chat_with_opan_ai_assistant_input, | |
outputs=[msg, chatbot, thread_id] | |
) | |
btn_create_question.click( | |
change_questions, | |
inputs = [password, df_string_output], | |
outputs = [btn_1, btn_2, btn_3] | |
) | |
# file_upload.change(process_file, inputs=file_upload, outputs=df_string_output) | |
file_upload.change(process_file, inputs=file_upload, outputs=[btn_1, btn_2, btn_3, df_summarise, df_string_output]) | |
# 当输入 YouTube 链接时触发 | |
process_youtube_link_output = [ | |
video_id, | |
btn_1, | |
btn_2, | |
btn_3, | |
df_string_output, | |
df_summarise, | |
key_moments_html, | |
mind_map, | |
mind_map_html, | |
transcript_html, | |
simple_html_content, | |
slide_image, | |
slide_text, | |
reading_passage, | |
content_subject, | |
content_grade, | |
] | |
youtube_link.change( | |
process_youtube_link, | |
inputs=[password,youtube_link], | |
outputs=process_youtube_link_output | |
) | |
youtube_link_btn.click( | |
process_youtube_link, | |
inputs=[password, youtube_link], | |
outputs=process_youtube_link_output | |
) | |
# 当输入网页链接时触发 | |
# web_link.change(process_web_link, inputs=web_link, outputs=[btn_1, btn_2, btn_3, df_summarise, df_string_output]) | |
# reading_passage event | |
reading_passage_create_button.click( | |
create_LLM_content, | |
inputs=[video_id, df_string_output, reading_passage_kind], | |
outputs=[reading_passage] | |
) | |
reading_passage_delete_button.click( | |
delete_LLM_content, | |
inputs=[video_id, reading_passage_kind], | |
outputs=[reading_passage] | |
) | |
reading_passage_edit_button.click( | |
enable_edit_mode, | |
inputs=[], | |
outputs=[reading_passage] | |
) | |
reading_passage_update_button.click( | |
update_LLM_content, | |
inputs=[video_id, reading_passage, reading_passage_kind], | |
outputs=[reading_passage] | |
) | |
# summary event | |
summary_create_button.click( | |
create_LLM_content, | |
inputs=[video_id, df_string_output, summary_kind], | |
outputs=[df_summarise] | |
) | |
summary_delete_button.click( | |
delete_LLM_content, | |
inputs=[video_id, summary_kind], | |
outputs=[df_summarise] | |
) | |
summary_edit_button.click( | |
enable_edit_mode, | |
inputs=[], | |
outputs=[df_summarise] | |
) | |
summary_update_button.click( | |
update_LLM_content, | |
inputs=[video_id, df_summarise, summary_kind], | |
outputs=[df_summarise] | |
) | |
# 教師版 | |
worksheet_content_btn.click( | |
get_ai_content, | |
inputs=[password, video_id, df_string_output, content_subject, content_grade, content_level, worksheet_algorithm, worksheet_content_type_name], | |
outputs=[worksheet_exam_result_original, worksheet_exam_result, worksheet_prompt, worksheet_exam_result_prompt] | |
) | |
lesson_plan_btn.click( | |
get_ai_content, | |
inputs=[password, video_id, df_string_output, content_subject, content_grade, content_level, lesson_plan_time, lesson_plan_content_type_name], | |
outputs=[lesson_plan_exam_result_original, lesson_plan_exam_result, lesson_plan_prompt, lesson_plan_exam_result_prompt] | |
) | |
exit_ticket_btn.click( | |
get_ai_content, | |
inputs=[password, video_id, df_string_output, content_subject, content_grade, content_level, exit_ticket_time, exit_ticket_content_type_name], | |
outputs=[exit_ticket_exam_result_original, exit_ticket_exam_result, exit_ticket_prompt, exit_ticket_exam_result_prompt] | |
) | |
# 生成結果微調 | |
worksheet_exam_result_fine_tune_btn.click( | |
generate_exam_fine_tune_result, | |
inputs=[password, worksheet_exam_result_prompt, df_string_output, worksheet_exam_result, worksheet_exam_result_fine_tune_prompt], | |
outputs=[worksheet_exam_result] | |
) | |
worksheet_download_exam_result_button.click( | |
download_exam_result, | |
inputs=[worksheet_exam_result], | |
outputs=[worksheet_exam_result_word_link] | |
) | |
worksheet_exam_result_retrun_original.click( | |
return_original_exam_result, | |
inputs=[worksheet_exam_result_original], | |
outputs=[worksheet_exam_result] | |
) | |
lesson_plan_exam_result_fine_tune_btn.click( | |
generate_exam_fine_tune_result, | |
inputs=[password, lesson_plan_exam_result_prompt, df_string_output, lesson_plan_exam_result, lesson_plan_exam_result_fine_tune_prompt], | |
outputs=[lesson_plan_exam_result] | |
) | |
lesson_plan_download_exam_result_button.click( | |
download_exam_result, | |
inputs=[lesson_plan_exam_result], | |
outputs=[lesson_plan_exam_result_word_link] | |
) | |
lesson_plan_exam_result_retrun_original.click( | |
return_original_exam_result, | |
inputs=[lesson_plan_exam_result_original], | |
outputs=[lesson_plan_exam_result] | |
) | |
exit_ticket_exam_result_fine_tune_btn.click( | |
generate_exam_fine_tune_result, | |
inputs=[password, exit_ticket_exam_result_prompt, df_string_output, exit_ticket_exam_result, exit_ticket_exam_result_fine_tune_prompt], | |
outputs=[exit_ticket_exam_result] | |
) | |
exit_ticket_download_exam_result_button.click( | |
download_exam_result, | |
inputs=[exit_ticket_exam_result], | |
outputs=[exit_ticket_exam_result_word_link] | |
) | |
exit_ticket_exam_result_retrun_original.click( | |
return_original_exam_result, | |
inputs=[exit_ticket_exam_result_original], | |
outputs=[exit_ticket_exam_result] | |
) | |
# init_params | |
demo.load( | |
init_params, | |
inputs =[youtube_link], | |
outputs = [admin, reading_passage_admin, summary_admmin, see_details, password , youtube_link] | |
) | |
demo.launch(allowed_paths=["videos"]) | |