Update app.py

app.py

@@ -9,7 +9,6 @@ import re
 import time
 import os 
 import numpy as np
-import openai
 from sklearn.cluster import AgglomerativeClustering
 from sklearn.metrics import silhouette_score
 
@@ -150,22 +149,6 @@ embedding_model = PretrainedSpeakerEmbedding(
     "speechbrain/spkrec-ecapa-voxceleb",
     device=torch.device("cuda" if torch.cuda.is_available() else "cpu"))
 
-def summarize_text(text):
-    response = openai.Completion.create(
-        engine="text-davinci-003",
-        prompt=f"Please summarize the following text: {text}",
-        max_tokens=100
-    )
-    return response.choices[0].text
-
-def emotion_analysis(text):
-    response = openai.Completion.create(
-        engine="text-davinci-003",
-        prompt=f"Please interpret the emotions in the following text: {text}",
-        max_tokens=100
-    )
-    return response.choices[0].text
-    
 def transcribe(microphone, file_upload):
     warn_output = ""
     if (microphone is not None) and (file_upload is not None):
@@ -234,79 +217,133 @@ def get_youtube(video_url):
     return abs_video_path
 
 def speech_to_text(video_file_path, selected_source_lang, whisper_model, num_speakers):
+    """
+    # Transcreva o link do youtube usando OpenAI Whisper
+
+    NOTA: Este modelo foi adaptado por Pedro Faria, para exemplo para a Biometrid, não deve ser usado para outros fins.
+    
+    1. Usando o modelo Whisper da Open AI para separar áudio em segmentos e gerar transcrições.
+    2. Gerando embeddings de alto-falante para cada segmento.
+    3. Aplicando clustering aglomerativo nos embeddings para identificar o falante de cada segmento.
+
+    O reconhecimento de fala é baseado em modelos do OpenAI Whisper https://github.com/openai/whisper
+    Speaker diarization model and pipeline from by https://github.com/pyannote/pyannote-audio
+    Modelo de diarização de alto-falante e pipeline desenvolvido por https://github.com/pyannote/pyannote-audio 
+    """
+    
+    # model = whisper.load_model(whisper_model)
+    # model = WhisperModel(whisper_model, device="cuda", compute_type="int8_float16")
     model = WhisperModel(whisper_model, compute_type="int8")
     time_start = time.time()
-    if video_file_path is None:
+    if(video_file_path == None):
         raise ValueError("Error no video input")
-
-    _, file_ending = os.path.splitext(f'{video_file_path}')
-    audio_file = video_file_path.replace(file_ending, ".wav")
-    os.system(f'ffmpeg -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{audio_file}"')
+    print(video_file_path)
+
+    try:
+        # Read and convert youtube video
+        _,file_ending = os.path.splitext(f'{video_file_path}')
+        print(f'file enging is {file_ending}')
+        audio_file = video_file_path.replace(file_ending, ".wav")
+        print("A iniciar a conversão para WAV")
+        os.system(f'ffmpeg -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{audio_file}"')
+        
+        # Get duration
+        with contextlib.closing(wave.open(audio_file,'r')) as f:
+            frames = f.getnframes()
+            rate = f.getframerate()
+            duration = frames / float(rate)
+        print(f"Conversão para WAV concluída, duração do arquivo de áudio.: {duration}")
+
+        # Transcribe audio
+        options = dict(language=selected_source_lang, beam_size=5, best_of=5)
+        transcribe_options = dict(task="transcribe", **options)
+        segments_raw, info = model.transcribe(audio_file, **transcribe_options)
+
+        # Convert back to original openai format
+        segments = []
+        i = 0
+        for segment_chunk in segments_raw:
+            chunk = {}
+            chunk["start"] = segment_chunk.start
+            chunk["end"] = segment_chunk.end
+            chunk["text"] = segment_chunk.text
+            segments.append(chunk)
+            i += 1
+        print("transcrição de audio com fast whisper terminada")
+    except Exception as e:
+        raise RuntimeError("Erro a converter o filme para audio")
+
+    try:
+        # Create embedding
+        def segment_embedding(segment):
+            audio = Audio()
+            start = segment["start"]
+            # Whisper overshoots the end timestamp in the last segment
+            end = min(duration, segment["end"])
+            clip = Segment(start, end)
+            waveform, sample_rate = audio.crop(audio_file, clip)
+            return embedding_model(waveform[None])
+
+        embeddings = np.zeros(shape=(len(segments), 192))
+        for i, segment in enumerate(segments):
+            embeddings[i] = segment_embedding(segment)
+        embeddings = np.nan_to_num(embeddings)
+        print(f'Embedding shape: {embeddings.shape}')
+
+        if num_speakers == 0:
+        # Find the best number of speakers
+            score_num_speakers = {}
     
-    with contextlib.closing(wave.open(audio_file, 'r')) as f:
-        frames = f.getnframes()
-        rate = f.getframerate()
-        duration = frames / float(rate)
-
-    options = dict(language=selected_source_lang, beam_size=5, best_of=5)
-    transcribe_options = dict(task="transcribe", **options)
-    segments_raw, info = model.transcribe(audio_file, **transcribe_options)
-
-    segments = []
-    for segment_chunk in segments_raw:
-        chunk = {"start": segment_chunk.start, "end": segment_chunk.end, "text": segment_chunk.text}
-        segments.append(chunk)
-
-    embeddings = np.zeros(shape=(len(segments), 192))
-    for i, segment in enumerate(segments):
-        embeddings[i] = segment_embedding(segment)
-    embeddings = np.nan_to_num(embeddings)
-
-    best_num_speaker = num_speakers if num_speakers != 0 else max(range(2, 10 + 1), key=lambda n: silhouette_score(embeddings, AgglomerativeClustering(n).fit(embeddings).labels_, metric='euclidean'))
-    clustering = AgglomerativeClustering(best_num_speaker).fit(embeddings)
-    labels = clustering.labels_
-    for i in range(len(segments)):
-        segments[i]["speaker"] = 'Participante ' + str(labels[i] + 1)
-
-    objects = {
-        'Start': [],
-        'End': [],
-        'Speaker': [],
-        'Text': []
-    }
-    text = ''
-    for (i, segment) in enumerate(segments):
-        if i == 0 or segments[i - 1]["speaker"] != segment["speaker"]:
-            objects['Start'].append(str(convert_time(segment["start"])))
-            objects['Speaker'].append(segment["speaker"])
-            if i != 0:
-                objects['End'].append(str(convert_time(segments[i - 1]["end"])))
-                objects['Text'].append(text)
-                text = ''
+            for num_speakers in range(2, 10+1):
+                clustering = AgglomerativeClustering(num_speakers).fit(embeddings)
+                score = silhouette_score(embeddings, clustering.labels_, metric='euclidean')
+                score_num_speakers[num_speakers] = score
+            best_num_speaker = max(score_num_speakers, key=lambda x:score_num_speakers[x])
+            print(f"O número estimado de participantes: {best_num_speaker} com pontuação de {score_num_speakers[best_num_speaker]} ")
+        else:
+            best_num_speaker = num_speakers
+            
+        # Assign speaker label   
+        clustering = AgglomerativeClustering(best_num_speaker).fit(embeddings)
+        labels = clustering.labels_
+        for i in range(len(segments)):
+            segments[i]["speaker"] = 'Participante ' + str(labels[i] + 1)
+
+        # Make output
+        objects = {
+            'Start' : [],
+            'End': [],
+            'Speaker': [],
+            'Text': []
+        }
+        text = ''
+        for (i, segment) in enumerate(segments):
+            if i == 0 or segments[i - 1]["speaker"] != segment["speaker"]:
+                objects['Start'].append(str(convert_time(segment["start"])))
+                objects['Speaker'].append(segment["speaker"])
+                if i != 0:
+                    objects['End'].append(str(convert_time(segments[i - 1]["end"])))
+                    objects['Text'].append(text)
+                    text = ''
             text += segment["text"] + ' '
-    objects['End'].append(str(convert_time(segments[i - 1]["end"])))
-    objects['Text'].append(text)
-
-    transcription = " ".join(objects['Text'])
-    summary = summarize_text(transcription)
-    emotions = emotion_analysis(transcription)
-
-    time_end = time.time()
-    time_diff = time_end - time_start
-    memory = psutil.virtual_memory()
-    gpu_utilization, gpu_memory = GPUInfo.gpu_usage()
-    gpu_utilization = gpu_utilization[0] if len(gpu_utilization) > 0 else 0
-    gpu_memory = gpu_memory[0] if len(gpu_memory) > 0 else 0
-    system_info = f"""
-    *Memoria: {memory.total / (1024 * 1024 * 1024):.2f}GB, utilizado: {memory.percent}%, disponivel: {memory.available / (1024 * 1024 * 1024):.2f}GB.* 
-    *Tempo de processamento: {time_diff:.5} segundos.*
-    *Utilização de GPU: {gpu_utilization}%, Memoria de GPU: {gpu_memory}MiB.*
-    """
-    save_path = "output/transcript_result.csv"
-    df_results = pd.DataFrame(objects)
-    df_results.to_csv(save_path, index=False, encoding="utf-8")
-    return df_results, system_info, save_path, summary, emotions
-
+        objects['End'].append(str(convert_time(segments[i - 1]["end"])))
+        objects['Text'].append(text)
+        
+        time_end = time.time()
+        time_diff = time_end - time_start
+        memory = psutil.virtual_memory()
+        gpu_utilization, gpu_memory = GPUInfo.gpu_usage()
+        gpu_utilization = gpu_utilization[0] if len(gpu_utilization) > 0 else 0
+        gpu_memory = gpu_memory[0] if len(gpu_memory) > 0 else 0
+        system_info = f"""
+        *Memoria: {memory.total / (1024 * 1024 * 1024):.2f}GB, utilizado: {memory.percent}%, disponivel: {memory.available / (1024 * 1024 * 1024):.2f}GB.* 
+        *Tempo de processamento: {time_diff:.5} segundos.*
+        *Utilização de GPU: {gpu_utilization}%, Memoria de GPU: {gpu_memory}MiB.*
+        """
+        save_path = "output/transcript_result.csv"
+        df_results = pd.DataFrame(objects)
+        df_results.to_csv(save_path, index=False, encoding="utf-8")
+        return df_results, system_info, save_path
     
     except Exception as e:
         raise RuntimeError("Erro a correr a inferência com um modelo local", e)
@@ -321,8 +358,6 @@ memory = psutil.virtual_memory()
 selected_source_lang = gr.Dropdown(choices=source_language_list, type="value", value="pt", label="Linguagem detectada no vídeo", interactive=True)
 selected_whisper_model = gr.Dropdown(choices=whisper_models, type="value", value="large-v2", label="Modelo Whisper selecionado", interactive=True)
 number_speakers = gr.Number(precision=0, value=2, label="Insira o número de participantes para obter melhores resultados. Se o valor for 0, o modelo encontrará automaticamente a melhor quantidade.", interactive=True)
-summary_text = gr.Textbox(label="Resumo da Transcrição", readonly=True)
-emotion_analysis_text = gr.Textbox(label="Análise de Emoções", readonly=True)
 system_info = gr.Markdown(f"*Memoria: {memory.total / (1024 * 1024 * 1024):.2f}GB, utilizado: {memory.percent}%, disponível: {memory.available / (1024 * 1024 * 1024):.2f}GB*")
 download_transcript = gr.File(label="Download transcript")
 transcription_df = gr.DataFrame(value=df_init,label="Dataframe da transcrição", row_count=(0, "dynamic"), max_rows = 10, wrap=True, overflow_row_behaviour='paginate')
@@ -383,9 +418,9 @@ with demo:
                 number_speakers.render()
                 transcribe_btn = gr.Button("Transcrever audio com diarização")
                 transcribe_btn.click(speech_to_text, 
-                     [video_in, selected_source_lang, selected_whisper_model, number_speakers], 
-                     [transcription_df, system_info, download_transcript, summary_text, emotion_analysis_text]
-                    )
+                                     [video_in, selected_source_lang, selected_whisper_model, number_speakers], 
+                                     [transcription_df, system_info, download_transcript]
+                                    )
                 
         with gr.Row():
             gr.Markdown('''
@@ -399,9 +434,8 @@ with demo:
                 transcription_df.render()
                 system_info.render()
 
-        with gr.Row():
-            summary_text.render()
-            emotion_analysis_text.render()        
+                
     
 
-demo.launch(debug=True)
+demo.launch(debug=True)
+⚛