app.py

from stable_whisper import modify_model,results_to_word_srt, results_to_sentence_srt
import whisper
import pysrt
import re
from pytube import YouTube
import os
from copy import deepcopy
from typing import List
import os
from langchain import HuggingFaceHub, PromptTemplate, LLMChain
from langchain_community.document_loaders import UnstructuredPDFLoader
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk import FreqDist
from nltk.metrics import jaccard_distance
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import nltk
nltk.download('stopwords')
nltk.download('punkt')
from moviepy.video.io.VideoFileClip import VideoFileClip
from moviepy.video.VideoClip import ImageClip
from datetime import datetime
import moviepy.editor as mpy
import gradio as gr

huggingfacehub_api_token = os.getenv("HF_TOKEN")
class VideoEditor():

    def __init__(self):
      repo_id = "mistralai/Mistral-7B-Instruct-v0.2"
      self.llm = HuggingFaceHub(
          huggingfacehub_api_token=huggingfacehub_api_token,
          repo_id=repo_id,
          model_kwargs={"temperature": 0.2,"max_new_tokens":1000}
      )
      # self.data_json = {'topics': []}


    def load_model(self,model_selected):
      """
      Load a pre-trained machine learning model specified by the `model_selected` parameter
      using the `whisper` library and modify it to output word timestamps.

      Parameters:
      -----------
      model_selected : str
          A string specifying the name of the pre-trained machine learning model to load.

      Returns:
      --------
      model : object
          A modified version of the loaded pre-trained machine learning model that outputs
          timestamps for individual words.

      """
      model = whisper.load_model(model_selected)
      modify_model(model)
      return model

    def whisper_result_to_srt(self,result):
        """
        Convert the output of the Whisper speech recognition model into SubRip subtitle format.

        Parameters:
        -----------
        result : dict
            A dictionary containing the output of the Whisper speech recognition model, including word-level
            timestamps.

        Returns:
        --------
        srt : str
            A string in SubRip subtitle format, containing the word-level transcriptions and timing information
            from the Whisper output.

        Notes:
        ------
        This function takes the output of the Whisper speech recognition model, which includes word-level timestamps
        for each segment of the input audio file, and converts it into SubRip subtitle format. The resulting subtitle
        file can be used to display captions or transcripts alongside a video recording of the original audio.
        """
        text = []
        for i,s in enumerate(result['segments']):
            text.append(str(i+1))
            time_start = s['start']
            hours, minutes, seconds = int(time_start/3600), (time_start/60) % 60, (time_start) % 60
            timestamp_start = "%02d:%02d:%06.3f" % (hours, minutes, seconds)
            timestamp_start = timestamp_start.replace('.',',')
            time_end = s['end']
            hours, minutes, seconds = int(time_end/3600), (time_end/60) % 60, (time_end) % 60
            timestamp_end = "%02d:%02d:%06.3f" % (hours, minutes, seconds)
            timestamp_end = timestamp_end.replace('.',',')
            text.append(timestamp_start + " --> " + timestamp_end)
            text.append(s['text'].strip() + "\n")
        return "\n".join(text)

    # model_selected  = 'tiny'
    def transcribe_video(self,vid, model_selected):
        """
        Transcribe the audio in a video file using a pre-trained machine learning model and return the transcription
        and its corresponding timestamps in a subtitle format.

        Parameters:
        -----------
        vid : str
            A string specifying the path to the video file to be transcribed.
        model_selected : str
            A string specifying the name of the pre-trained machine learning model to use for transcription.

        Returns:
        --------
        result : dict
            A dictionary containing the transcription and its corresponding timestamps in a subtitle format.

        """
        model = self.load_model(model_selected)
        options = whisper.DecodingOptions(fp16=False)
        result = model.transcribe(vid, **options.__dict__)
        result['srt'] = self.whisper_result_to_srt(result)
        return result

    def to_srt(self,lines: List[dict], strip=False) -> str:
        """
        lines: List[dict]
            [{start:<start-timestamp-of-text>, end:<end-timestamp-of-text>, text:<str-of-text>}, ...]
        """

        def secs_to_hhmmss(secs):
            mm, ss = divmod(secs, 60)
            hh, mm = divmod(mm, 60)
            return f'{hh:0>2.0f}:{mm:0>2.0f}:{ss:0>6.3f}'.replace(".", ",")

        srt_str = '\n'.join(
            f'{i}\n'
            f'{secs_to_hhmmss(sub["start"])} --> {secs_to_hhmmss(sub["end"])}\n'
            f'{sub["text"].strip() if strip else sub["text"]}\n'
            for i, sub in enumerate(lines, 1))

        # if save_path:
        #     with open(save_path, 'w', encoding='utf-8') as f:
        #         f.write(srt_str)
        #     print(f'Saved: {os.path.abspath(save_path)}')

        return srt_str

    def tighten_timestamps(self,res: dict, end_at_last_word=True, end_before_period=False, start_at_first_word=False) -> dict:
        res = deepcopy(res)
        for i in range(len(res['segments'])):
            if start_at_first_word:
                res['segments'][i]['start'] = res['segments'][i]['word_timestamps'][0]['timestamp']
            if end_before_period and \
                    res['segments'][i]['word_timestamps'][-1] == '.' and \
                    len(res['segments'][i]['word_timestamps']) > 1:
                res['segments'][i]['end'] = res['segments'][i]['word_timestamps'][-2]['timestamp']
            elif end_at_last_word:
                res['segments'][i]['end'] = res['segments'][i]['word_timestamps'][-1]['timestamp']

        return res

    def results_to_sentence_srt(self,res: dict,
                                end_at_last_word=False,
                                end_before_period=False,
                                start_at_first_word=False,
                                strip=False):
        """

        Parameters
        ----------
        res: dict
            results from modified model
        srt_path: str
            output path of srt
        end_at_last_word: bool
            set end-of-sentence to timestamp-of-last-token
        end_before_period: bool
            set end-of-sentence to timestamp-of-last-non-period-token
        start_at_first_word: bool
            set start-of-sentence to timestamp-of-first-token
        strip: bool
            perform strip() on each sentence

        """
        strict = any((end_at_last_word, end_before_period, start_at_first_word))
        segs = self.tighten_timestamps(res,
                                  end_at_last_word=end_at_last_word,
                                  end_before_period=end_before_period,
                                  start_at_first_word=start_at_first_word)['segments'] \
            if strict else res['segments']

        max_idx = len(segs) - 1
        i = 1
        while i <= max_idx:
            if not (segs[i]['end'] - segs[i]['start']):
                if segs[i - 1]['end'] == segs[i]['end']:
                    segs[i - 1]['text'] += (' ' + segs[i]['text'].strip())
                    del segs[i]
                    max_idx -= 1
                    continue
                else:
                    segs[i]['start'] = segs[i - 1]['end']
            i += 1

        srt = self.to_srt(segs, strip=strip)
        return srt

    def extract_timestamps_and_text(self,input_text):
        timestamp_pattern = re.compile(r'(\d{2}:\d{2}:\d{2}.\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}.\d{3})\n(.+)')

        matches = timestamp_pattern.findall(input_text)

        data = []

        for match in matches:
            start_timestamp, end_timestamp, text = match
            data.append({
                'start_timestamp': start_timestamp,
                'end_timestamp': end_timestamp,
                'text': text.strip()
            })

        return data

    def sentence_timestamp(self,data):
      result_sentences = []

      current_sentence = ""
      current_start_timestamp = ""

      for entry in data:
          text = entry['text']
          start_timestamp = entry['start_timestamp']
          end_timestamp = entry['end_timestamp']

          # If the current sentence is empty, update start timestamp
          if not current_sentence:
              current_start_timestamp = start_timestamp

          # Concatenate sentences until a sentence ends with a full stop
          current_sentence += " " + text
          if text.endswith('.'):
              result_sentences.append({
                  'start_timestamp': current_start_timestamp,
                  'end_timestamp': end_timestamp,
                  'text': current_sentence.strip()
              })
              current_sentence = ""
      return result_sentences

    def timestamp_text_to_list(self,result_sentences):
      text_list = [item['text'] for item in result_sentences]

      return text_list

    def list_to_json(self,text_list):
      jsonfile = {
          "sentences": text_list
      }
      json_text = str(jsonfile)
      return json_text


    def video_qa_generate_contract(self,text,question):


        template = """you are the german language and universal language expert .your task is  analyze the given  text and user ask any question about given text answer to the user question.your returning answer must in user's language.otherwise reply i don't know.
        extracted_text:{text}
        user_question:{question}"""

        prompt = PromptTemplate(template=template, input_variables=["text","question"])
        llm_chain = LLMChain(prompt=prompt, verbose=True, llm=self.llm)

        result = llm_chain.run({"text":text,"question":question})
        print()
        print()
        print("this is answer:",result)
        return result


    def topic_generate_contract(self,json_text,subrip):


          template = """your first task is extract all topics discussed in the given content.

          second task is analyze the given  paragraph and extract answer for the first task's extracted topics.
          don't genarate answer yourself just extract  related answer from the given paragraph.

          returing answer format:
          Topic:Topic
          Sentence:*Topic* Sentence

          ```content:{content}```
          ```paragraph:{paragraph}```
          """

          prompt = PromptTemplate(template=template, input_variables=["content","paragraph"])
          llm_chain = LLMChain(prompt=prompt, verbose=True, llm=self.llm)

          result = llm_chain.run({"content":json_text,"paragraph":subrip['text']})
          return result


    def video_qa_preprocess_sentence(self,sentence):
        stop_words = set(stopwords.words('english'))
        words = word_tokenize(sentence.lower())
        filtered_words = [word for word in words if word.isalnum() and word not in stop_words]
        return filtered_words

    def compute_similarity(self,sentence1, sentence2):
        words1 = self.video_qa_preprocess_sentence(sentence1)
        words2 = self.video_qa_preprocess_sentence(sentence2)

        freq_dist1 = FreqDist(words1)
        freq_dist2 = FreqDist(words2)

        jaccard = 1 - jaccard_distance(set(freq_dist1), set(freq_dist2))

        return jaccard

    def video_qa_find_most_similar(self,sentence_list, target_sentence):
        similarities = [self.compute_similarity(target_sentence, sentence) for sentence in sentence_list]

        # Find the index of the most similar sentence
        most_similar_index = similarities.index(max(similarities))

        # Return the most similar sentence
        return sentence_list[most_similar_index]


    def video_qa_start_end_timestamp(self,result,answer):
      appended_text = []

      for item in result:
          appended_text.append(item['text'])

      # Find the most similar sentence
      matched_sentence = self.video_qa_find_most_similar(appended_text, answer)
      start_time=""
      end_time=""
      for entry in result:
        if matched_sentence in entry['text']:
          start_time = entry['start_timestamp']
          end_time = entry['end_timestamp']
          print(start_time+"\n"+end_time)
      return start_time,end_time

    # Function to preprocess and tokenize a sentence
    def preprocess_sentence(self,sentence):
        stop_words = set(stopwords.words('english'))
        ps = PorterStemmer()

        # Tokenize and remove stopwords
        words = word_tokenize(sentence)
        words = [ps.stem(word.lower()) for word in words if word.isalnum() and word.lower() not in stop_words]

        return ' '.join(words)

    # Function to find the most similar sentence in list1 for a given sentence in list2
    def topic_find_most_similar(self,sentence, list1):
        similarities = []
        processed_sentence = self.preprocess_sentence(sentence)

        for candidate_sentence in list1:
            similarity = self.calculate_cosine_similarity(processed_sentence, candidate_sentence)
            similarities.append(similarity)

        # Find the index of the most similar sentence in list1
        max_similarity_index = similarities.index(max(similarities))

        return list1[max_similarity_index]

    # Function to calculate cosine similarity between two sentences
    def calculate_cosine_similarity(self,sentence1, sentence2):
        # Create a TF-IDF vectorizer
        vectorizer = TfidfVectorizer()
        tfidf_matrix = vectorizer.fit_transform([sentence1, sentence2])

        # Calculate cosine similarity
        cosine_sim = cosine_similarity(tfidf_matrix[0], tfidf_matrix[1])[0][0]

        return cosine_sim

    def timestamp_to_seconds(self,timestamp):
        time_format = "%H:%M:%S,%f"
        dt = datetime.strptime(timestamp, time_format)
        return dt.hour * 3600 + dt.minute * 60 + dt.second + dt.microsecond / 1e6

    def cut_video(self,input_file, output_file, start_timestamp, end_timestamp):
        # Convert timestamps to seconds
        start_time = self.timestamp_to_seconds(start_timestamp)
        end_time = self.timestamp_to_seconds(end_timestamp)

        # Use moviepy to cut both video and audio
        video_clip = VideoFileClip(input_file).subclip(start_time, end_time)
        video_clip.write_videofile(output_file, codec='libx264', audio_codec='aac', temp_audiofile='temp-audio.m4a', remove_temp=True)

    def start_end_timestamp(self,result,matched_sentence):
      start_time=""
      end_time=""
      for entry in result:
        if matched_sentence in entry['text']:
          start_time = entry['start_timestamp']
          end_time = entry['end_timestamp']
          # print(start_time+"\n"+end_time)
      return start_time,end_time

    def video_write_funcion(self,vid,answer,text_list,result_sentences):
        video = mpy.VideoFileClip(vid)
        topics = {}
        topics_list = answer.strip().split("\n\n")  # Remove leading/trailing whitespaces
        for topic in topics_list:
            lines = topic.split("\n")
            if len(lines) > 0:
                topic = lines[0].split(":")[1].strip()
                sentence = "".join([line.split(":")[1].strip() for line in lines[1:]])

            sentence_list = sentence.split(".")
            unique_similar_sentences = []
            list1 = text_list
            # Find the most similar sentence in list1 for each sentence in list2
            for sentence2 in sentence_list:
                most_similar_sentence = self.topic_find_most_similar(sentence2, list1)

                # Check if the sentence is not already in the list before appending
                if most_similar_sentence not in unique_similar_sentences:
                    unique_similar_sentences.append(most_similar_sentence)

            # Print the unique most similar sentences
            clips = []
            for sentence in unique_similar_sentences:
                # print(type(sentence))
                start_time,end_time = self.start_end_timestamp(result_sentences,sentence)
                clip = video.subclip(start_time, end_time)
                clips.append(clip)
            concatenated_clip = mpy.concatenate_videoclips(clips)
            topics[topic] = concatenated_clip

        for topic, clip in topics.items():
            clip.write_videofile(f"{topic}.mp4")


    def video_qa_main(self,input_path,video,question):
      if input_path:
        input_path = self.Download(input_path)
        subtitle = self.transcribe_video(input_path,'medium')
      elif video:
        subtitle = self.transcribe_video(video,'medium')
        input_path = video
      print(subtitle['text'])
      text = subtitle['text']
      answer = self.video_qa_generate_contract(text,question)
      print("video_qa_generate_contract")

      subrip_text  = self.results_to_sentence_srt(subtitle)
      result = self.extract_timestamps_and_text(subrip_text)
      sent = self.sentence_timestamp(result)
      start_time,end_time = self.video_qa_start_end_timestamp(sent,answer)
      output_video_path = 'output_video.mp4'

      self.cut_video(input_path, output_video_path, start_time, end_time)
      return output_video_path

    def Download(self,link):
        youtubeObject = YouTube(link)
        youtubeObject = youtubeObject.streams.get_highest_resolution()
        try:
            file_name = youtubeObject.download()
            return file_name
        except:
            print("An error has occurred")
        print("Download is completed successfully")

    def topic_main(self,input_path,video):

      if input_path:
        input_path = self.Download(input_path)
        subrip = self.transcribe_video(input_path,'medium')
      elif video:
        subrip = self.transcribe_video(video,'medium')
        input_path = video
      print(subrip['text'])
      text = self.results_to_sentence_srt(subrip)
      print("results_to_sentence_srt")
      data = self.extract_timestamps_and_text(text)
      print("extract_timestamps_and_text")
      result_sentences = self.sentence_timestamp(data)
      text_list = self.timestamp_text_to_list(result_sentences)
      # print(text_list)
      json_text = self.list_to_json(text_list)
      # print(json_text)
      print("list_to_json")
      answer = self.topic_generate_contract(json_text,subrip)
      # print(answer)
      print("topic_generate_contract")
      self.video_write_funcion(input_path,answer,text_list,result_sentences)
      return "Topic Video Writted Successfully."


with gr.Blocks(css="style.css",theme="freddyaboulton/test-blue") as demo:
  video_editor = VideoEditor()
  gr.HTML("""<center><h1>Video Question Answering & Topic Extracter</h1></center>""")
  with gr.Tab("Video QA"):
    with gr.Row():
      youtube_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
    with gr.Row():
      video = gr.Video(sources="upload",height=200,width=300)
    with gr.Row():
      query = gr.Textbox(label="Query")

    with gr.Row():
      output_video = gr.Video(height=200,width=300)

    # if video and query:
    #   submit_btn.click(video_editor.video_qa_main,[video,query],output_video)
    # elif youtube_link and query:
    query.submit(video_editor.video_qa_main,[youtube_link,video,query],output_video)
  with gr.Tab("Topic Extract"):
    with gr.Row():
      yt_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
    with gr.Row():
      video = gr.Video(sources="upload",height=200,width=300)
    with gr.Row():
      submit_btn = gr.Button(value="Submit")
    with gr.Row():
      textbox = gr.Textbox(label = "Status")

      submit_btn.click(video_editor.topic_main,[yt_link,video],textbox)
demo.launch()