Update app.py

app.py

@@ -1,7 +1,9 @@
+
 from stable_whisper import modify_model,results_to_word_srt, results_to_sentence_srt
 import whisper
 import pysrt
 import re
+from pytube import YouTube
 import os
 from copy import deepcopy
 from typing import List
@@ -12,29 +14,32 @@ from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize
 from nltk import FreqDist
 from nltk.metrics import jaccard_distance
-
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import PorterStemmer
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import nltk
+nltk.download('stopwords')
+nltk.download('punkt')
 from moviepy.video.io.VideoFileClip import VideoFileClip
 from moviepy.video.VideoClip import ImageClip
 from datetime import datetime
+import moviepy.editor as mpy
 import gradio as gr
-import nltk
-nltk.download('stopwords')
-nltk.download('punkt')
 
 huggingfacehub_api_token = os.getenv("HF_TOKEN")
+class VideoEditor():
 
-class VideoQA:
     def __init__(self):
-        # self.loader = UnstructuredPDFLoader("/content/Document_ Introduction to Python (1).pdf")
-        # self.extracted_text=self.loader.load()
+      repo_id = "mistralai/Mistral-7B-Instruct-v0.2"
+      self.llm = HuggingFaceHub(
+          huggingfacehub_api_token=huggingfacehub_api_token,
+          repo_id=repo_id,
+          model_kwargs={"temperature": 0.2,"max_new_tokens":1000}
+      )
+      # self.data_json = {'topics': []}
 
-        # self.huggingfacehub_api_token =  # Replace with your Hugging Face token
-        self.repo_id = "mistralai/Mistral-7B-Instruct-v0.1"
-        self.llm = HuggingFaceHub(
-            huggingfacehub_api_token=huggingfacehub_api_token,
-            repo_id=self.repo_id,
-            model_kwargs={"temperature": 0.2, "max_new_tokens": 800}
-        )
 
     def load_model(self,model_selected):
       """
@@ -119,7 +124,6 @@ class VideoQA:
         result['srt'] = self.whisper_result_to_srt(result)
         return result
 
-
     def to_srt(self,lines: List[dict], strip=False) -> str:
         """
         lines: List[dict]
@@ -204,8 +208,6 @@ class VideoQA:
         srt = self.to_srt(segs, strip=strip)
         return srt
 
-
-
     def extract_timestamps_and_text(self,input_text):
         timestamp_pattern = re.compile(r'(\d{2}:\d{2}:\d{2}.\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}.\d{3})\n(.+)')
 
@@ -223,8 +225,46 @@ class VideoQA:
 
         return data
 
+    def sentence_timestamp(self,data):
+      result_sentences = []
+
+      current_sentence = ""
+      current_start_timestamp = ""
 
-    def generate_contract(self,text,question):
+      for entry in data:
+          text = entry['text']
+          start_timestamp = entry['start_timestamp']
+          end_timestamp = entry['end_timestamp']
+
+          # If the current sentence is empty, update start timestamp
+          if not current_sentence:
+              current_start_timestamp = start_timestamp
+
+          # Concatenate sentences until a sentence ends with a full stop
+          current_sentence += " " + text
+          if text.endswith('.'):
+              result_sentences.append({
+                  'start_timestamp': current_start_timestamp,
+                  'end_timestamp': end_timestamp,
+                  'text': current_sentence.strip()
+              })
+              current_sentence = ""
+      return result_sentences
+
+    def timestamp_text_to_list(self,result_sentences):
+      text_list = [item['text'] for item in result_sentences]
+
+      return text_list
+
+    def list_to_json(self,text_list):
+      jsonfile = {
+          "sentences": text_list
+      }
+      json_text = str(jsonfile)
+      return json_text
+
+
+    def video_qa_generate_contract(self,text,question):
 
 
         template = """you are the german language and universal language expert .your task is  analyze the given  text and user ask any question about given text answer to the user question.your returning answer must in user's language.otherwise reply i don't know.
@@ -241,15 +281,38 @@ class VideoQA:
         return result
 
 
-    def preprocess_sentence(self,sentence):
+    def topic_generate_contract(self,json_text,subrip):
+
+
+          template = """your first task is extract all topics discussed in the given content.
+
+          second task is analyze the given  paragraph and extract answer for the first task's extracted topics.
+          don't genarate answer yourself just extract  related answer from the given paragraph.
+
+          returing answer format:
+          Topic:Topic
+          Sentence:*Topic* Sentence
+
+          ```content:{content}```
+          ```paragraph:{paragraph}```
+          """
+
+          prompt = PromptTemplate(template=template, input_variables=["content","paragraph"])
+          llm_chain = LLMChain(prompt=prompt, verbose=True, llm=self.llm)
+
+          result = llm_chain.run({"content":json_text,"paragraph":subrip['text']})
+          return result
+
+
+    def video_qa_preprocess_sentence(self,sentence):
         stop_words = set(stopwords.words('english'))
         words = word_tokenize(sentence.lower())
         filtered_words = [word for word in words if word.isalnum() and word not in stop_words]
         return filtered_words
 
     def compute_similarity(self,sentence1, sentence2):
-        words1 = self.preprocess_sentence(sentence1)
-        words2 = self.preprocess_sentence(sentence2)
+        words1 = self.video_qa_preprocess_sentence(sentence1)
+        words2 = self.video_qa_preprocess_sentence(sentence2)
 
         freq_dist1 = FreqDist(words1)
         freq_dist2 = FreqDist(words2)
@@ -258,7 +321,7 @@ class VideoQA:
 
         return jaccard
 
-    def find_most_similar(self,sentence_list, target_sentence):
+    def video_qa_find_most_similar(self,sentence_list, target_sentence):
         similarities = [self.compute_similarity(target_sentence, sentence) for sentence in sentence_list]
 
         # Find the index of the most similar sentence
@@ -268,14 +331,14 @@ class VideoQA:
         return sentence_list[most_similar_index]
 
 
-    def start_end_timestamp(self,result,answer):
+    def video_qa_start_end_timestamp(self,result,answer):
       appended_text = []
 
       for item in result:
           appended_text.append(item['text'])
 
       # Find the most similar sentence
-      matched_sentence = self.find_most_similar(appended_text, answer)
+      matched_sentence = self.video_qa_find_most_similar(appended_text, answer)
       start_time=""
       end_time=""
       for entry in result:
@@ -285,7 +348,41 @@ class VideoQA:
           print(start_time+"\n"+end_time)
       return start_time,end_time
 
+    # Function to preprocess and tokenize a sentence
+    def preprocess_sentence(self,sentence):
+        stop_words = set(stopwords.words('english'))
+        ps = PorterStemmer()
+
+        # Tokenize and remove stopwords
+        words = word_tokenize(sentence)
+        words = [ps.stem(word.lower()) for word in words if word.isalnum() and word.lower() not in stop_words]
+
+        return ' '.join(words)
+
+    # Function to find the most similar sentence in list1 for a given sentence in list2
+    def topic_find_most_similar(self,sentence, list1):
+        similarities = []
+        processed_sentence = self.preprocess_sentence(sentence)
+
+        for candidate_sentence in list1:
+            similarity = self.calculate_cosine_similarity(processed_sentence, candidate_sentence)
+            similarities.append(similarity)
+
+        # Find the index of the most similar sentence in list1
+        max_similarity_index = similarities.index(max(similarities))
+
+        return list1[max_similarity_index]
+
+    # Function to calculate cosine similarity between two sentences
+    def calculate_cosine_similarity(self,sentence1, sentence2):
+        # Create a TF-IDF vectorizer
+        vectorizer = TfidfVectorizer()
+        tfidf_matrix = vectorizer.fit_transform([sentence1, sentence2])
+
+        # Calculate cosine similarity
+        cosine_sim = cosine_similarity(tfidf_matrix[0], tfidf_matrix[1])[0][0]
 
+        return cosine_sim
 
     def timestamp_to_seconds(self,timestamp):
         time_format = "%H:%M:%S,%f"
@@ -300,63 +397,136 @@ class VideoQA:
         # Use moviepy to cut both video and audio
         video_clip = VideoFileClip(input_file).subclip(start_time, end_time)
         video_clip.write_videofile(output_file, codec='libx264', audio_codec='aac', temp_audiofile='temp-audio.m4a', remove_temp=True)
-        
-    def sentence_timestamp(self,data):
-      result_sentences = []
-
-      current_sentence = ""
-      current_start_timestamp = ""
-
-      for entry in data:
-          text = entry['text']
-          start_timestamp = entry['start_timestamp']
-          end_timestamp = entry['end_timestamp']
 
-          # If the current sentence is empty, update start timestamp
-          if not current_sentence:
-              current_start_timestamp = start_timestamp
-
-          # Concatenate sentences until a sentence ends with a full stop
-          current_sentence += " " + text
-          if text.endswith('.'):
-              result_sentences.append({
-                  'start_timestamp': current_start_timestamp,
-                  'end_timestamp': end_timestamp,
-                  'text': current_sentence.strip()
-              })
-              current_sentence = ""
-      return result_sentences
-
-    def main(self,input_video_path,question):
+    def start_end_timestamp(self,result,matched_sentence):
+      start_time=""
+      end_time=""
+      for entry in result:
+        if matched_sentence in entry['text']:
+          start_time = entry['start_timestamp']
+          end_time = entry['end_timestamp']
+          # print(start_time+"\n"+end_time)
+      return start_time,end_time
 
-      subtitle = self.transcribe_video(input_video_path,'medium')
+    def video_write_funcion(self,vid,answer,text_list,result_sentences):
+        video = mpy.VideoFileClip(vid)
+        topics = {}
+        topics_list = answer.strip().split("\n\n")  # Remove leading/trailing whitespaces
+        for topic in topics_list:
+            lines = topic.split("\n")
+            if len(lines) > 0:
+                topic = lines[0].split(":")[1].strip()
+                sentence = "".join([line.split(":")[1].strip() for line in lines[1:]])
+
+            sentence_list = sentence.split(".")
+            unique_similar_sentences = []
+            list1 = text_list
+            # Find the most similar sentence in list1 for each sentence in list2
+            for sentence2 in sentence_list:
+                most_similar_sentence = self.topic_find_most_similar(sentence2, list1)
+
+                # Check if the sentence is not already in the list before appending
+                if most_similar_sentence not in unique_similar_sentences:
+                    unique_similar_sentences.append(most_similar_sentence)
+
+            # Print the unique most similar sentences
+            clips = []
+            for sentence in unique_similar_sentences:
+                # print(type(sentence))
+                start_time,end_time = self.start_end_timestamp(result_sentences,sentence)
+                clip = video.subclip(start_time, end_time)
+                clips.append(clip)
+            concatenated_clip = mpy.concatenate_videoclips(clips)
+            topics[topic] = concatenated_clip
+
+        for topic, clip in topics.items():
+            clip.write_videofile(f"{topic}.mp4")
+
+
+    def video_qa_main(self,input_path,video,question):
+      if input_path:
+        input_path = self.Download(input_path)
+        subtitle = self.transcribe_video(input_path,'medium')
+      elif video:
+        subtitle = self.transcribe_video(video,'medium')
+        input_path = video
+      print(subtitle['text'])
       text = subtitle['text']
-      answer = self.generate_contract(text,question)
+      answer = self.video_qa_generate_contract(text,question)
+      print("video_qa_generate_contract")
 
       subrip_text  = self.results_to_sentence_srt(subtitle)
       result = self.extract_timestamps_and_text(subrip_text)
-      sent = self.sentence_timestamp(result)  
-      start_time,end_time = self.start_end_timestamp(sent,answer)
+      sent = self.sentence_timestamp(result)
+      start_time,end_time = self.video_qa_start_end_timestamp(sent,answer)
       output_video_path = 'output_video.mp4'
 
-      self.cut_video(input_video_path, output_video_path, start_time, end_time)
+      self.cut_video(input_path, output_video_path, start_time, end_time)
       return output_video_path
 
-    def gradio_interface(self):
-
-        with gr.Blocks(css="style.css",theme=gr.themes.Soft()) as demo:
-          gr.HTML("""<center><h1>Video Question Answering</h1></center>""")
-          with gr.Row():
-            video = gr.Video(elem_classes="videosize")
-          with gr.Row():
-            query = gr.Textbox(label="Query")
-          with gr.Row():
-            output_video = gr.Video(elem_classes="videosize")
-
-          query.submit(self.main,[video,query],output_video)
-        demo.launch(debug=True)
-
-if __name__=="__main__":
-  video_qa = VideoQA()
-  video_qa.gradio_interface()
-
+    def Download(self,link):
+        youtubeObject = YouTube(link)
+        youtubeObject = youtubeObject.streams.get_highest_resolution()
+        try:
+            file_name = youtubeObject.download()
+            return file_name
+        except:
+            print("An error has occurred")
+        print("Download is completed successfully")
+
+    def topic_main(self,input_path,video):
+
+      if input_path:
+        input_path = self.Download(input_path)
+        subrip = self.transcribe_video(input_path,'medium')
+      elif video:
+        subrip = self.transcribe_video(video,'medium')
+        input_path = video
+      print(subrip['text'])
+      text = self.results_to_sentence_srt(subrip)
+      print("results_to_sentence_srt")
+      data = self.extract_timestamps_and_text(text)
+      print("extract_timestamps_and_text")
+      result_sentences = self.sentence_timestamp(data)
+      text_list = self.timestamp_text_to_list(result_sentences)
+      # print(text_list)
+      json_text = self.list_to_json(text_list)
+      # print(json_text)
+      print("list_to_json")
+      answer = self.topic_generate_contract(json_text,subrip)
+      # print(answer)
+      print("topic_generate_contract")
+      self.video_write_funcion(input_path,answer,text_list,result_sentences)
+      return "Topic Video Writted Successfully."
+
+
+with gr.Blocks() as demo:
+  video_editor = VideoEditor()
+  gr.HTML("""<center><h1>Video Question Answering & Topic Extracter</h1></center>""")
+  with gr.Tab("Video QA"):
+    with gr.Row():
+      youtube_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
+    with gr.Row():
+      video = gr.Video(sources="upload",height=200,width=300)
+    with gr.Row():
+      query = gr.Textbox(label="Query")
+
+    with gr.Row():
+      output_video = gr.Video(height=200,width=300)
+
+    # if video and query:
+    #   submit_btn.click(video_editor.video_qa_main,[video,query],output_video)
+    # elif youtube_link and query:
+    query.submit(video_editor.video_qa_main,[youtube_link,video,query],output_video)
+  with gr.Tab("Topic Extract"):
+    with gr.Row():
+      yt_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
+    with gr.Row():
+      video = gr.Video(height=200,width=300)
+    with gr.Row():
+      submit_btn = gr.Button(value="Submit")
+    with gr.Row():
+      textbox = gr.Textbox(label = "Status")
+
+      submit_btn.click(video_editor.topic_main,[yt_link,video],textbox)
+demo.launch()