Create main_query_tube.py

main_query_tube.py

@@ -0,0 +1,74 @@
+!pip install -Uqq youtube-transcript-api
+!pip install -Uqq pytube
+!pip3 install -Uqq thirdai[neural_db]
+
+from youtube_transcript_api import YouTubeTranscriptApi
+import pandas as pd
+from pytube import Playlist
+import re
+import time
+
+def get_video_links_from_playlist(playlist_url):
+    playlist = Playlist(playlist_url)
+    
+    video_links = [video.watch_url for video in playlist.videos]
+    return video_links
+
+def extract_video_id(video_url):
+    # Regular expression pattern to match YouTube video ID
+    pattern = r"(?:v=|v\/|embed\/|youtu.be\/|\/v\/|\/e\/|watch\?v=|youtube.com\/user\/[^#]*#([^\/]*?\/)*)((\w|-){11})"
+
+    match = re.search(pattern, video_url)
+    if match:
+        return match.group(2)
+    else:
+        return None
+
+def create_csv(transcript_lst):
+    
+    flat_data = [item for sublist in transcript_lst for item in sublist]
+    df = pd.DataFrame(flat_data)
+    df['end_time'] = df['start'] + df['duration']
+    df = df[['text', 'start', 'duration', 'end_time', 'video_serial_number']]
+    return df
+
+def create_data(playlist_link):
+    all_playlist_videos_lst = get_video_links_from_playlist(playlist_link)
+    # create individual video id list
+    video_id_lst = []
+    for video_link in all_playlist_videos_lst:
+        video_id_lst.append(extract_video_id(video_link))
+    # get transcripts for all video_ids
+    transcript_lst = []
+    for video_serial in range(len(video_id_lst)):
+        video_id = video_id_lst[video_serial]
+        try:
+            video_transcript = YouTubeTranscriptApi.get_transcript(video_id)
+            new_transcript = []
+            for i in range(0, len(video_transcript), 4):
+                start_time = video_transcript[i]['start']
+                duration = 0
+                text = ''
+                for j in range(4):
+                    index = i + j
+                    if index < len(video_transcript):
+                        text += video_transcript[index]['text']
+                        text += ' '
+                        duration += video_transcript[index]['duration']
+                dict = {'text' : text,
+                        'start' : start_time,
+                        'duration' : duration,}
+                       'video_serial_number':video_serial+1}
+                # print('serail_number: ',video_serial)
+                new_transcript.append(dict)
+            transcript_lst.append(new_transcript)
+        except:
+            continue 
+
+    # make dataframe from transcript list
+    if len(transcript_lst)!=0:
+        csv_file = create_csv(transcript_lst)
+        csv_file['text'] = csv_file['text'].str.replace(r'\n', ' ')
+        return csv_file
+    return 0
+