"""Streamlit demo to visualize auto-annotated Foley segments from movie clips.""" import os from os.path import join, exists, dirname, abspath, basename import json from glob import glob from tqdm import tqdm import numpy as np import pandas as pd import streamlit as st from moviepy.video.io.VideoFileClip import VideoFileClip import warnings warnings.simplefilter(action='ignore') curr_filepath = abspath(__file__) repo_path = dirname(dirname(curr_filepath)) def load_json(path: str) -> dict: """Helper to load json file""" with open(path, 'rb') as f: data = json.load(f) return data def tqdm_iterator(items, desc=None, bar_format=None, **kwargs): tqdm._instances.clear() iterator = tqdm( items, desc=desc, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}', **kwargs, ) return iterator def get_data_root_from_hostname(): import socket data_root_lib = { "diva": "/ssd/pbagad/datasets/", "node": "/var/scratch/pbagad/datasets/", "fs4": "/var/scratch/pbagad/datasets/", } hostname = socket.gethostname() hostname = hostname[:4] data_root = data_root_lib.get(hostname, "NA") return data_root def load_clips_df(df_path, data_dir, verbose=True, use_local=False): assert exists(df_path), f"File {df_path} does not exist" df = pd.read_csv(df_path) print(df.columns) if verbose: print("Number of clips:", len(df)) # filter out clips that are not downloaded df["video_path"] = df["videoid"].apply(lambda x: join(data_dir, "pytube_videos", f"{x}.mp4")) if use_local: df = df[df["video_path"].apply(exists)] if verbose: print("Number of clips (with videos available):", len(df)) df["audio_path"] = df["videoid"].apply(lambda x: join(data_dir, "pytube_audio", f"{x}.wav")) if use_local: df = df[df["audio_path"].apply(exists)] if verbose: print("Number of clips (with audio available):", len(df)) df["annot_path"] = df["videoid"].apply(lambda x: join(data_dir, "annotations", f"{x}.json")) if use_local: df = df[df["annot_path"].apply(exists)] if verbose: print("Number of clips (with annotations available):", len(df)) return df def summarize_classification_probs(silence, probs): summary = [f"Silence: {silence}"] summary += [f"{l.capitalize()}: {p}" for (l, p) in probs] return " | ".join(summary) def cut_video_in_segments(video_path, segments): video = VideoFileClip(video_path) tmp_dir = os.path.join(os.path.expanduser("~"), "tmp") clip_paths = [f"{tmp_dir}/clip_{i}.mp4" for i in range(len(segments))] iterator = tqdm_iterator( zip(segments, clip_paths), total=len(segments), desc="Preparing clips", ) clips = [ video.subclip(x, y).write_videofile(f, logger=None, verbose=False) \ for (x, y), f in iterator ] return clip_paths def process_sample(row): video_path = row["video_path"] audio_path = row["audio_path"] annot = load_json(row["annot_filtered"]) seg_indices = [i for i, flag in enumerate(annot["keep_status"]) if flag] keys = ["non_speech_segments", "silence_prob", "audiomae_on_audioset", "duration"] for k in keys: annot[k] = [x for i, x in enumerate(annot[k]) if i in seg_indices] del annot["keep_status"] labels = [ summarize_classification_probs( annot["silence_prob"][i], annot["audiomae_on_audioset"][i] ) for i in range(len(annot["non_speech_segments"])) ] clip_paths = cut_video_in_segments(video_path, annot["non_speech_segments"]) return clip_paths, labels, annot["non_speech_segments"], annot["duration"] def make_grid(cols,rows): grid = [0]*cols for i in range(cols): with st.container(): grid[i] = st.columns(rows) return grid if __name__ == "__main__": # Streamlit app code st.set_page_config(layout="wide") st.title("Foley Segments from Condensed Movies Dataset 🎬") st.markdown( "> **Note**: This demo shows cut out segments from clips in the [Condensed Movies](https://www.robots.ox.ac.uk/~vgg/data/condensed-movies/) dataset. "\ "The segments are adjudged to have Foley sounds based on AudioMAE predictions on AudioSet classes. "\ "The segments with duration not in [2s, 30s] or those with silence or high probability of speech/music sounds are filtered out. "\ "However, segments can still be noisy. Furthermore, even if a clip has "\ "Foley, there can still be background music/score which we have not removed yet." ) st.markdown( """> Warning: Currently, each clip can be played only once. Replaying starts the clip from beginning of the video.""", unsafe_allow_html=True ) st.markdown( "**Instructions**: Click the **Reload** button to see segments from a new clip. "\ "Reloading the page is not necessary." ) csv_path = "./clips.csv" ann_dirs = glob(join(".", "annotations_", "*")) annot_paths = glob(join(".", "annotations_*", "*_filtered.json")) per_video_width = 360 per_video_height = 240 print("Total number of clips: {}".format(len(annot_paths))) if "data" not in st.session_state: # store video ids video_ids = [basename(x).split("_filtered.json")[0] for x in annot_paths] # load annotation data data = [load_json(p) for p in annot_paths] num_foley_per_clip = [sum(d["keep_status"]) for d in data] num_foley_segments = np.sum(num_foley_per_clip) data = [d for d, n in zip(data, num_foley_per_clip) if n > 0] # get movie titles df = pd.read_csv(csv_path) titles = df["title"].values # store variables st.session_state.titles = titles st.session_state.video_ids = video_ids st.session_state.data = data st.session_state.num_foley_segments = num_foley_segments reload_button = st.button("Reload") index = np.random.randint(0, len(st.session_state.data)) if reload_button: index = np.random.randint(0, len(st.session_state.data)) # Gather data annot = st.session_state.data[index] video_id = st.session_state.video_ids[index] seg_indices = [i for i, flag in enumerate(annot["keep_status"]) if flag] keys = ["non_speech_segments", "silence_prob", "audiomae_on_audioset", "duration"] for k in keys: annot[k] = [x for i, x in enumerate(annot[k]) if i in seg_indices] del annot["keep_status"] labels = [ summarize_classification_probs( annot["silence_prob"][i], annot["audiomae_on_audioset"][i] ) for i in range(len(annot["non_speech_segments"])) ] segments, durations = annot["non_speech_segments"], annot["duration"] movie = st.session_state.titles[index] st.markdown(f"Showing Foley segments from a clip in movie: **{movie}**") # Create a grid of videos grid = make_grid(3, 3) # Add videos to the grid for idx in range(0, min(len(segments), 9)): i, j = idx // 3, idx % 3 start, end = segments[idx] duration = durations[idx] grid[i][j].caption(f"Segment duration: {duration}") url = f"https://www.youtube.com/embed/{video_id}?start={int(start)}&end={int(end)}" html_code = f""" """ grid[i][j].markdown(html_code, unsafe_allow_html=True) grid[i][j].caption(f"{labels[idx]}") st.markdown("##### Some stats") st.write(f"Total number of unique clips: {len(st.session_state.data)}") st.write("Total number of foley segments: {}".format(st.session_state.num_foley_segments))