feat: Add dynamic image backgrounds and spectrogram opacity

This commit introduces a major feature allowing users to add image backgrounds to the generated spectrogram video. It also enhances the visualizer by making it semi-transparent when a background is present.

feat: Add custom resolution and improve audio quality

This commit introduces two significant user-facing enhancements: the ability to customize the output video dimensions and a major improvement in audio fidelity for transcoded files.

app.py

@@ -9,7 +9,7 @@ import subprocess
 import matplotlib.font_manager as fm
 from typing import Tuple, List, Dict
 from mutagen.flac import FLAC
-from moviepy import CompositeVideoClip, TextClip, VideoClip, AudioFileClip
+from moviepy import CompositeVideoClip, TextClip, VideoClip, AudioFileClip, ImageClip
 
 # --- Font Scanning and Management ---
 def get_font_display_name(font_path: str) -> Tuple[str, str]:
@@ -128,7 +128,7 @@ SYSTEM_FONTS_MAP, FONT_DISPLAY_NAMES = get_font_data()
 print(f"Scan complete. Found {len(FONT_DISPLAY_NAMES)} available fonts.")
 
 
-# --- CUE Sheet Parsing Logic (Unchanged) ---
+# --- CUE Sheet Parsing Logic ---
 def cue_time_to_seconds(time_str: str) -> float:
     try:
         minutes, seconds, frames = map(int, time_str.split(':'))
@@ -160,7 +160,7 @@ def parse_cue_sheet_manually(cue_data: str) -> List[Dict[str, any]]:
     return tracks
 
 
-# --- Add a function to increase framerate using FFmpeg ---
+# --- FFmpeg Framerate Conversion ---
 def increase_video_framerate(input_path: str, output_path: str, target_fps: int = 24):
     """
     Uses FFmpeg to increase the video's framerate without re-encoding.
@@ -203,7 +203,9 @@ def increase_video_framerate(input_path: str, output_path: str, target_fps: int
 
 # --- Main Processing Function ---
 def process_audio_to_video(
-    audio_path: str, spec_fg_color: str, spec_bg_color: str, 
+    audio_path: str, image_paths: List[str],
+    video_width: int, video_height: int,
+    spec_fg_color: str, spec_bg_color: str, 
     font_name: str, font_size: int, font_color: str,
     font_bg_color: str, font_bg_alpha: float,
     pos_h: str, pos_v: str
@@ -216,7 +218,8 @@ def process_audio_to_video(
     temp_fps1_path = f"temp_{timestamp}_fps1.mp4"
     final_output_path = f"final_video_{timestamp}_fps24.mp4"
 
-    WIDTH, HEIGHT, RENDER_FPS = 1280, 720, 1 # Render at 1 FPS
+    WIDTH, HEIGHT = int(video_width), int(video_height)
+    RENDER_FPS = 1 # Render at 1 FPS
     PLAYBACK_FPS = 24 # Final playback framerate
     
     # --- A robust color parser for hex and rgb() strings ---
@@ -252,17 +255,102 @@ def process_audio_to_video(
         y, sr = librosa.load(audio_path, sr=None, mono=True)
         duration = librosa.get_duration(y=y, sr=sr)
         
+        # --- Image Processing Logic ---
+        image_clips = []
+        # Check if any images were uploaded.
+        if image_paths and len(image_paths) > 0:
+            print(f"Found {len(image_paths)} images to process.")
+            
+            # First, try to parse the CUE sheet from the audio file.
+            tracks = []
+            if audio_path.lower().endswith('.flac'):
+                try:
+                    audio_meta = FLAC(audio_path)
+                    if 'cuesheet' in audio_meta.tags:
+                        tracks = parse_cue_sheet_manually(audio_meta.tags['cuesheet'][0])
+                        print(f"Successfully parsed {len(tracks)} tracks from CUE sheet.")
+                except Exception as e: print(f"Warning: Could not read or parse CUE sheet: {e}")
+
+            # --- HELPER FUNCTION FOR ROBUST IMAGE CLIPS ---
+            def create_image_layer(img_path, start, dur):
+                """
+                Creates an image layer that fits entirely within the video frame.
+                It scales the image down to fit and centers it on a transparent background.
+                """
+                # This function implements a "cover" scaling mode to ensure the image
+                # fills the entire video frame without leaving black bars.
+                try:
+                    img_clip_raw = ImageClip(img_path)
+                    
+                    # 1. Calculate scaling factor to "contain" the image (fit inside).
+                    #    We use min() to find the ratio that requires the most shrinkage,
+                    #    ensuring the whole image fits without being cropped.
+                    scale_factor = min(WIDTH / img_clip_raw.w, HEIGHT / img_clip_raw.h)
+                    
+                    # 2. Resize the image so it fits perfectly within the video dimensions.
+                    resized_image_clip = img_clip_raw.resized(scale_factor)
+                    
+                    # 3. Create a composite clip to position the resized image on a
+                    #    correctly-sized transparent canvas. This is the key to preventing overflow.
+                    final_layer = CompositeVideoClip(
+                        [resized_image_clip.with_position("center")],
+                        size=(WIDTH, HEIGHT)
+                    )
+                    
+                    # 4. Set the timing on the final composite layer.
+                    return final_layer.with_duration(dur).with_start(start)
+                except Exception as e:
+                    print(f"Warning: Failed to process image '{img_path}'. Skipping. Error: {e}")
+                    return None
+            # --- END OF HELPER FUNCTION ---
+            
+            # Mode 1: If CUE tracks match the number of images, align them.
+            if tracks and len(tracks) == len(image_paths):
+                print("Image count matches track count. Aligning images with tracks.")
+                for i, (track, img_path) in enumerate(zip(tracks, image_paths)):
+                    start_time = track.get('start_time', 0)
+                    # The end time of a track is the start time of the next, or the total duration for the last track.
+                    end_time = tracks[i+1].get('start_time', duration) if i + 1 < len(tracks) else duration
+                    img_duration = end_time - start_time
+                    if img_duration <= 0: continue
+                    
+                    # Create an ImageClip for the duration of the track.
+                    clip = create_image_layer(img_path, start_time, img_duration)
+                    if clip:
+                        image_clips.append(clip)
+
+            # Mode 2: If no CUE or mismatch, distribute images evenly across the audio duration.
+            else:
+                if tracks: print("Image count does not match track count. Distributing images evenly.")
+                else: print("No CUE sheet found. Distributing images evenly.")
+
+                img_duration = duration / len(image_paths)
+                for i, img_path in enumerate(image_paths):
+                    start_time = i * img_duration
+                    # Create an ImageClip for a calculated segment of time.                    
+                    clip = create_image_layer(img_path, start_time, img_duration)
+                    if clip:
+                        image_clips.append(clip)
+
         # Spectrogram calculation
         N_FFT, HOP_LENGTH, N_BANDS = 2048, 512, 32
         MIN_DB, MAX_DB = -80.0, 0.0
         S_mel = librosa.feature.melspectrogram(y=y, sr=sr, n_fft=N_FFT, hop_length=HOP_LENGTH, n_mels=N_BANDS, fmax=sr/2)
         S_mel_db = librosa.power_to_db(S_mel, ref=np.max)
         
-        # Frame generation logic
+        # Frame generation logic for the spectrogram
         def frame_generator(t):
-            frame = np.full((HEIGHT, WIDTH, 3), bg_rgb, dtype=np.uint8)
-            for i in range(1, 9):
-                y_pos = int(i * (HEIGHT / 9)); frame[y_pos-1:y_pos, :] = grid_rgb
+            # If images are used as background, the spectrogram's own background should be transparent.
+            # Otherwise, use the selected background color.
+            # Here, we will use a simple opacity setting on the final clip, so we always generate the frame.
+            frame_bg = bg_rgb if not image_clips else (0,0,0) # Use black if it will be made transparent later
+            frame = np.full((HEIGHT, WIDTH, 3), frame_bg, dtype=np.uint8)
+
+            # Draw the grid lines only if no images are being used.
+            if not image_clips:
+                for i in range(1, 9):
+                    y_pos = int(i * (HEIGHT / 9)); frame[y_pos-1:y_pos, :] = grid_rgb
+
             time_idx = int((t / duration) * (S_mel_db.shape[1] - 1))
             bar_width = WIDTH / N_BANDS
             for i in range(N_BANDS):
@@ -279,6 +367,13 @@ def process_audio_to_video(
             return frame
             
         video_clip = VideoClip(frame_function=frame_generator, duration=duration)
+
+        # --- NEW: Set Spectrogram Opacity ---
+        # If image clips were created, make the spectrogram layer 50% transparent.
+        if image_clips:
+            print("Applying 50% opacity to spectrogram layer.")
+            video_clip = video_clip.with_opacity(0.5)
+
         audio_clip = AudioFileClip(audio_path)
         
         # CUE Sheet title overlay logic
@@ -286,10 +381,11 @@ def process_audio_to_video(
         tracks = []
         if audio_path.lower().endswith('.flac'):
             try:
-                audio = FLAC(audio_path); tracks = parse_cue_sheet_manually(audio.tags['cuesheet'][0])
-                print(f"Successfully parsed {len(tracks)} tracks from CUE sheet...")
-            except Exception as e:
-                print(f"Warning: Could not read or parse CUE sheet: {e}")
+                audio_meta = FLAC(audio_path)
+                if 'cuesheet' in audio_meta.tags:
+                    tracks = parse_cue_sheet_manually(audio_meta.tags['cuesheet'][0])
+            except Exception:
+                pass # Already handled above
         
         if tracks:
             font_path = SYSTEM_FONTS_MAP.get(font_name)
@@ -297,26 +393,41 @@ def process_audio_to_video(
             
             # Use the robust parser for text colors as well
             font_bg_rgb = parse_color_to_rgb(font_bg_color)
-            font_bg_rgba = (*font_bg_rgb, int(font_bg_alpha * 255))
 
             position = (pos_h.lower(), pos_v.lower())
 
             print(f"Using font: {font_name}, Size: {font_size}, Position: {position}")
+            
+            # Create the RGBA tuple for the background color.
+            # The alpha value is converted from a 0.0-1.0 float to a 0-255 integer.
+            bg_color_tuple = (font_bg_rgb[0], font_bg_rgb[1], font_bg_rgb[2], int(font_bg_alpha * 255))
+            
+            # 1. Define a maximum width for the caption. 90% of the video width is a good choice.
+            caption_width = int(WIDTH * 0.9)
 
             for i, track in enumerate(tracks):
-                start_time = track.get('start_time', 0)
-                title, end_time = track.get('title', 'Unknown Track'), tracks[i+1].get('start_time', duration) if i + 1 < len(tracks) else duration
+                start_time, title = track.get('start_time', 0), track.get('title', 'Unknown Track')
+                end_time = tracks[i+1].get('start_time', duration) if i + 1 < len(tracks) else duration
                 text_duration = end_time - start_time
                 if text_duration <= 0: continue
                 
-                # Note: TextClip's `color` argument can handle color names like 'white' directly
-                txt_clip = (TextClip(text=f"{i+1}. {title}", font_size=font_size, color=font_color, font=font_path, bg_color=font_bg_rgba)
+                txt_clip = (TextClip(text=f"{i+1}. {title}",
+                                     font_size=font_size,
+                                     color=font_color,
+                                     font=font_path, 
+                                     bg_color=bg_color_tuple,
+                                     method='caption', # <-- Set method to caption
+                                     size=(caption_width, None)) # <-- Provide size for wrapping
                             .with_position(position)
                             .with_duration(text_duration)
                             .with_start(start_time))
                 text_clips.append(txt_clip)
 
-        final_clip = CompositeVideoClip([video_clip] + text_clips).with_audio(audio_clip)
+        # --- Clip Composition ---
+        # The final composition order is important: images at the bottom, then spectrogram, then text.
+        # The base layer is now the list of image clips.
+        final_layers = image_clips + [video_clip] + text_clips
+        final_clip = CompositeVideoClip(final_layers, size=(WIDTH, HEIGHT)).with_audio(audio_clip)
         
         # Step 1: Render the slow, 1 FPS intermediate file
         print(f"Step 1/2: Rendering base video at {RENDER_FPS} FPS...")
@@ -324,18 +435,18 @@ def process_audio_to_video(
             # Attempt to copy audio stream directly
             print("Attempting to copy audio stream directly...")
             final_clip.write_videofile(
-                temp_fps1_path, codec="libx264", audio_codec="copy", fps=RENDER_FPS, 
+                temp_fps1_path, codec="libx264", audio_codec="copy", fps=RENDER_FPS,
                 logger='bar', threads=os.cpu_count(), preset='ultrafast'
             )
             print("Audio stream successfully copied!")
         except Exception:
             # Fallback to AAC encoding if copy fails
-            print("Direct audio copy failed, falling back to AAC encoding...")
+            print("Direct audio copy failed, falling back to high-quality AAC encoding...")
             final_clip.write_videofile(
-                temp_fps1_path, codec="libx264", audio_codec="aac", fps=RENDER_FPS, 
-                logger='bar', threads=os.cpu_count(), preset='ultrafast'
-            )
-            print("AAC audio encoding complete.")
+                temp_fps1_path, codec="libx264", audio_codec="aac",
+                audio_bitrate="320k", fps=RENDER_FPS,
+                logger='bar', threads=os.cpu_count(), preset='ultrafast')
+            print("High-quality AAC audio encoding complete.")
         
         final_clip.close()
 
@@ -361,25 +472,46 @@ with gr.Blocks(title="Spectrogram Video Generator") as iface:
         with gr.Column(scale=1):
             audio_input = gr.Audio(type="filepath", label="Upload Audio File")
             
+            # --- Image Upload Component ---
+            gr.Markdown(
+                """
+                ### Background Image Options (Optional)
+
+                Upload one or more images to create a dynamic background for the video. The display behavior changes based on your audio file and the number of images provided.
+
+                *   **Mode 1: CUE Sheet Synchronization**
+                    If your audio file contains an embedded CUE sheet AND the number of images you upload **exactly matches** the number of tracks, the images will be synchronized with the tracks. The first image will appear during the first track, the second during the second, and so on.
+
+                *   **Mode 2: Even Time Distribution**
+                    In all other cases (e.g., the audio has no CUE sheet, or the number of images and tracks do not match), the images will be displayed sequentially. The total duration of the video will be divided equally among all uploaded images.
+
+                **Note:** When any image is used as a background, the spectrogram visualizer will automatically become **semi-transparent** to ensure the background is clearly visible.
+                """
+            )
+            image_uploads = gr.File(
+                label="Upload Background Images", 
+                file_count="multiple", # Allow multiple files
+                file_types=["image"]   # Accept only image formats
+            )
+            
             with gr.Accordion("Visualizer Options", open=True):
+                with gr.Row():
+                    width_input = gr.Number(value=1920, label="Video Width (px)", precision=0)
+                    height_input = gr.Number(value=1080, label="Video Height (px)", precision=0)
                 fg_color = gr.ColorPicker(value="#71808c", label="Spectrogram Bar Top Color")
-                bg_color = gr.ColorPicker(value="#2C3E50", label="Background Color")
+                bg_color = gr.ColorPicker(value="#2C3E50", label="Background Color (if no images)")
             
             with gr.Accordion("Text Overlay Options", open=True):
-                
-                # --- CORE CORRECTION: Add clarification text ---
                 gr.Markdown(
                     "**Note:** These options only take effect if the input audio file has an embedded CUE sheet."
                 )
-                gr.Markdown("---") # Add a separator line
-                # --- CORRECTION END ---
-
+                gr.Markdown("---")
                 gr.Markdown("If your CUE sheet contains non-English characters, please select a compatible font.")
                 default_font = "Microsoft JhengHei" if "Microsoft JhengHei" in FONT_DISPLAY_NAMES else ("Arial" if "Arial" in FONT_DISPLAY_NAMES else (FONT_DISPLAY_NAMES[0] if FONT_DISPLAY_NAMES else None))
                 font_name_dd = gr.Dropdown(choices=FONT_DISPLAY_NAMES, value=default_font, label="Font Family")
                 
                 with gr.Row():
-                    font_size_slider = gr.Slider(minimum=12, maximum=128, value=40, step=1, label="Font Size")
+                    font_size_slider = gr.Slider(minimum=12, maximum=256, value=80, step=1, label="Font Size")
                     font_color_picker = gr.ColorPicker(value="#FFFFFF", label="Font Color")
                 
                 with gr.Row():
@@ -396,10 +528,13 @@ with gr.Blocks(title="Spectrogram Video Generator") as iface:
         with gr.Column(scale=2):
             video_output = gr.Video(label="Generated Video")
     
+    # --- Add image_uploads to the inputs list ---
     submit_btn.click(
         fn=process_audio_to_video,
         inputs=[
-            audio_input, fg_color, bg_color, 
+            audio_input, image_uploads,
+            width_input, height_input,
+            fg_color, bg_color, 
             font_name_dd, font_size_slider, font_color_picker,
             font_bg_color_picker, font_bg_alpha_slider,
             pos_h_radio, pos_v_radio