work on basic monitor (no gpu for now)

README.md

@@ -120,6 +120,33 @@ As this is not automatic, then click on "Restart" in the space dev mode UI widge
 
 I haven't tested it, but you can try to provided Dockerfile
 
+### Prerequisites
+
+About Python:
+
+I haven't tested Python 3.11 or 3.12, but I noticed some incompatibilities with Python 3.13 dependencies failing to install.
+
+So I recommend you to install [pyenv](https://github.com/pyenv/pyenv) to switch between versions of Python.
+
+If you are on macOS, you might already have some versions of Python installed, you can see them by typing:
+
+```bash
+% python3.10 --version
+Python 3.10.16
+% python3.11 --version
+Python 3.11.11
+% python3.12 --version
+Python 3.12.9
+% python3.13 --version
+Python 3.13.2
+```
+
+Once pyenv is installed you can type:
+
+```bash
+pyenv install 3.10.16
+```
+
 ### Full installation in local
 
 the full installation requires:
@@ -127,7 +154,7 @@ the full installation requires:
 - CUDA 12
 - Python 3.10
 
-This is because of flash attention, which is defined in the `requirements.txt` using an URL to download a prebuilt wheel (python bindings for a native library)
+This is because of flash attention, which is defined in the `requirements.txt` using an URL to download a prebuilt wheel expecting this exact configuration (python bindings for a native library)
 
 ```bash
 ./setup.sh
@@ -153,7 +180,7 @@ Here is how to do solution 3:
 Note: please make sure you properly define the environment variables for `STORAGE_PATH` (eg. `/data/`) and `HF_HOME` (eg. `/data/huggingface/`)
 
 ```bash
-python app.py
+python3.10 app.py
 ```
 
 ### Running locally

app.py

@@ -14,6 +14,7 @@ from vms.config import (
     OUTPUT_PATH, ASK_USER_TO_DUPLICATE_SPACE, 
     HF_API_TOKEN
 )
+
 from vms.ui.video_trainer_ui import VideoTrainerUI
 
 # Configure logging
@@ -37,7 +38,9 @@ To avoid overpaying for your space, you can configure the auto-sleep settings to
 
     # Create the main application UI
     ui = VideoTrainerUI()
-    return ui.create_ui()
+    app = ui.create_ui()
+
+    return app
 
 def main():
     """Main entry point for the application"""

requirements.txt

@@ -2,6 +2,7 @@ numpy>=1.26.4
 
 # to quote a-r-r-o-w/finetrainers:
 # It is recommended to use Pytorch 2.5.1 or above for training. Previous versions can lead to completely black videos, OOM errors, or other issues and are not tested.
+# on some system (Python 3.13+) those do not work:
 torch==2.5.1
 torchvision==0.20.1
 torchao==0.6.1
@@ -41,4 +42,7 @@ git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
 
 # for our frontend
 gradio==5.20.1
-gradio_toggle
+gradio_toggle
+
+# used for the monitor
+matplotlib

requirements_without_flash_attention.txt

@@ -2,11 +2,12 @@ numpy>=1.26.4
 
 # to quote a-r-r-o-w/finetrainers:
 # It is recommended to use Pytorch 2.5.1 or above for training. Previous versions can lead to completely black videos, OOM errors, or other issues and are not tested.
+
+# on some system (Python 3.13+) those do not work:
 torch==2.5.1
 torchvision==0.20.1
 torchao==0.6.1
 
-
 huggingface_hub
 hf_transfer>=0.1.8
 diffusers @ git+https://github.com/huggingface/diffusers.git@main
@@ -40,4 +41,7 @@ git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
 
 # for our frontend
 gradio==5.20.1
-gradio_toggle
+gradio_toggle
+
+# used for the monitor
+matplotlib

run.sh

@@ -2,4 +2,12 @@
 
 source .venv/bin/activate
 
-USE_MOCK_CAPTIONING_MODEL=True python app.py
+echo "if run.sh fails due to python being not found, edit run.sh to replace with another version of python"
+
+# if you are on a mac, you can try to replace "python3.10" with:
+# python3.10
+# python3.11 (not tested)
+# python3.12 (not tested)
+# python3.13 (tested, fails to install)
+
+USE_MOCK_CAPTIONING_MODEL=True python3.10 app.py

setup_no_captions.sh

@@ -1,10 +1,18 @@
 #!/usr/bin/env bash
 
-python -m venv .venv
+echo "if install fails due to python being not found, edit setup_no_captions.sh to replace with another version of python"
+
+# if you are on a mac, you can try to replace "python3.10" with:
+# python3.10
+# python3.11 (not tested)
+# python3.12 (not tested)
+# python3.13 (tested, fails to install)
+
+python3.10 -m venv .venv
 
 source .venv/bin/activate
 
-python -m pip install -r requirements_without_flash_attention.txt
+python3.10 -m pip install -r requirements_without_flash_attention.txt
 
 # if you require flash attention, please install it manually for your operating system
 

vms/services/__init__.py

@@ -1,5 +1,6 @@
 from .captioner import CaptioningProgress, CaptioningService
 from .importer import ImportService
+from .monitoring import MonitoringService
 from .splitter import SplittingService
 from .trainer import TrainingService
 
@@ -7,6 +8,7 @@ __all__ = [
     'CaptioningProgress',
     'CaptioningService',
     'ImportService',
+    'MonitoringService',
     'SplittingService',
     'TrainingService',
 ]

vms/services/importer/__init__.py

@@ -0,0 +1,11 @@
+"""
+Import module for Video Model Studio.
+Handles file uploads, YouTube downloads, and Hugging Face Hub dataset integration.
+"""
+
+from .import_service import ImportService
+from .file_upload import FileUploadHandler
+from .youtube import YouTubeDownloader
+from .hub_dataset import HubDatasetBrowser
+
+__all__ = ['ImportService', 'FileUploadHandler', 'YouTubeDownloader', 'HubDatasetBrowser']

vms/services/{importer.py → importer/file_upload.py}

@@ -1,3 +1,8 @@
+"""
+File upload handler for Video Model Studio.
+Processes uploaded files including videos, images, ZIPs, and WebDataset archives.
+"""
+
 import os
 import shutil
 import zipfile
@@ -5,16 +10,18 @@ import tarfile
 import tempfile
 import gradio as gr
 from pathlib import Path
-from typing import List, Dict, Optional, Tuple
-from pytubefix import YouTube
+from typing import List, Dict, Optional, Tuple, Any, Union
 import logging
+import traceback
 
-from ..config import NORMALIZE_IMAGES_TO, TRAINING_VIDEOS_PATH, VIDEOS_TO_SPLIT_PATH, STAGING_PATH, DEFAULT_PROMPT_PREFIX
-from ..utils import normalize_image, is_image_file, is_video_file, add_prefix_to_caption, webdataset_handler
+from vms.config import NORMALIZE_IMAGES_TO, TRAINING_VIDEOS_PATH, VIDEOS_TO_SPLIT_PATH, STAGING_PATH, DEFAULT_PROMPT_PREFIX
+from vms.utils import normalize_image, is_image_file, is_video_file, add_prefix_to_caption, webdataset_handler
 
 logger = logging.getLogger(__name__)
 
-class ImportService:
+class FileUploadHandler:
+    """Handles processing of uploaded files"""
+    
     def process_uploaded_files(self, file_paths: List[str]) -> str:
         """Process uploaded file (ZIP, TAR, MP4, or image)
         
@@ -24,11 +31,15 @@ class ImportService:
         Returns:
             Status message string
         """
+        if not file_paths or len(file_paths) == 0:
+            logger.warning("No files provided to process_uploaded_files")
+            return "No files provided"
+            
         for file_path in file_paths:
             file_path = Path(file_path)
             try:
                 original_name = file_path.name
-                print("original_name = ", original_name)
+                logger.info(f"Processing uploaded file: {original_name}")
 
                 # Determine file type from name
                 file_ext = file_path.suffix.lower()
@@ -42,9 +53,11 @@ class ImportService:
                 elif is_image_file(file_path):
                     return self.process_image_file(file_path, original_name)
                 else:
+                    logger.error(f"Unsupported file type: {file_ext}")
                     raise gr.Error(f"Unsupported file type: {file_ext}")
 
             except Exception as e:
+                logger.error(f"Error processing file {file_path}: {str(e)}", exc_info=True)
                 raise gr.Error(f"Error processing file: {str(e)}")
 
     def process_image_file(self, file_path: Path, original_name: str) -> str:
@@ -68,10 +81,13 @@ class ImportService:
                 target_path = STAGING_PATH / f"{stem}___{counter}.{NORMALIZE_IMAGES_TO}"
                 counter += 1
 
+            logger.info(f"Processing image file: {original_name} -> {target_path}")
+            
             # Convert to normalized format and remove black bars
             success = normalize_image(file_path, target_path)
             
             if not success:
+                logger.error(f"Failed to process image: {original_name}")
                 raise gr.Error(f"Failed to process image: {original_name}")
 
             # Handle caption
@@ -86,6 +102,7 @@ class ImportService:
             return f"Successfully stored image: {target_path.name}"
 
         except Exception as e:
+            logger.error(f"Error processing image file: {str(e)}", exc_info=True)
             raise gr.Error(f"Error processing image file: {str(e)}")
 
     def process_zip_file(self, file_path: Path) -> str:
@@ -102,6 +119,8 @@ class ImportService:
             image_count = 0
             tar_count = 0
             
+            logger.info(f"Processing ZIP file: {file_path}")
+            
             # Create temporary directory
             with tempfile.TemporaryDirectory() as temp_dir:
                 # Extract ZIP
@@ -121,6 +140,7 @@ class ImportService:
                         try:
                             # Check if it's a WebDataset tar file
                             if file.lower().endswith('.tar'):
+                                logger.info(f"Processing WebDataset archive from ZIP: {file}")
                                 # Process WebDataset shard
                                 vid_count, img_count = webdataset_handler.process_webdataset_shard(
                                     file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
@@ -136,6 +156,7 @@ class ImportService:
                                     target_path = VIDEOS_TO_SPLIT_PATH / f"{file_path.stem}___{counter}{file_path.suffix}"
                                     counter += 1
                                 shutil.copy2(file_path, target_path)
+                                logger.info(f"Extracted video from ZIP: {file} -> {target_path.name}")
                                 video_count += 1
                                 
                             elif is_image_file(file_path):
@@ -146,6 +167,7 @@ class ImportService:
                                     target_path = STAGING_PATH / f"{file_path.stem}___{counter}.{NORMALIZE_IMAGES_TO}"
                                     counter += 1
                                 if normalize_image(file_path, target_path):
+                                    logger.info(f"Extracted image from ZIP: {file} -> {target_path.name}")
                                     image_count += 1
                                 
                             # Copy associated caption file if it exists
@@ -153,13 +175,15 @@ class ImportService:
                             if txt_path.exists() and not file.lower().endswith('.tar'):
                                 if is_video_file(file_path):
                                     shutil.copy2(txt_path, target_path.with_suffix('.txt'))
+                                    logger.info(f"Copied caption file for {file}")
                                 elif is_image_file(file_path):
                                     caption = txt_path.read_text()
                                     caption = add_prefix_to_caption(caption, DEFAULT_PROMPT_PREFIX)
                                     target_path.with_suffix('.txt').write_text(caption)
+                                    logger.info(f"Processed caption for {file}")
                                     
                         except Exception as e:
-                            logger.error(f"Error processing {file_path.name}: {str(e)}")
+                            logger.error(f"Error processing {file_path.name} from ZIP: {str(e)}", exc_info=True)
                             continue
 
             # Generate status message
@@ -172,13 +196,16 @@ class ImportService:
                 parts.append(f"{image_count} image{'s' if image_count != 1 else ''}")
                 
             if not parts:
+                logger.warning("No supported media files found in ZIP")
                 return "No supported media files found in ZIP"
                 
             status = f"Successfully stored {', '.join(parts)}"
+            logger.info(status)
             gr.Info(status)
             return status
             
         except Exception as e:
+            logger.error(f"Error processing ZIP: {str(e)}", exc_info=True)
             raise gr.Error(f"Error processing ZIP: {str(e)}")
 
     def process_tar_file(self, file_path: Path) -> str:
@@ -191,6 +218,7 @@ class ImportService:
             Status message string
         """
         try:
+            logger.info(f"Processing WebDataset TAR file: {file_path}")
             video_count, image_count = webdataset_handler.process_webdataset_shard(
                 file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
             )
@@ -203,13 +231,16 @@ class ImportService:
                 parts.append(f"{image_count} image{'s' if image_count != 1 else ''}")
                 
             if not parts:
+                logger.warning("No supported media files found in WebDataset")
                 return "No supported media files found in WebDataset"
                 
             status = f"Successfully extracted {' and '.join(parts)} from WebDataset"
+            logger.info(status)
             gr.Info(status)
             return status
             
         except Exception as e:
+            logger.error(f"Error processing WebDataset tar file: {str(e)}", exc_info=True)
             raise gr.Error(f"Error processing WebDataset tar file: {str(e)}")
 
     def process_mp4_file(self, file_path: Path, original_name: str) -> str:
@@ -233,60 +264,15 @@ class ImportService:
                 target_path = VIDEOS_TO_SPLIT_PATH / f"{stem}___{counter}.mp4"
                 counter += 1
 
+            logger.info(f"Processing video file: {original_name} -> {target_path}")
+            
             # Copy the file to the target location
             shutil.copy2(file_path, target_path)
 
+            logger.info(f"Successfully stored video: {target_path.name}")
             gr.Info(f"Successfully stored video: {target_path.name}")
             return f"Successfully stored video: {target_path.name}"
 
         except Exception as e:
-            raise gr.Error(f"Error processing video file: {str(e)}")
-
-    def download_youtube_video(self, url: str, progress=None) -> Dict:
-        """Download a video from YouTube
-        
-        Args:
-            url: YouTube video URL
-            progress: Optional Gradio progress indicator
-            
-        Returns:
-            Dict with status message and error (if any)
-        """
-        try:
-            # Extract video ID and create YouTube object
-            yt = YouTube(url, on_progress_callback=lambda stream, chunk, bytes_remaining: 
-                progress((1 - bytes_remaining / stream.filesize), desc="Downloading...")
-                if progress else None)
-            
-            video_id = yt.video_id
-            output_path = VIDEOS_TO_SPLIT_PATH / f"{video_id}.mp4"
-            
-            # Download highest quality progressive MP4
-            if progress:
-                print("Getting video streams...")
-                progress(0, desc="Getting video streams...")
-            video = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first()
-            
-            if not video:
-                print("Could not find a compatible video format")
-                gr.Error("Could not find a compatible video format")
-                return "Could not find a compatible video format"
-            
-            # Download the video
-            if progress:
-                print("Starting YouTube video download...")
-                progress(0, desc="Starting download...")
-            
-            video.download(output_path=str(VIDEOS_TO_SPLIT_PATH), filename=f"{video_id}.mp4")
-            
-            # Update UI
-            if progress:
-                print("YouTube video download complete!")
-                gr.Info("YouTube video download complete!")
-                progress(1, desc="Download complete!")
-            return f"Successfully downloaded video: {yt.title}"
-            
-        except Exception as e:
-            print(e)
-            gr.Error(f"Error downloading video: {str(e)}")
-            return f"Error downloading video: {str(e)}"
+            logger.error(f"Error processing video file: {str(e)}", exc_info=True)
+            raise gr.Error(f"Error processing video file: {str(e)}")

vms/services/importer/hub_dataset.py

@@ -0,0 +1,521 @@
+"""
+Hugging Face Hub dataset browser for Video Model Studio.
+Handles searching, viewing, and downloading datasets from the Hub.
+"""
+
+import os
+import shutil
+import tempfile
+import asyncio
+import logging
+import gradio as gr
+from pathlib import Path
+from typing import List, Dict, Optional, Tuple, Any, Union
+
+from huggingface_hub import (
+    HfApi, 
+    hf_hub_download, 
+    snapshot_download, 
+    list_datasets
+)
+
+from vms.config import NORMALIZE_IMAGES_TO, TRAINING_VIDEOS_PATH, VIDEOS_TO_SPLIT_PATH, STAGING_PATH, DEFAULT_PROMPT_PREFIX
+from vms.utils import normalize_image, is_image_file, is_video_file, add_prefix_to_caption, webdataset_handler
+
+logger = logging.getLogger(__name__)
+
+class HubDatasetBrowser:
+    """Handles interactions with Hugging Face Hub datasets"""
+    
+    def __init__(self, hf_api: HfApi):
+        """Initialize with HfApi instance
+        
+        Args:
+            hf_api: Hugging Face Hub API instance
+        """
+        self.hf_api = hf_api
+    
+    def search_datasets(self, query: str) -> List[List[str]]:
+        """Search for datasets on the Hugging Face Hub
+        
+        Args:
+            query: Search query string
+            
+        Returns:
+            List of datasets matching the query [id, title, downloads]
+        """
+        try:
+            # Start with some filters to find video-related datasets
+            search_terms = query.strip() if query and query.strip() else "video"
+            logger.info(f"Searching datasets with query: '{search_terms}'")
+            
+            # Fetch datasets that match the search
+            datasets = list(self.hf_api.list_datasets(
+                search=search_terms,
+                limit=50
+            ))
+            
+            # Format results for display
+            results = []
+            for ds in datasets:
+                # Extract relevant information
+                dataset_id = ds.id
+                
+                # Safely get the title with fallbacks
+                card_data = getattr(ds, "card_data", None)
+                title = ""
+                
+                if card_data is not None and isinstance(card_data, dict):
+                    title = card_data.get("name", "")
+                
+                if not title:
+                    # Use the last part of the repo ID as a fallback
+                    title = dataset_id.split("/")[-1]
+                
+                # Safely get downloads
+                downloads = getattr(ds, "downloads", 0)
+                if downloads is None:
+                    downloads = 0
+                
+                results.append([dataset_id, title, downloads])
+            
+            # Sort by downloads (most downloaded first)
+            results.sort(key=lambda x: x[2] if x[2] is not None else 0, reverse=True)
+            
+            logger.info(f"Found {len(results)} datasets matching '{search_terms}'")
+            return results
+        
+        except Exception as e:
+            logger.error(f"Error searching datasets: {str(e)}", exc_info=True)
+            return [[f"Error: {str(e)}", "", ""]]
+            
+    def get_dataset_info(self, dataset_id: str) -> Tuple[str, Dict[str, int], Dict[str, List[str]]]:
+        """Get detailed information about a dataset
+        
+        Args:
+            dataset_id: The dataset ID to get information for
+            
+        Returns:
+            Tuple of (markdown_info, file_counts, file_groups)
+            - markdown_info: Markdown formatted string with dataset information
+            - file_counts: Dictionary with counts of each file type
+            - file_groups: Dictionary with lists of filenames grouped by type
+        """
+        try:
+            if not dataset_id:
+                logger.warning("No dataset ID provided to get_dataset_info")
+                return "No dataset selected", {}, {}
+                
+            logger.info(f"Getting info for dataset: {dataset_id}")
+                
+            # Get detailed information about the dataset
+            dataset_info = self.hf_api.dataset_info(dataset_id)
+            
+            # Format the information for display
+            info_text = f"## {dataset_info.id}\n\n"
+            
+            # Add description if available (with safer access)
+            card_data = getattr(dataset_info, "card_data", None)
+            description = ""
+            
+            if card_data is not None and isinstance(card_data, dict):
+                description = card_data.get("description", "")
+                
+            if description:
+                info_text += f"{description[:500]}{'...' if len(description) > 500 else ''}\n\n"
+            
+            # Add basic stats (with safer access)
+            downloads = getattr(dataset_info, 'downloads', None)
+            info_text += f"**Downloads:** {downloads if downloads is not None else 'N/A'}\n"
+            
+            last_modified = getattr(dataset_info, 'last_modified', None)
+            info_text += f"**Last modified:** {last_modified if last_modified is not None else 'N/A'}\n"
+            
+            # Show tags if available (with safer access)
+            tags = getattr(dataset_info, "tags", None) or []
+            if tags:
+                info_text += f"**Tags:** {', '.join(tags[:10])}\n\n"
+            
+            # Group files by type
+            file_groups = {
+                "video": [],
+                "webdataset": []
+            }
+            
+            siblings = getattr(dataset_info, "siblings", None) or []
+            
+            # Extract files by type
+            for s in siblings:
+                if not hasattr(s, 'rfilename'):
+                    continue
+                    
+                filename = s.rfilename
+                if filename.lower().endswith((".mp4", ".webm")):
+                    file_groups["video"].append(filename)
+                elif filename.lower().endswith(".tar"):
+                    file_groups["webdataset"].append(filename)
+            
+            # Create file counts dictionary
+            file_counts = {
+                "video": len(file_groups["video"]),
+                "webdataset": len(file_groups["webdataset"])
+            }
+            
+            logger.info(f"Successfully retrieved info for dataset: {dataset_id}")
+            return info_text, file_counts, file_groups
+            
+        except Exception as e:
+            logger.error(f"Error getting dataset info: {str(e)}", exc_info=True)
+            return f"Error loading dataset information: {str(e)}", {}, {}
+    
+    async def download_file_group(self, dataset_id: str, file_type: str, enable_splitting: bool = True) -> str:
+        """Download all files of a specific type from the dataset
+        
+        Args:
+            dataset_id: The dataset ID
+            file_type: Either "video" or "webdataset"
+            enable_splitting: Whether to enable automatic video splitting
+            
+        Returns:
+            Status message
+        """
+        try:
+            # Get dataset info to retrieve file list
+            _, _, file_groups = self.get_dataset_info(dataset_id)
+            
+            # Get the list of files for the specified type
+            files = file_groups.get(file_type, [])
+            
+            if not files:
+                return f"No {file_type} files found in the dataset"
+            
+            logger.info(f"Downloading {len(files)} {file_type} files from dataset {dataset_id}")
+            
+            # Track counts for status message
+            video_count = 0
+            image_count = 0
+            
+            # Create a temporary directory for downloads
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+                
+                # Process all files of the requested type
+                for filename in files:
+                    try:
+                        # Download the file
+                        file_path = hf_hub_download(
+                            repo_id=dataset_id,
+                            filename=filename,
+                            repo_type="dataset",
+                            local_dir=temp_path
+                        )
+                        
+                        file_path = Path(file_path)
+                        logger.info(f"Downloaded file to {file_path}")
+                        
+                        # Process based on file type
+                        if file_type == "video":
+                            # Choose target directory based on auto-splitting setting
+                            target_dir = VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH
+                            target_path = target_dir / file_path.name
+                            
+                            # Make sure filename is unique
+                            counter = 1
+                            while target_path.exists():
+                                stem = Path(file_path.name).stem
+                                if "___" in stem:
+                                    base_stem = stem.split("___")[0]
+                                else:
+                                    base_stem = stem
+                                target_path = target_dir / f"{base_stem}___{counter}{Path(file_path.name).suffix}"
+                                counter += 1
+                                
+                            # Copy the video file
+                            shutil.copy2(file_path, target_path)
+                            logger.info(f"Processed video: {file_path.name} -> {target_path.name}")
+                            
+                            # Try to download caption if it exists
+                            try:
+                                txt_filename = f"{Path(filename).stem}.txt"
+                                for possible_path in [
+                                    Path(filename).with_suffix('.txt').as_posix(),
+                                    (Path(filename).parent / txt_filename).as_posix(),
+                                ]:
+                                    try:
+                                        txt_path = hf_hub_download(
+                                            repo_id=dataset_id,
+                                            filename=possible_path,
+                                            repo_type="dataset",
+                                            local_dir=temp_path
+                                        )
+                                        shutil.copy2(txt_path, target_path.with_suffix('.txt'))
+                                        logger.info(f"Copied caption for {file_path.name}")
+                                        break
+                                    except Exception:
+                                        # Caption file doesn't exist at this path, try next
+                                        pass
+                            except Exception as e:
+                                logger.warning(f"Error trying to download caption: {e}")
+                            
+                            video_count += 1
+                            
+                        elif file_type == "webdataset":
+                            # Process the WebDataset archive
+                            try:
+                                logger.info(f"Processing WebDataset file: {file_path}")
+                                vid_count, img_count = webdataset_handler.process_webdataset_shard(
+                                    file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
+                                )
+                                video_count += vid_count
+                                image_count += img_count
+                            except Exception as e:
+                                logger.error(f"Error processing WebDataset file {file_path}: {str(e)}", exc_info=True)
+                    
+                    except Exception as e:
+                        logger.warning(f"Error processing file {filename}: {e}")
+                
+                # Generate status message
+                if file_type == "video":
+                    return f"Successfully imported {video_count} videos from dataset {dataset_id}"
+                elif file_type == "webdataset":
+                    parts = []
+                    if video_count > 0:
+                        parts.append(f"{video_count} video{'s' if video_count != 1 else ''}")
+                    if image_count > 0:
+                        parts.append(f"{image_count} image{'s' if image_count != 1 else ''}")
+                        
+                    if parts:
+                        return f"Successfully imported {' and '.join(parts)} from WebDataset archives"
+                    else:
+                        return f"No media was found in the WebDataset archives"
+                
+                return f"Unknown file type: {file_type}"
+                
+        except Exception as e:
+            error_msg = f"Error downloading {file_type} files: {str(e)}"
+            logger.error(error_msg, exc_info=True)
+            return error_msg
+    
+    async def download_dataset(self, dataset_id: str, enable_splitting: bool = True) -> Tuple[str, str]:
+        """Download a dataset and process its video/image content
+        
+        Args:
+            dataset_id: The dataset ID to download
+            enable_splitting: Whether to enable automatic video splitting
+            
+        Returns:
+            Tuple of (loading_msg, status_msg)
+        """
+        if not dataset_id:
+            logger.warning("No dataset ID provided for download")
+            return "No dataset selected", "Please select a dataset first"
+        
+        try:
+            logger.info(f"Starting download of dataset: {dataset_id}")
+            loading_msg = f"## Downloading dataset: {dataset_id}\n\nThis may take some time depending on the dataset size..."
+            status_msg = f"Downloading dataset: {dataset_id}..."
+            
+            # Get dataset info to check for available files
+            dataset_info = self.hf_api.dataset_info(dataset_id)
+            
+            # Check if there are video files or WebDataset files
+            video_files = []
+            tar_files = []
+            
+            siblings = getattr(dataset_info, "siblings", None) or []
+            if siblings:
+                video_files = [s.rfilename for s in siblings if hasattr(s, 'rfilename') and s.rfilename.lower().endswith((".mp4", ".webm"))]
+                tar_files = [s.rfilename for s in siblings if hasattr(s, 'rfilename') and s.rfilename.lower().endswith(".tar")]
+            
+            # Create a temporary directory for downloads
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_path = Path(temp_dir)
+                
+                # If we have video files, download them individually
+                if video_files:
+                    loading_msg = f"{loading_msg}\n\nDownloading {len(video_files)} video files..."
+                    logger.info(f"Downloading {len(video_files)} video files from {dataset_id}")
+                    
+                    for i, video_file in enumerate(video_files):
+                        # Download the video file
+                        try:
+                            file_path = hf_hub_download(
+                                repo_id=dataset_id,
+                                filename=video_file,
+                                repo_type="dataset",
+                                local_dir=temp_path
+                            )
+                            
+                            # Look for associated caption file
+                            txt_filename = f"{Path(video_file).stem}.txt"
+                            txt_path = None
+                            for possible_path in [
+                                Path(video_file).with_suffix('.txt').as_posix(),
+                                (Path(video_file).parent / txt_filename).as_posix(),
+                            ]:
+                                try:
+                                    txt_path = hf_hub_download(
+                                        repo_id=dataset_id,
+                                        filename=possible_path,
+                                        repo_type="dataset",
+                                        local_dir=temp_path
+                                    )
+                                    logger.info(f"Found caption file for {video_file}: {possible_path}")
+                                    break
+                                except Exception as e:
+                                    # Caption file doesn't exist at this path, try next
+                                    logger.debug(f"No caption at {possible_path}: {str(e)}")
+                                    pass
+                                
+                            status_msg = f"Downloaded video {i+1}/{len(video_files)} from {dataset_id}"
+                            logger.info(status_msg)
+                        except Exception as e:
+                            logger.warning(f"Error downloading {video_file}: {e}")
+                
+                # If we have tar files, download them
+                if tar_files:
+                    loading_msg = f"{loading_msg}\n\nDownloading {len(tar_files)} WebDataset files..."
+                    logger.info(f"Downloading {len(tar_files)} WebDataset files from {dataset_id}")
+                    
+                    for i, tar_file in enumerate(tar_files):
+                        try:
+                            file_path = hf_hub_download(
+                                repo_id=dataset_id,
+                                filename=tar_file,
+                                repo_type="dataset",
+                                local_dir=temp_path
+                            )
+                            status_msg = f"Downloaded WebDataset {i+1}/{len(tar_files)} from {dataset_id}"
+                            logger.info(status_msg)
+                        except Exception as e:
+                            logger.warning(f"Error downloading {tar_file}: {e}")
+                
+                # If no specific files were found, try downloading the entire repo
+                if not video_files and not tar_files:
+                    loading_msg = f"{loading_msg}\n\nDownloading entire dataset repository..."
+                    logger.info(f"No specific media files found, downloading entire repository for {dataset_id}")
+                    
+                    try:
+                        snapshot_download(
+                            repo_id=dataset_id,
+                            repo_type="dataset",
+                            local_dir=temp_path
+                        )
+                        status_msg = f"Downloaded entire repository for {dataset_id}"
+                        logger.info(status_msg)
+                    except Exception as e:
+                        logger.error(f"Error downloading dataset snapshot: {e}", exc_info=True)
+                        return loading_msg, f"Error downloading dataset: {str(e)}"
+                
+                # Process the downloaded files
+                loading_msg = f"{loading_msg}\n\nProcessing downloaded files..."
+                logger.info(f"Processing downloaded files from {dataset_id}")
+                
+                # Count imported files
+                video_count = 0
+                image_count = 0
+                tar_count = 0
+                
+                # Process function for the event loop
+                async def process_files():
+                    nonlocal video_count, image_count, tar_count
+                    
+                    # Process all files in the temp directory
+                    for root, _, files in os.walk(temp_path):
+                        for file in files:
+                            file_path = Path(root) / file
+                            
+                            # Process videos
+                            if file.lower().endswith((".mp4", ".webm")):
+                                # Choose target path based on auto-splitting setting
+                                target_dir = VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH
+                                target_path = target_dir / file_path.name
+                                
+                                # Make sure filename is unique
+                                counter = 1
+                                while target_path.exists():
+                                    stem = Path(file_path.name).stem
+                                    if "___" in stem:
+                                        base_stem = stem.split("___")[0]
+                                    else:
+                                        base_stem = stem
+                                    target_path = target_dir / f"{base_stem}___{counter}{Path(file_path.name).suffix}"
+                                    counter += 1
+                                    
+                                # Copy the video file
+                                shutil.copy2(file_path, target_path)
+                                logger.info(f"Processed video from dataset: {file_path.name} -> {target_path.name}")
+                                
+                                # Copy associated caption file if it exists
+                                txt_path = file_path.with_suffix('.txt')
+                                if txt_path.exists():
+                                    shutil.copy2(txt_path, target_path.with_suffix('.txt'))
+                                    logger.info(f"Copied caption for {file_path.name}")
+                                    
+                                video_count += 1
+                                
+                            # Process images
+                            elif is_image_file(file_path):
+                                target_path = STAGING_PATH / f"{file_path.stem}.{NORMALIZE_IMAGES_TO}"
+                                
+                                counter = 1
+                                while target_path.exists():
+                                    target_path = STAGING_PATH / f"{file_path.stem}___{counter}.{NORMALIZE_IMAGES_TO}"
+                                    counter += 1
+                                    
+                                if normalize_image(file_path, target_path):
+                                    logger.info(f"Processed image from dataset: {file_path.name} -> {target_path.name}")
+                                    
+                                    # Copy caption if available
+                                    txt_path = file_path.with_suffix('.txt')
+                                    if txt_path.exists():
+                                        caption = txt_path.read_text()
+                                        caption = add_prefix_to_caption(caption, DEFAULT_PROMPT_PREFIX)
+                                        target_path.with_suffix('.txt').write_text(caption)
+                                        logger.info(f"Processed caption for {file_path.name}")
+                                    
+                                    image_count += 1
+                                
+                            # Process WebDataset files
+                            elif file.lower().endswith(".tar"):
+                                # Process the WebDataset archive
+                                try:
+                                    logger.info(f"Processing WebDataset file from dataset: {file}")
+                                    vid_count, img_count = webdataset_handler.process_webdataset_shard(
+                                        file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
+                                    )
+                                    tar_count += 1
+                                    video_count += vid_count
+                                    image_count += img_count
+                                    logger.info(f"Extracted {vid_count} videos and {img_count} images from {file}")
+                                except Exception as e:
+                                    logger.error(f"Error processing WebDataset file {file_path}: {str(e)}", exc_info=True)
+                                    
+                # Run the processing asynchronously
+                await process_files()
+                
+                # Generate final status message
+                parts = []
+                if video_count > 0:
+                    parts.append(f"{video_count} video{'s' if video_count != 1 else ''}")
+                if image_count > 0:
+                    parts.append(f"{image_count} image{'s' if image_count != 1 else ''}")
+                if tar_count > 0:
+                    parts.append(f"{tar_count} WebDataset archive{'s' if tar_count != 1 else ''}")
+                
+                if parts:
+                    status = f"Successfully imported {', '.join(parts)} from dataset {dataset_id}"
+                    loading_msg = f"{loading_msg}\n\n✅ Success! {status}"
+                    logger.info(status)
+                else:
+                    status = f"No supported media files found in dataset {dataset_id}"
+                    loading_msg = f"{loading_msg}\n\n⚠️ {status}"
+                    logger.warning(status)
+                
+                gr.Info(status)
+                return loading_msg, status
+                
+        except Exception as e:
+            error_msg = f"Error downloading dataset {dataset_id}: {str(e)}"
+            logger.error(error_msg, exc_info=True)
+            return f"Error: {error_msg}", error_msg

vms/services/importer/import_service.py

@@ -0,0 +1,102 @@
+"""
+Main Import Service for Video Model Studio.
+Delegates to specialized handler classes for different import types.
+"""
+
+import logging
+from typing import List, Dict, Optional, Tuple, Any, Union
+from pathlib import Path
+import gradio as gr
+
+from huggingface_hub import HfApi
+
+from .file_upload import FileUploadHandler
+from .youtube import YouTubeDownloader
+from .hub_dataset import HubDatasetBrowser
+from vms.config import HF_API_TOKEN
+
+logger = logging.getLogger(__name__)
+
+class ImportService:
+    """Main service class for handling imports from various sources"""
+    
+    def __init__(self):
+        """Initialize the import service and handlers"""
+        self.hf_api = HfApi(token=HF_API_TOKEN)
+        self.file_handler = FileUploadHandler()
+        self.youtube_handler = YouTubeDownloader()
+        self.hub_browser = HubDatasetBrowser(self.hf_api)
+    
+    def process_uploaded_files(self, file_paths: List[str]) -> str:
+        """Process uploaded file (ZIP, TAR, MP4, or image)
+        
+        Args:
+            file_paths: File paths to the uploaded files from Gradio
+                
+        Returns:
+            Status message string
+        """
+        if not file_paths or len(file_paths) == 0:
+            logger.warning("No files provided to process_uploaded_files")
+            return "No files provided"
+        
+        return self.file_handler.process_uploaded_files(file_paths)
+    
+    def download_youtube_video(self, url: str, progress=None) -> str:
+        """Download a video from YouTube
+        
+        Args:
+            url: YouTube video URL
+            progress: Optional Gradio progress indicator
+            
+        Returns:
+            Status message string
+        """
+        return self.youtube_handler.download_video(url, progress)
+    
+    def search_datasets(self, query: str) -> List[List[str]]:
+        """Search for datasets on the Hugging Face Hub
+        
+        Args:
+            query: Search query string
+            
+        Returns:
+            List of datasets matching the query [id, title, downloads]
+        """
+        return self.hub_browser.search_datasets(query)
+    
+    def get_dataset_info(self, dataset_id: str) -> Tuple[str, Dict[str, int], Dict[str, List[str]]]:
+        """Get detailed information about a dataset
+        
+        Args:
+            dataset_id: The dataset ID to get information for
+            
+        Returns:
+            Tuple of (markdown_info, file_counts, file_groups)
+        """
+        return self.hub_browser.get_dataset_info(dataset_id)
+    
+    async def download_dataset(self, dataset_id: str, enable_splitting: bool = True) -> Tuple[str, str]:
+        """Download a dataset and process its video/image content
+        
+        Args:
+            dataset_id: The dataset ID to download
+            enable_splitting: Whether to enable automatic video splitting
+            
+        Returns:
+            Tuple of (loading_msg, status_msg)
+        """
+        return await self.hub_browser.download_dataset(dataset_id, enable_splitting)
+    
+    async def download_file_group(self, dataset_id: str, file_type: str, enable_splitting: bool = True) -> str:
+        """Download a group of files (videos or WebDatasets)
+        
+        Args:
+            dataset_id: The dataset ID
+            file_type: Type of file ("video" or "webdataset")
+            enable_splitting: Whether to enable automatic video splitting
+            
+        Returns:
+            Status message
+        """
+        return await self.hub_browser.download_file_group(dataset_id, file_type, enable_splitting)

vms/services/importer/youtube.py

@@ -0,0 +1,73 @@
+"""
+YouTube downloader for Video Model Studio.
+Handles downloading videos from YouTube URLs.
+"""
+
+import logging
+import gradio as gr
+from pathlib import Path
+from typing import Optional, Any, Union, Callable
+
+from pytubefix import YouTube
+
+from vms.config import VIDEOS_TO_SPLIT_PATH
+
+logger = logging.getLogger(__name__)
+
+class YouTubeDownloader:
+    """Handles downloading videos from YouTube"""
+    
+    def download_video(self, url: str, progress: Optional[Callable] = None) -> str:
+        """Download a video from YouTube
+        
+        Args:
+            url: YouTube video URL
+            progress: Optional Gradio progress indicator
+            
+        Returns:
+            Status message string
+        """
+        if not url or not url.strip():
+            logger.warning("No YouTube URL provided")
+            return "Please enter a YouTube URL"
+            
+        try:
+            logger.info(f"Downloading YouTube video: {url}")
+            
+            # Extract video ID and create YouTube object
+            yt = YouTube(url, on_progress_callback=lambda stream, chunk, bytes_remaining: 
+                progress((1 - bytes_remaining / stream.filesize), desc="Downloading...")
+                if progress else None)
+            
+            video_id = yt.video_id
+            output_path = VIDEOS_TO_SPLIT_PATH / f"{video_id}.mp4"
+            
+            # Download highest quality progressive MP4
+            if progress:
+                logger.debug("Getting video streams...")
+                progress(0, desc="Getting video streams...")
+            video = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first()
+            
+            if not video:
+                logger.error("Could not find a compatible video format")
+                gr.Error("Could not find a compatible video format")
+                return "Could not find a compatible video format"
+            
+            # Download the video
+            if progress:
+                logger.info("Starting YouTube video download...")
+                progress(0, desc="Starting download...")
+            
+            video.download(output_path=str(VIDEOS_TO_SPLIT_PATH), filename=f"{video_id}.mp4")
+            
+            # Update UI
+            if progress:
+                logger.info("YouTube video download complete!")
+                gr.Info("YouTube video download complete!")
+                progress(1, desc="Download complete!")
+            return f"Successfully downloaded video: {yt.title}"
+            
+        except Exception as e:
+            logger.error(f"Error downloading YouTube video: {str(e)}", exc_info=True)
+            gr.Error(f"Error downloading video: {str(e)}")
+            return f"Error downloading video: {str(e)}"

vms/services/monitoring.py

@@ -0,0 +1,361 @@
+"""
+System monitoring service for Video Model Studio.
+Tracks system resources like CPU, memory, and other metrics.
+"""
+
+import os
+import time
+import logging
+import platform
+import threading
+from datetime import datetime, timedelta
+from collections import deque
+from typing import Dict, List, Optional, Tuple, Any
+
+import psutil
+
+# Force the use of the Agg backend which is thread-safe
+import matplotlib
+matplotlib.use('Agg')  # Must be before importing pyplot
+import matplotlib.pyplot as plt
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+class MonitoringService:
+    """Service for monitoring system resources and performance"""
+    
+    def __init__(self, history_minutes: int = 10, sample_interval: int = 5):
+        """Initialize the monitoring service
+        
+        Args:
+            history_minutes: How many minutes of history to keep
+            sample_interval: How many seconds between samples
+        """
+        self.history_minutes = history_minutes
+        self.sample_interval = sample_interval
+        self.max_samples = (history_minutes * 60) // sample_interval
+        
+        # Initialize data structures for metrics
+        self.timestamps = deque(maxlen=self.max_samples)
+        self.cpu_percent = deque(maxlen=self.max_samples)
+        self.memory_percent = deque(maxlen=self.max_samples)
+        self.memory_used = deque(maxlen=self.max_samples)
+        self.memory_available = deque(maxlen=self.max_samples)
+        
+        # CPU temperature history (might not be available on all systems)
+        self.cpu_temp = deque(maxlen=self.max_samples)
+        
+        # Per-core CPU history
+        self.cpu_cores_percent = {}
+        
+        # Track if the monitoring thread is running
+        self.is_running = False
+        self.thread = None
+        
+        # Initialize with current values
+        self.collect_metrics()
+        
+    def collect_metrics(self) -> Dict[str, Any]:
+        """Collect current system metrics
+        
+        Returns:
+            Dictionary of current metrics
+        """
+        metrics = {
+            'timestamp': datetime.now(),
+            'cpu_percent': psutil.cpu_percent(interval=0.1),
+            'memory_percent': psutil.virtual_memory().percent,
+            'memory_used': psutil.virtual_memory().used / (1024**3),  # GB
+            'memory_available': psutil.virtual_memory().available / (1024**3),  # GB
+            'cpu_temp': None,
+            'per_cpu_percent': psutil.cpu_percent(interval=0.1, percpu=True)
+        }
+        
+        # Try to get CPU temperature (platform specific)
+        try:
+            if platform.system() == 'Linux':
+                # Try to get temperature from psutil
+                temps = psutil.sensors_temperatures()
+                for name, entries in temps.items():
+                    if name.startswith(('coretemp', 'k10temp', 'cpu_thermal')):
+                        metrics['cpu_temp'] = entries[0].current
+                        break
+            elif platform.system() == 'Darwin':  # macOS
+                # On macOS, we could use SMC reader but it requires additional dependencies
+                # Leaving as None for now
+                pass
+            elif platform.system() == 'Windows':
+                # Windows might require WMI, leaving as None for simplicity
+                pass
+        except (AttributeError, KeyError, IndexError, NotImplementedError):
+            # Sensors not available
+            pass
+        
+        return metrics
+    
+    def update_history(self, metrics: Dict[str, Any]) -> None:
+        """Update metric history with new values
+        
+        Args:
+            metrics: New metrics to add to history
+        """
+        self.timestamps.append(metrics['timestamp'])
+        self.cpu_percent.append(metrics['cpu_percent'])
+        self.memory_percent.append(metrics['memory_percent'])
+        self.memory_used.append(metrics['memory_used'])
+        self.memory_available.append(metrics['memory_available'])
+        
+        if metrics['cpu_temp'] is not None:
+            self.cpu_temp.append(metrics['cpu_temp'])
+        
+        # Update per-core CPU metrics
+        for i, percent in enumerate(metrics['per_cpu_percent']):
+            if i not in self.cpu_cores_percent:
+                self.cpu_cores_percent[i] = deque(maxlen=self.max_samples)
+            self.cpu_cores_percent[i].append(percent)
+    
+    def start_monitoring(self) -> None:
+        """Start background thread for collecting metrics"""
+        if self.is_running:
+            logger.warning("Monitoring thread already running")
+            return
+            
+        self.is_running = True
+        
+        def _monitor_loop():
+            while self.is_running:
+                try:
+                    metrics = self.collect_metrics()
+                    self.update_history(metrics)
+                    time.sleep(self.sample_interval)
+                except Exception as e:
+                    logger.error(f"Error in monitoring thread: {str(e)}", exc_info=True)
+                    time.sleep(self.sample_interval)
+        
+        self.thread = threading.Thread(target=_monitor_loop, daemon=True)
+        self.thread.start()
+        logger.info("System monitoring thread started")
+    
+    def stop_monitoring(self) -> None:
+        """Stop the monitoring thread"""
+        if not self.is_running:
+            return
+            
+        self.is_running = False
+        if self.thread:
+            self.thread.join(timeout=1.0)
+            logger.info("System monitoring thread stopped")
+    
+    def get_current_metrics(self) -> Dict[str, Any]:
+        """Get current system metrics
+        
+        Returns:
+            Dictionary with current system metrics
+        """
+        return self.collect_metrics()
+    
+    def get_system_info(self) -> Dict[str, Any]:
+        """Get general system information
+        
+        Returns:
+            Dictionary with system details
+        """
+        cpu_info = {
+            'cores_physical': psutil.cpu_count(logical=False),
+            'cores_logical': psutil.cpu_count(logical=True),
+            'current_frequency': None,
+            'architecture': platform.machine(),
+        }
+        
+        # Try to get CPU frequency
+        try:
+            cpu_freq = psutil.cpu_freq()
+            if cpu_freq:
+                cpu_info['current_frequency'] = cpu_freq.current
+        except Exception:
+            pass
+            
+        memory_info = {
+            'total': psutil.virtual_memory().total / (1024**3),  # GB
+            'available': psutil.virtual_memory().available / (1024**3),  # GB
+            'used': psutil.virtual_memory().used / (1024**3),  # GB
+            'percent': psutil.virtual_memory().percent
+        }
+        
+        disk_info = {}
+        for part in psutil.disk_partitions(all=False):
+            if os.name == 'nt' and ('cdrom' in part.opts or part.fstype == ''):
+                # Skip CD-ROM drives on Windows
+                continue
+            try:
+                usage = psutil.disk_usage(part.mountpoint)
+                disk_info[part.mountpoint] = {
+                    'total': usage.total / (1024**3),  # GB
+                    'used': usage.used / (1024**3),  # GB
+                    'free': usage.free / (1024**3),  # GB
+                    'percent': usage.percent
+                }
+            except PermissionError:
+                continue
+        
+        sys_info = {
+            'system': platform.system(),
+            'version': platform.version(),
+            'platform': platform.platform(),
+            'processor': platform.processor(),
+            'hostname': platform.node(),
+            'python_version': platform.python_version(),
+            'uptime': time.time() - psutil.boot_time()
+        }
+        
+        return {
+            'cpu': cpu_info,
+            'memory': memory_info,
+            'disk': disk_info,
+            'system': sys_info,
+        }
+    
+    def generate_cpu_plot(self) -> plt.Figure:
+        """Generate a plot of CPU usage over time
+        
+        Returns:
+            Matplotlib figure with CPU usage plot
+        """
+        fig, ax = plt.subplots(figsize=(10, 5))
+        
+        if not self.timestamps:
+            ax.set_title("No CPU data available yet")
+            return fig
+            
+        x = [t.strftime('%H:%M:%S') for t in self.timestamps]
+        if len(x) > 10:
+            # Show fewer x-axis labels for readability
+            step = len(x) // 10
+            ax.set_xticks(range(0, len(x), step))
+            ax.set_xticklabels([x[i] for i in range(0, len(x), step)])
+        
+        ax.plot(x, list(self.cpu_percent), 'b-', label='CPU Usage %')
+        
+        if self.cpu_temp and len(self.cpu_temp) > 0:
+            # Plot temperature on a secondary y-axis if available
+            ax2 = ax.twinx()
+            ax2.plot(x[:len(self.cpu_temp)], list(self.cpu_temp), 'r-', label='CPU Temp °C')
+            ax2.set_ylabel('Temperature (°C)', color='r')
+            ax2.tick_params(axis='y', colors='r')
+            
+        ax.set_title('CPU Usage Over Time')
+        ax.set_xlabel('Time')
+        ax.set_ylabel('Usage %')
+        ax.grid(True, alpha=0.3)
+        ax.set_ylim(0, 100)
+        
+        # Add legend
+        lines, labels = ax.get_legend_handles_labels()
+        if hasattr(locals(), 'ax2'):
+            lines2, labels2 = ax2.get_legend_handles_labels()
+            ax.legend(lines + lines2, labels + labels2, loc='upper left')
+        else:
+            ax.legend(loc='upper left')
+            
+        plt.tight_layout()
+        return fig
+    
+    def generate_memory_plot(self) -> plt.Figure:
+        """Generate a plot of memory usage over time
+        
+        Returns:
+            Matplotlib figure with memory usage plot
+        """
+        fig, ax = plt.subplots(figsize=(10, 5))
+        
+        if not self.timestamps:
+            ax.set_title("No memory data available yet")
+            return fig
+            
+        x = [t.strftime('%H:%M:%S') for t in self.timestamps]
+        if len(x) > 10:
+            # Show fewer x-axis labels for readability
+            step = len(x) // 10
+            ax.set_xticks(range(0, len(x), step))
+            ax.set_xticklabels([x[i] for i in range(0, len(x), step)])
+        
+        ax.plot(x, list(self.memory_percent), 'g-', label='Memory Usage %')
+        
+        # Add secondary y-axis for absolute memory values
+        ax2 = ax.twinx()
+        ax2.plot(x, list(self.memory_used), 'm--', label='Used (GB)')
+        ax2.plot(x, list(self.memory_available), 'c--', label='Available (GB)')
+        ax2.set_ylabel('Memory (GB)')
+        
+        ax.set_title('Memory Usage Over Time')
+        ax.set_xlabel('Time')
+        ax.set_ylabel('Usage %')
+        ax.grid(True, alpha=0.3)
+        ax.set_ylim(0, 100)
+        
+        # Add legend
+        lines, labels = ax.get_legend_handles_labels()
+        lines2, labels2 = ax2.get_legend_handles_labels()
+        ax.legend(lines + lines2, labels + labels2, loc='upper left')
+            
+        plt.tight_layout()
+        return fig
+    
+    def generate_per_core_plot(self) -> plt.Figure:
+        """Generate a plot of per-core CPU usage
+        
+        Returns:
+            Matplotlib figure with per-core CPU usage
+        """
+        num_cores = len(self.cpu_cores_percent)
+        if num_cores == 0:
+            # No data yet
+            fig, ax = plt.subplots(figsize=(10, 5))
+            ax.set_title("No per-core CPU data available yet")
+            return fig
+            
+        # Determine grid layout based on number of cores
+        if num_cores <= 4:
+            rows, cols = 2, 2
+        elif num_cores <= 6:
+            rows, cols = 2, 3
+        elif num_cores <= 9:
+            rows, cols = 3, 3
+        elif num_cores <= 12:
+            rows, cols = 3, 4
+        else:
+            rows, cols = 4, 4
+            
+        fig, axes = plt.subplots(rows, cols, figsize=(12, 8), sharex=True, sharey=True)
+        axes = axes.flatten()
+        
+        x = [t.strftime('%H:%M:%S') for t in self.timestamps]
+        if len(x) > 5:
+            # Show fewer x-axis labels for readability
+            step = len(x) // 5
+        else:
+            step = 1
+            
+        for i, (core_id, percentages) in enumerate(self.cpu_cores_percent.items()):
+            if i >= len(axes):
+                break
+                
+            ax = axes[i]
+            ax.plot(x[:len(percentages)], list(percentages), 'b-')
+            ax.set_title(f'Core {core_id}')
+            ax.set_ylim(0, 100)
+            ax.grid(True, alpha=0.3)
+            
+            # Add x-axis labels sparingly for readability
+            if i >= len(axes) - cols:  # Only for bottom row
+                ax.set_xticks(range(0, len(x), step))
+                ax.set_xticklabels([x[i] for i in range(0, len(x), step)], rotation=45)
+                
+        # Hide unused subplots
+        for i in range(num_cores, len(axes)):
+            axes[i].set_visible(False)
+            
+        plt.tight_layout()
+        return fig

vms/tabs/__init__.py

@@ -6,6 +6,7 @@ from .import_tab import ImportTab
 from .split_tab import SplitTab
 from .caption_tab import CaptionTab
 from .train_tab import TrainTab
+from .monitor_tab import MonitorTab
 from .manage_tab import ManageTab
 
 __all__ = [
@@ -13,5 +14,6 @@ __all__ = [
     'SplitTab',
     'CaptionTab',
     'TrainTab',
+    'MonitorTab',
     'ManageTab'
 ]

vms/tabs/import_tab/__init__.py

@@ -0,0 +1,10 @@
+"""
+Import tab for Video Model Studio.
+"""
+
+from .upload_tab import UploadTab
+from .youtube_tab import YouTubeTab
+from .hub_tab import HubTab
+from .import_tab import ImportTab
+
+__all__ = ['UploadTab', 'YouTubeTab', 'HubTab', 'ImportTab']

vms/tabs/import_tab/hub_tab.py

@@ -0,0 +1,273 @@
+"""
+Hugging Face Hub tab for Video Model Studio UI.
+Handles browsing, searching, and importing datasets from the Hugging Face Hub.
+"""
+
+import gradio as gr
+import logging
+import asyncio
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Tuple
+
+from ..base_tab import BaseTab
+
+logger = logging.getLogger(__name__)
+
+class HubTab(BaseTab):
+    """Hub tab for importing datasets from Hugging Face Hub"""
+    
+    def __init__(self, app_state):
+        super().__init__(app_state)
+        self.id = "hub_tab"
+        self.title = "Import from Hugging Face"
+    
+    def create(self, parent=None) -> gr.Tab:
+        """Create the Hub tab UI components"""
+        with gr.Tab(self.title, id=self.id) as tab:
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown("## Import from Hub datasets")
+                
+                with gr.Row():
+                    gr.Markdown("Search for datasets with videos or WebDataset archives:")
+                
+                with gr.Row():
+                    self.components["dataset_search"] = gr.Textbox(
+                        label="Search Hugging Face Datasets",
+                        placeholder="Search for video datasets..."
+                    )
+                
+                with gr.Row():
+                    self.components["dataset_search_btn"] = gr.Button("Search Datasets", variant="primary")
+                
+                # Dataset browser results section
+                with gr.Row(visible=False) as dataset_results_row:
+                    self.components["dataset_results_row"] = dataset_results_row
+                    
+                    with gr.Column(scale=3):
+                        self.components["dataset_results"] = gr.Dataframe(
+                            headers=["id", "title", "downloads"],
+                            interactive=False,
+                            wrap=True,
+                            row_count=10,
+                            label="Dataset Results"
+                        )
+                    
+                    with gr.Column(scale=3):
+                        # Dataset info and state
+                        self.components["dataset_info"] = gr.Markdown("Select a dataset to see details")
+                        self.components["dataset_id"] = gr.State(value=None)
+                        self.components["file_type"] = gr.State(value=None)
+                        
+                        # Files section that appears when a dataset is selected
+                        with gr.Column(visible=False) as files_section:
+                            self.components["files_section"] = files_section
+                            
+                            gr.Markdown("## Files:")
+                            
+                            # Video files row (appears if videos are present)
+                            with gr.Row(visible=False) as video_files_row:
+                                self.components["video_files_row"] = video_files_row
+                                
+                                with gr.Column(scale=4):
+                                    self.components["video_count_text"] = gr.Markdown("Contains 0 video files")
+                                
+                                with gr.Column(scale=1):
+                                    self.components["download_videos_btn"] = gr.Button("Download", variant="primary")
+                            
+                            # WebDataset files row (appears if tar files are present)
+                            with gr.Row(visible=False) as webdataset_files_row:
+                                self.components["webdataset_files_row"] = webdataset_files_row
+                                
+                                with gr.Column(scale=4):
+                                    self.components["webdataset_count_text"] = gr.Markdown("Contains 0 WebDataset (.tar) files")
+                                
+                                with gr.Column(scale=1):
+                                    self.components["download_webdataset_btn"] = gr.Button("Download", variant="primary")
+                        
+                        # Status and loading indicators
+                        self.components["dataset_loading"] = gr.Markdown(visible=False)
+            
+            return tab
+    
+    def connect_events(self) -> None:
+        """Connect event handlers to UI components"""
+        # Dataset search event
+        self.components["dataset_search_btn"].click(
+            fn=self.search_datasets,
+            inputs=[self.components["dataset_search"]],
+            outputs=[
+                self.components["dataset_results"],
+                self.components["dataset_results_row"]
+            ]
+        )
+        
+        # Dataset selection event - FIX HERE
+        self.components["dataset_results"].select(
+            fn=self.display_dataset_info,
+            outputs=[
+                self.components["dataset_info"],
+                self.components["dataset_id"],
+                self.components["files_section"],
+                self.components["video_files_row"],
+                self.components["video_count_text"],
+                self.components["webdataset_files_row"],
+                self.components["webdataset_count_text"]
+            ]
+        )
+        
+        # Download videos button
+        self.components["download_videos_btn"].click(
+            fn=self.set_file_type_and_return,
+            outputs=[self.components["file_type"]]
+        ).then(
+            fn=self.download_file_group,
+            inputs=[
+                self.components["dataset_id"],
+                self.components["enable_automatic_video_split"],
+                self.components["file_type"]
+            ],
+            outputs=[
+                self.components["dataset_loading"],
+                self.components["import_status"]
+            ]
+        ).success(
+            fn=self.app.tabs["import_tab"].on_import_success,
+            inputs=[
+                self.components["enable_automatic_video_split"],
+                self.components["enable_automatic_content_captioning"],
+                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
+            ],
+            outputs=[
+                self.app.tabs_component,
+                self.app.tabs["split_tab"].components["video_list"],
+                self.app.tabs["split_tab"].components["detect_status"]
+            ]
+        )
+        
+        # Download WebDataset button
+        self.components["download_webdataset_btn"].click(
+            fn=self.set_file_type_and_return_webdataset,
+            outputs=[self.components["file_type"]]
+        ).then(
+            fn=self.download_file_group,
+            inputs=[
+                self.components["dataset_id"],
+                self.components["enable_automatic_video_split"],
+                self.components["file_type"]
+            ],
+            outputs=[
+                self.components["dataset_loading"],
+                self.components["import_status"]
+            ]
+        ).success(
+            fn=self.app.tabs["import_tab"].on_import_success,
+            inputs=[
+                self.components["enable_automatic_video_split"],
+                self.components["enable_automatic_content_captioning"],
+                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
+            ],
+            outputs=[
+                self.app.tabs_component,
+                self.app.tabs["split_tab"].components["video_list"],
+                self.app.tabs["split_tab"].components["detect_status"]
+            ]
+        )
+
+    def set_file_type_and_return(self):
+        """Set file type to video and return it"""
+        return "video"
+    
+    def set_file_type_and_return_webdataset(self):
+        """Set file type to webdataset and return it"""
+        return "webdataset"
+    
+    def search_datasets(self, query: str):
+        """Search datasets on the Hub matching the query"""
+        try:
+            logger.info(f"Searching for datasets with query: '{query}'")
+            results = self.app.importer.search_datasets(query)
+            return results, gr.update(visible=True)
+        except Exception as e:
+            logger.error(f"Error searching datasets: {str(e)}", exc_info=True)
+            return [[f"Error: {str(e)}", "", ""]], gr.update(visible=True)
+    
+    def display_dataset_info(self, evt: gr.SelectData):
+        """Display detailed information about the selected dataset"""
+        try:
+            if not evt or not evt.value:
+                logger.warning("No dataset selected in display_dataset_info")
+                return (
+                    "No dataset selected",  # dataset_info
+                    None,                   # dataset_id
+                    gr.update(visible=False), # files_section
+                    gr.update(visible=False), # video_files_row
+                    "",                     # video_count_text
+                    gr.update(visible=False), # webdataset_files_row
+                    ""                      # webdataset_count_text
+                )
+            
+            dataset_id = evt.value[0] if isinstance(evt.value, list) else evt.value
+            logger.info(f"Getting dataset info for: {dataset_id}")
+            
+            # Use the importer service to get dataset info
+            info_text, file_counts, _ = self.app.importer.get_dataset_info(dataset_id)
+            
+            # Get counts of each file type
+            video_count = file_counts.get("video", 0)
+            webdataset_count = file_counts.get("webdataset", 0)
+            
+            # Return all the required outputs individually
+            return (
+                info_text,                                # dataset_info
+                dataset_id,                              # dataset_id
+                gr.update(visible=True),                 # files_section
+                gr.update(visible=video_count > 0),      # video_files_row
+                f"Contains {video_count} video file{'s' if video_count != 1 else ''}", # video_count_text
+                gr.update(visible=webdataset_count > 0), # webdataset_files_row
+                f"Contains {webdataset_count} WebDataset (.tar) file{'s' if webdataset_count != 1 else ''}" # webdataset_count_text
+            )
+        except Exception as e:
+            logger.error(f"Error displaying dataset info: {str(e)}", exc_info=True)
+            return (
+                f"Error loading dataset information: {str(e)}", # dataset_info
+                None,                                          # dataset_id
+                gr.update(visible=False),                      # files_section
+                gr.update(visible=False),                      # video_files_row
+                "",                                            # video_count_text
+                gr.update(visible=False),                      # webdataset_files_row
+                ""                                             # webdataset_count_text
+            )
+    
+    def download_file_group(self, dataset_id: str, enable_splitting: bool, file_type: str) -> Tuple[gr.update, str]:
+        """Handle download of a group of files (videos or WebDatasets)"""
+        try:
+            if not dataset_id:
+                return gr.update(visible=False), "No dataset selected"
+            
+            logger.info(f"Starting download of {file_type} files from dataset: {dataset_id}")
+            
+            # Show loading indicator
+            loading_msg = gr.update(
+                value=f"## Downloading {file_type} files from {dataset_id}\n\nThis may take some time...",
+                visible=True
+            )
+            status_msg = f"Downloading {file_type} files from {dataset_id}..."
+            
+            # Use the async version in a non-blocking way
+            asyncio.create_task(self._download_file_group_bg(dataset_id, file_type, enable_splitting))
+            
+            return loading_msg, status_msg
+            
+        except Exception as e:
+            error_msg = f"Error initiating download: {str(e)}"
+            logger.error(error_msg, exc_info=True)
+            return gr.update(visible=False), error_msg
+    
+    async def _download_file_group_bg(self, dataset_id: str, file_type: str, enable_splitting: bool):
+        """Background task for group file download"""
+        try:
+            # This will execute in the background
+            await self.app.importer.download_file_group(dataset_id, file_type, enable_splitting)
+        except Exception as e:
+            logger.error(f"Error in background file group download: {str(e)}", exc_info=True)

vms/tabs/{import_tab.py → import_tab/import_tab.py}

@@ -1,33 +1,42 @@
 """
-Import tab for Video Model Studio UI
+Parent import tab for Video Model Studio UI that contains sub-tabs
 """
 
 import gradio as gr
 import logging
 import asyncio
 from pathlib import Path
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Tuple
 
-from .base_tab import BaseTab
-from ..config import (
-    VIDEOS_TO_SPLIT_PATH, DEFAULT_PROMPT_PREFIX, DEFAULT_CAPTIONING_BOT_INSTRUCTIONS
+from ..base_tab import BaseTab
+from .upload_tab import UploadTab
+from .youtube_tab import YouTubeTab
+from .hub_tab import HubTab
+
+from vms.config import (
+    VIDEOS_TO_SPLIT_PATH, DEFAULT_PROMPT_PREFIX, DEFAULT_CAPTIONING_BOT_INSTRUCTIONS,
+    STAGING_PATH
 )
 
 logger = logging.getLogger(__name__)
 
 class ImportTab(BaseTab):
-    """Import tab for uploading videos and images"""
+    """Import tab for uploading videos and images, and browsing datasets"""
     
     def __init__(self, app_state):
         super().__init__(app_state)
         self.id = "import_tab"
         self.title = "1️⃣  Import"
+        # Initialize sub-tabs
+        self.upload_tab = UploadTab(app_state)
+        self.youtube_tab = YouTubeTab(app_state)
+        self.hub_tab = HubTab(app_state)
     
     def create(self, parent=None) -> gr.TabItem:
-        """Create the Import tab UI components"""
+        """Create the Import tab UI components with three sub-tabs"""
         with gr.TabItem(self.title, id=self.id) as tab:
             with gr.Row():
-                gr.Markdown("## Automatic splitting and captioning")
+                gr.Markdown("## Import settings")
             
             with gr.Row():
                 self.components["enable_automatic_video_split"] = gr.Checkbox(
@@ -42,38 +51,19 @@ class ImportTab(BaseTab):
                     value=False,
                     visible=True,
                 )
+            
+            # Create tabs for different import methods
+            with gr.Tabs() as import_tabs:
+                # Create each sub-tab
+                self.upload_tab.create(import_tabs)
+                self.youtube_tab.create(import_tabs)
+                self.hub_tab.create(import_tabs)
                 
-            with gr.Row():
-                with gr.Column(scale=3):
-                    with gr.Row():
-                        with gr.Column():
-                            gr.Markdown("## Import files")
-                            gr.Markdown("You can upload either:")
-                            gr.Markdown("- A single MP4 video file")
-                            gr.Markdown("- A ZIP archive containing multiple videos/images and optional caption files")
-                            gr.Markdown("- A WebDataset shard (.tar file)")
-                            gr.Markdown("- A ZIP archive containing WebDataset shards (.tar files)")
-                                
-                    with gr.Row():
-                        self.components["files"] = gr.Files(
-                            label="Upload Images, Videos, ZIP or WebDataset",
-                            file_types=[".jpg", ".jpeg", ".png", ".webp", ".webp", ".avif", ".heic", ".mp4", ".zip", ".tar"],
-                            type="filepath"
-                        )
-       
-                with gr.Column(scale=3):
-                    with gr.Row():
-                        with gr.Column():
-                            gr.Markdown("## Import a YouTube video")
-                            gr.Markdown("You can also use a YouTube video as reference, by pasting its URL here:")
-
-                    with gr.Row():
-                        self.components["youtube_url"] = gr.Textbox(
-                            label="Import YouTube Video",
-                            placeholder="https://www.youtube.com/watch?v=..."
-                        )
-                    with gr.Row():
-                        self.components["youtube_download_btn"] = gr.Button("Download YouTube Video", variant="secondary")
+                # Store references to sub-tabs
+                self.components["upload_tab"] = self.upload_tab
+                self.components["youtube_tab"] = self.youtube_tab
+                self.components["hub_tab"] = self.hub_tab
+            
             with gr.Row():
                 self.components["import_status"] = gr.Textbox(label="Status", interactive=False)
 
@@ -81,47 +71,17 @@ class ImportTab(BaseTab):
     
     def connect_events(self) -> None:
         """Connect event handlers to UI components"""
-        # File upload event
-        self.components["files"].upload(
-            fn=lambda x: self.app.importer.process_uploaded_files(x),
-            inputs=[self.components["files"]],
-            outputs=[self.components["import_status"]]
-        ).success(
-            fn=self.update_titles_after_import,
-            inputs=[
-                self.components["enable_automatic_video_split"], 
-                self.components["enable_automatic_content_captioning"], 
-                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
-            ],
-            outputs=[
-                self.app.tabs_component,  # Main tabs component 
-                self.app.tabs["split_tab"].components["video_list"],
-                self.app.tabs["split_tab"].components["detect_status"],
-                self.app.tabs["split_tab"].components["split_title"],
-                self.app.tabs["caption_tab"].components["caption_title"],
-                self.app.tabs["train_tab"].components["train_title"]
-            ]
-        )
-        
-        # YouTube download event
-        self.components["youtube_download_btn"].click(
-            fn=self.app.importer.download_youtube_video,
-            inputs=[self.components["youtube_url"]],
-            outputs=[self.components["import_status"]]
-        ).success(
-            fn=self.on_import_success,
-            inputs=[
-                self.components["enable_automatic_video_split"],
-                self.components["enable_automatic_content_captioning"],
-                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
-            ],
-            outputs=[
-                self.app.tabs_component,
-                self.app.tabs["split_tab"].components["video_list"],
-                self.app.tabs["split_tab"].components["detect_status"]
-            ]
-        )
-        
+        # Set shared components from parent tab to sub-tabs first
+        for subtab in [self.upload_tab, self.youtube_tab, self.hub_tab]:
+            subtab.components["import_status"] = self.components["import_status"]
+            subtab.components["enable_automatic_video_split"] = self.components["enable_automatic_video_split"]
+            subtab.components["enable_automatic_content_captioning"] = self.components["enable_automatic_content_captioning"]
+            
+        # Then connect events for each sub-tab
+        self.upload_tab.connect_events()
+        self.youtube_tab.connect_events()
+        self.hub_tab.connect_events()
+    
     async def on_import_success(self, enable_splitting, enable_automatic_content_captioning, prompt_prefix):
         """Handle successful import of files"""
         videos = self.app.tabs["split_tab"].list_unprocessed_videos()

vms/tabs/import_tab/upload_tab.py

@@ -0,0 +1,74 @@
+"""
+Upload tab for Video Model Studio UI.
+Handles manual file uploads for videos, images, and archives.
+"""
+
+import gradio as gr
+import logging
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from ..base_tab import BaseTab
+
+logger = logging.getLogger(__name__)
+
+class UploadTab(BaseTab):
+    """Upload tab for manual file uploads"""
+    
+    def __init__(self, app_state):
+        super().__init__(app_state)
+        self.id = "upload_tab"
+        self.title = "Manual Upload"
+    
+    def create(self, parent=None) -> gr.Tab:
+        """Create the Upload tab UI components"""
+        with gr.Tab(self.title, id=self.id) as tab:
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown("## Manual upload of video files")
+                
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            gr.Markdown("You can upload either:")
+                        with gr.Row():
+                            gr.Markdown("- A single MP4 video file")
+                        with gr.Row():
+                            gr.Markdown("- A ZIP archive containing multiple videos/images and optional caption files")
+                        with gr.Row():
+                            gr.Markdown("- A WebDataset shard (.tar file)")
+                        with gr.Row():
+                            gr.Markdown("- A ZIP archive containing WebDataset shards (.tar files)")
+                    with gr.Column():
+                        with gr.Row():
+                            self.components["files"] = gr.Files(
+                                label="Upload Images, Videos, ZIP or WebDataset",
+                                file_types=[".jpg", ".jpeg", ".png", ".webp", ".webp", ".avif", ".heic", ".mp4", ".zip", ".tar"],
+                                type="filepath"
+                            )
+            
+            return tab
+    
+    def connect_events(self) -> None:
+        """Connect event handlers to UI components"""
+        # File upload event
+        self.components["files"].upload(
+            fn=lambda x: self.app.importer.process_uploaded_files(x),
+            inputs=[self.components["files"]],
+            outputs=[self.components["import_status"]]  # This comes from parent tab
+        ).success(
+            fn=self.app.tabs["import_tab"].update_titles_after_import,
+            inputs=[
+                self.components["enable_automatic_video_split"], 
+                self.components["enable_automatic_content_captioning"], 
+                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
+            ],
+            outputs=[
+                self.app.tabs_component,  # Main tabs component 
+                self.app.tabs["split_tab"].components["video_list"],
+                self.app.tabs["split_tab"].components["detect_status"],
+                self.app.tabs["split_tab"].components["split_title"],
+                self.app.tabs["caption_tab"].components["caption_title"],
+                self.app.tabs["train_tab"].components["train_title"]
+            ]
+        )

vms/tabs/import_tab/youtube_tab.py

@@ -0,0 +1,67 @@
+"""
+YouTube tab for Video Model Studio UI.
+Handles downloading videos from YouTube URLs.
+"""
+
+import gradio as gr
+import logging
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from ..base_tab import BaseTab
+
+logger = logging.getLogger(__name__)
+
+class YouTubeTab(BaseTab):
+    """YouTube tab for downloading videos from YouTube"""
+    
+    def __init__(self, app_state):
+        super().__init__(app_state)
+        self.id = "youtube_tab"
+        self.title = "Download from YouTube"
+    
+    def create(self, parent=None) -> gr.Tab:
+        """Create the YouTube tab UI components"""
+        with gr.Tab(self.title, id=self.id) as tab:
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown("## Import a YouTube video")
+                
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            gr.Markdown("You can use a YouTube video as reference, by pasting its URL here:")
+                        with gr.Row():
+                            gr.Markdown("Please be aware of the [know limitations](https://stackoverflow.com/questions/78160027/how-to-solve-http-error-400-bad-request-in-pytube) and [issues](https://stackoverflow.com/questions/79226520/pytube-throws-http-error-403-forbidden-since-a-few-days)")
+
+                    with gr.Column():
+                        self.components["youtube_url"] = gr.Textbox(
+                            label="Import YouTube Video",
+                            placeholder="https://www.youtube.com/watch?v=..."
+                        )
+                
+                with gr.Row():
+                    self.components["youtube_download_btn"] = gr.Button("Download YouTube Video", variant="primary")
+            
+            return tab
+    
+    def connect_events(self) -> None:
+        """Connect event handlers to UI components"""
+        # YouTube download event
+        self.components["youtube_download_btn"].click(
+            fn=self.app.importer.download_youtube_video,
+            inputs=[self.components["youtube_url"]],
+            outputs=[self.components["import_status"]]  # This comes from parent tab
+        ).success(
+            fn=self.app.tabs["import_tab"].on_import_success,
+            inputs=[
+                self.components["enable_automatic_video_split"],
+                self.components["enable_automatic_content_captioning"],
+                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
+            ],
+            outputs=[
+                self.app.tabs_component,
+                self.app.tabs["split_tab"].components["video_list"],
+                self.app.tabs["split_tab"].components["detect_status"]
+            ]
+        )

vms/tabs/manage_tab.py

@@ -23,7 +23,7 @@ class ManageTab(BaseTab):
     def __init__(self, app_state):
         super().__init__(app_state)
         self.id = "manage_tab"
-        self.title = "5️⃣  Manage"
+        self.title = "6️⃣  Manage"
     
     def create(self, parent=None) -> gr.TabItem:
         """Create the Manage tab UI components"""

vms/tabs/monitor_tab.py

@@ -0,0 +1,407 @@
+"""
+System monitoring tab for Video Model Studio UI.
+Displays system metrics like CPU, memory usage, and temperatures.
+"""
+
+import gradio as gr
+import time
+import logging
+from pathlib import Path
+import os
+import psutil
+from typing import Dict, Any, List, Optional, Tuple
+from datetime import datetime, timedelta
+
+from .base_tab import BaseTab
+from ..config import STORAGE_PATH
+
+logger = logging.getLogger(__name__)
+
+def get_folder_size(path):
+    """Calculate the total size of a folder in bytes"""
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(path):
+        for filename in filenames:
+            file_path = os.path.join(dirpath, filename)
+            if not os.path.islink(file_path):  # Skip symlinks
+                total_size += os.path.getsize(file_path)
+    return total_size
+
+def human_readable_size(size_bytes):
+    """Convert a size in bytes to a human-readable string"""
+    if size_bytes == 0:
+        return "0 B"
+    size_names = ("B", "KB", "MB", "GB", "TB", "PB")
+    i = 0
+    while size_bytes >= 1024 and i < len(size_names) - 1:
+        size_bytes /= 1024
+        i += 1
+    return f"{size_bytes:.2f} {size_names[i]}"
+
+class MonitorTab(BaseTab):
+    """Monitor tab for system resource monitoring"""
+    
+    def __init__(self, app_state):
+        super().__init__(app_state)
+        self.id = "monitor_tab"
+        self.title = "4️⃣  Monitor"
+        self.refresh_interval = 2  # Changed from 5 to 2 seconds
+    
+    def create(self, parent=None) -> gr.TabItem:
+        """Create the Monitor tab UI components"""
+        with gr.TabItem(self.title, id=self.id) as tab:
+            with gr.Row():
+                gr.Markdown("## System Monitoring")
+  
+            # Current metrics
+            with gr.Row():
+                with gr.Column(scale=1):
+                    self.components["current_metrics"] = gr.Markdown("Loading current metrics...")
+            
+            # CPU and Memory charts in tabs
+            with gr.Tabs() as metrics_tabs:
+                with gr.Tab(label="CPU Usage") as cpu_tab:
+                    self.components["cpu_plot"] = gr.Plot()
+                
+                with gr.Tab(label="Memory Usage") as memory_tab:
+                    self.components["memory_plot"] = gr.Plot()
+                    
+                with gr.Tab(label="Per-Core CPU") as per_core_tab:
+                    self.components["per_core_plot"] = gr.Plot()
+
+            # System information summary in columns
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.Markdown("### System Information")
+                    self.components["system_info"] = gr.Markdown("Loading system information...")
+                
+                with gr.Column(scale=1):
+                    gr.Markdown("### CPU Information")
+                    self.components["cpu_info"] = gr.Markdown("Loading CPU information...")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.Markdown("### Memory Information")
+                    self.components["memory_info"] = gr.Markdown("Loading memory information...")
+                
+                with gr.Column(scale=1):
+                    gr.Markdown("### Storage Information")
+                    self.components["storage_info"] = gr.Markdown("Loading storage information...")
+          
+            # Toggle for enabling/disabling auto-refresh
+            with gr.Row():
+                self.components["auto_refresh"] = gr.Checkbox(
+                    label=f"Auto refresh (every {self.refresh_interval} seconds)",
+                    value=True,
+                    info="Automatically refresh system metrics"
+                )
+                self.components["refresh_btn"] = gr.Button("Refresh Now")
+            
+            # Timer for auto-refresh
+            self.components["refresh_timer"] = gr.Timer(
+                value=self.refresh_interval
+            )
+            
+        return tab
+    
+    def connect_events(self) -> None:
+        """Connect event handlers to UI components"""
+        # Manual refresh button
+        self.components["refresh_btn"].click(
+            fn=self.refresh_all,
+            outputs=[
+                self.components["system_info"],
+                self.components["cpu_info"],
+                self.components["memory_info"],
+                self.components["storage_info"],
+                self.components["current_metrics"],
+                self.components["cpu_plot"],
+                self.components["memory_plot"],
+                self.components["per_core_plot"]
+            ]
+        )
+        
+        # Auto-refresh timer
+        self.components["refresh_timer"].tick(
+            fn=self.conditional_refresh,
+            inputs=[self.components["auto_refresh"]],
+            outputs=[
+                self.components["system_info"],
+                self.components["cpu_info"],
+                self.components["memory_info"],
+                self.components["storage_info"],
+                self.components["current_metrics"],
+                self.components["cpu_plot"],
+                self.components["memory_plot"],
+                self.components["per_core_plot"]
+            ]
+        )
+    
+    def on_enter(self):
+        """Called when the tab is selected"""
+        # Start monitoring service if not already running
+        if not self.app.monitor.is_running:
+            self.app.monitor.start_monitoring()
+        
+        # Trigger initial refresh
+        return self.refresh_all()
+    
+    def conditional_refresh(self, auto_refresh: bool) -> Tuple:
+        """Only refresh if auto-refresh is enabled
+        
+        Args:
+            auto_refresh: Whether auto-refresh is enabled
+            
+        Returns:
+            Updated components or unchanged components
+        """
+        if auto_refresh:
+            return self.refresh_all()
+        
+        # Return current values unchanged if auto-refresh is disabled
+        return (
+            self.components["system_info"].value,
+            self.components["cpu_info"].value,
+            self.components["memory_info"].value,
+            self.components["storage_info"].value,
+            self.components["current_metrics"].value,
+            self.components["cpu_plot"].value,
+            self.components["memory_plot"].value, 
+            self.components["per_core_plot"].value
+        )
+    
+    def refresh_all(self) -> Tuple:
+        """Refresh all monitoring components
+        
+        Returns:
+            Updated values for all components
+        """
+        try:
+            # Get system info
+            system_info = self.app.monitor.get_system_info()
+            
+            # Split system info into separate components
+            system_info_html = self.format_system_info(system_info)
+            cpu_info_html = self.format_cpu_info(system_info)
+            memory_info_html = self.format_memory_info(system_info)
+            storage_info_html = self.format_storage_info()
+            
+            # Get current metrics
+            current_metrics = self.app.monitor.get_current_metrics()
+            metrics_html = self.format_current_metrics(current_metrics)
+            
+            # Generate plots
+            cpu_plot = self.app.monitor.generate_cpu_plot()
+            memory_plot = self.app.monitor.generate_memory_plot()
+            per_core_plot = self.app.monitor.generate_per_core_plot()
+            
+            return (
+                system_info_html, 
+                cpu_info_html, 
+                memory_info_html, 
+                storage_info_html, 
+                metrics_html, 
+                cpu_plot, 
+                memory_plot, 
+                per_core_plot
+            )
+            
+        except Exception as e:
+            logger.error(f"Error refreshing monitoring data: {str(e)}", exc_info=True)
+            error_msg = f"Error retrieving data: {str(e)}"
+            return (
+                error_msg,
+                error_msg,
+                error_msg,
+                error_msg,
+                error_msg,
+                None, None, None
+            )
+    
+    def format_system_info(self, system_info: Dict[str, Any]) -> str:
+        """Format system information as HTML
+        
+        Args:
+            system_info: System information dictionary
+            
+        Returns:
+            Formatted HTML string
+        """
+        sys = system_info['system']
+        uptime_str = self.format_uptime(sys['uptime'])
+        
+        html = f"""
+**System:** {sys['system']} ({sys['platform']})  
+**Hostname:** {sys['hostname']}  
+**Uptime:** {uptime_str}  
+**Python Version:** {sys['python_version']}
+        """
+        return html
+    
+    def format_cpu_info(self, system_info: Dict[str, Any]) -> str:
+        """Format CPU information as HTML
+        
+        Args:
+            system_info: System information dictionary
+            
+        Returns:
+            Formatted HTML string
+        """
+        cpu = system_info['cpu']
+        sys = system_info['system']
+        
+        # Format CPU frequency
+        cpu_freq = "N/A"
+        if cpu['current_frequency']:
+            cpu_freq = f"{cpu['current_frequency'] / 1000:.2f} GHz"
+            
+        html = f"""
+**Processor:** {sys['processor'] or cpu['architecture']}  
+**Physical Cores:** {cpu['cores_physical']}  
+**Logical Cores:** {cpu['cores_logical']}  
+**Current Frequency:** {cpu_freq}
+        """
+        return html
+    
+    def format_memory_info(self, system_info: Dict[str, Any]) -> str:
+        """Format memory information as HTML
+        
+        Args:
+            system_info: System information dictionary
+            
+        Returns:
+            Formatted HTML string
+        """
+        memory = system_info['memory']
+        
+        html = f"""
+**Total Memory:** {memory['total']:.2f} GB  
+**Available Memory:** {memory['available']:.2f} GB  
+**Used Memory:** {memory['used']:.2f} GB  
+**Usage:** {memory['percent']}%
+        """
+        return html
+    
+    def format_storage_info(self) -> str:
+        """Format storage information as HTML, focused on STORAGE_PATH
+        
+        Returns:
+            Formatted HTML string
+        """
+        try:
+            # Get total size of STORAGE_PATH
+            total_size = get_folder_size(STORAGE_PATH)
+            total_size_readable = human_readable_size(total_size)
+            
+            html = f"**Total Storage Used:** {total_size_readable}\n\n"
+            
+            # Get size of each subfolder
+            html += "**Subfolder Sizes:**\n\n"
+            
+            for subfolder in sorted(STORAGE_PATH.iterdir()):
+                if subfolder.is_dir():
+                    folder_size = get_folder_size(subfolder)
+                    folder_size_readable = human_readable_size(folder_size)
+                    percentage = (folder_size / total_size * 100) if total_size > 0 else 0
+                    
+                    folder_name = subfolder.name
+                    html += f"* **{folder_name}**: {folder_size_readable} ({percentage:.1f}%)\n"
+            
+            return html
+            
+        except Exception as e:
+            logger.error(f"Error getting folder sizes: {str(e)}", exc_info=True)
+            return f"Error getting folder sizes: {str(e)}"
+    
+    def format_current_metrics(self, metrics: Dict[str, Any]) -> str:
+        """Format current metrics as HTML
+        
+        Args:
+            metrics: Current metrics dictionary
+            
+        Returns:
+            Formatted HTML string
+        """
+        timestamp = metrics['timestamp'].strftime('%Y-%m-%d %H:%M:%S')
+        
+        # Style for CPU usage
+        cpu_style = "color: green;"
+        if metrics['cpu_percent'] > 90:
+            cpu_style = "color: red; font-weight: bold;"
+        elif metrics['cpu_percent'] > 70:
+            cpu_style = "color: orange;"
+            
+        # Style for memory usage
+        mem_style = "color: green;"
+        if metrics['memory_percent'] > 90:
+            mem_style = "color: red; font-weight: bold;"
+        elif metrics['memory_percent'] > 70:
+            mem_style = "color: orange;"
+            
+        # Temperature info
+        temp_html = ""
+        if metrics['cpu_temp'] is not None:
+            temp_style = "color: green;"
+            if metrics['cpu_temp'] > 80:
+                temp_style = "color: red; font-weight: bold;"
+            elif metrics['cpu_temp'] > 70:
+                temp_style = "color: orange;"
+                
+            temp_html = f"""
+**CPU Temperature:** <span style="{temp_style}">{metrics['cpu_temp']:.1f}°C</span>
+"""
+            
+        html = f"""
+**CPU Usage:** <span style="{cpu_style}">{metrics['cpu_percent']:.1f}%</span>
+**Memory Usage:** <span style="{mem_style}">{metrics['memory_percent']:.1f}% ({metrics['memory_used']:.2f}/{metrics['memory_available']:.2f} GB)</span>  
+{temp_html}
+        """
+        
+        # Add per-CPU core info
+        html += "\n"
+        
+        per_cpu = metrics['per_cpu_percent']
+        cols = 4  # 4 cores per row
+        
+        # Create a grid layout for cores
+        for i in range(0, len(per_cpu), cols):
+            row_cores = per_cpu[i:i+cols]
+            row_html = ""
+            
+            for j, usage in enumerate(row_cores):
+                core_id = i + j
+                core_style = "color: green;"
+                if usage > 90:
+                    core_style = "color: red; font-weight: bold;"
+                elif usage > 70:
+                    core_style = "color: orange;"
+                    
+                row_html += f"**Core {core_id}:** <span style='{core_style}'>{usage:.1f}%</span>&nbsp;&nbsp;&nbsp;"
+                
+            html += row_html + "\n"
+            
+        return html
+    
+    def format_uptime(self, seconds: float) -> str:
+        """Format uptime in seconds to a human-readable string
+        
+        Args:
+            seconds: Uptime in seconds
+            
+        Returns:
+            Formatted uptime string
+        """
+        days = int(seconds // 86400)
+        seconds %= 86400
+        hours = int(seconds // 3600)
+        seconds %= 3600
+        minutes = int(seconds // 60)
+        
+        parts = []
+        if days > 0:
+            parts.append(f"{days} day{'s' if days != 1 else ''}")
+        if hours > 0 or days > 0:
+            parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
+        parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
+        
+        return ", ".join(parts)

vms/ui/video_trainer_ui.py

@@ -5,7 +5,7 @@ import logging
 import asyncio
 from typing import Any, Optional, Dict, List, Union, Tuple
 
-from ..services import TrainingService, CaptioningService, SplittingService, ImportService
+from ..services import TrainingService, CaptioningService, SplittingService, ImportService, MonitoringService
 from ..config import (
     STORAGE_PATH, VIDEOS_TO_SPLIT_PATH, STAGING_PATH, OUTPUT_PATH,
     TRAINING_PATH, LOG_FILE_PATH, TRAINING_PRESETS, TRAINING_VIDEOS_PATH, MODEL_PATH, OUTPUT_PATH,
@@ -28,7 +28,7 @@ from ..utils import (
     format_media_title,
     TrainingLogParser
 )
-from ..tabs import ImportTab, SplitTab, CaptionTab, TrainTab, ManageTab
+from ..tabs import ImportTab, SplitTab, CaptionTab, TrainTab, MonitorTab, ManageTab
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -44,7 +44,11 @@ class VideoTrainerUI:
         self.splitter = SplittingService()
         self.importer = ImportService()
         self.captioner = CaptioningService()
-        
+        self.monitor = MonitoringService()
+
+        # Start the monitoring service on app creation
+        self.monitor.start_monitoring()
+    
         # Recovery status from any interrupted training
         recovery_result = self.trainer.recover_interrupted_training()
         # Add null check for recovery_result
@@ -81,6 +85,7 @@ class VideoTrainerUI:
                 self.tabs["split_tab"] = SplitTab(self)
                 self.tabs["caption_tab"] = CaptionTab(self)
                 self.tabs["train_tab"] = TrainTab(self)
+                self.tabs["monitor_tab"] = MonitorTab(self)
                 self.tabs["manage_tab"] = ManageTab(self)
                 
                 # Create tab UI components