{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Video Augmentation using META SAM-2 Model with YOLO model and Stability AI","metadata":{}},{"cell_type":"markdown","source":"### Importing Images with Annoted text file for Yolov8n Model Training","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### upload your image directory with .txt annoted file in the format required by yolo model for training, with video on which model has to predict.\n\n### incase if wants to use pre_trained YOLO model, jump to section of pretrained model., or incase want to manually put coordinates on a frame jump to section of video segmenting.","metadata":{}},{"cell_type":"markdown","source":"### Installing Required Libraries","metadata":{}},{"cell_type":"code","source":"!pip install ultralytics opencv-python\n!pip install -U ipywidgets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Yolov8n Model training ","metadata":{}},{"cell_type":"markdown","source":"## Yaml file creation and model training\n","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Load YOLOv8 model configuration (e.g., YOLOv8 nano model)\nmodel = YOLO('yolov8n.yaml')\n\n# Create a dataset.yaml file for YOLOv8 training\ndataset_yaml_content = \"\"\"\ntrain: \"/kaggle/input/yolov-train-data/Bottle\"\nval: \"/kaggle/input/yolov-train-data/Bottle\"\nnc: 1  # Number of classes (1 in this case)\nnames: ['bottle']\n\"\"\"\n\n# Save the dataset.yaml file\nwith open('dataset.yaml', 'w') as f:\n    f.write(dataset_yaml_content)\n\n    \n\n# Train the model with the specified dataset and parameters\nmodel.train(\n    data='dataset.yaml',  # Path to the dataset.yaml file\n    epochs=100,           # Increase epochs for better results with small datasets\n    imgsz=1024,           # Use the resized image dimensions\n    batch=1,              # Set batch size to 4 due to limited data\n    patience=50,          # Early stopping if no improvement\n    lr0=0.0001,            # Start with a lower learning rate\n    augment=True,  # Enable data augmentation\n#     weights='yolov8n.pt'  # Start training with pre-trained weights (optional)\n)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Note: You may have to enter wandb.ai api if using Kaggle","metadata":{}},{"cell_type":"markdown","source":"## prediction on an Image","metadata":{}},{"cell_type":"code","source":"# Load a test image\nimg = cv2.imread('/kaggle/input/yolov-train-data/Bottle/IMG202408142240012.jpg')\n\n# Predict\nresults = model.predict(img)\n\n# Alternatively, you can use matplotlib to display the results\nplt.imshow(results[0].plot())  # `plot` returns an image with bounding boxes drawn\nplt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting on Video & detecting the First Frame, and its center coordinates","metadata":{}},{"cell_type":"code","source":"# Process the video\nvideo_path = '/kaggle/input/yolov-train-data/VID202408142242002.mp4'\ncap = cv2.VideoCapture(video_path)\n\nx_center=0\ny_center=0\nframe_number = 0\nobject_detected = False\n\nwhile cap.isOpened():\n    ret, frame = cap.read()\n    if not ret:\n        break\n\n    frame_number += 1\n\n    # Run YOLOv8 detection\n    results = model(frame)\n\n    for r in results:\n        if r.boxes:  # Check if any object is detected\n            for box in r.boxes:\n                # Get the bounding box coordinates\n                x1, y1, x2, y2 = box.xyxy[0].cpu().numpy()\n\n                # Calculate the center coordinates\n                x_center = int((x1 + x2) / 2)\n                y_center = int((y1 + y2) / 2)\n                \n                # Print the first frame number and center coordinates\n                print(f\"First detection at frame: {frame_number}\")\n                print(f\"Center coordinates: (x={x_center}, y={y_center})\")\n\n                object_detected = True\n                break\n\n    if object_detected:\n        break\n\ncap.release()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"x_center:\",x_center)\nprint(\"y_center:\",y_center)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using Yolov8s pretrained model for direct detection and getting the frame","metadata":{}},{"cell_type":"markdown","source":"#### just mention class name and it will return frame no. and coordinates","metadata":{}},{"cell_type":"code","source":"# Load the YOLOv8s model\nmodel = YOLO('yolov8s.pt')  # Make sure the model is trained on the \"bottle\" class\n\n# Process the video\nvideo_path = '/kaggle/input/yolov-train-data/VID202408142242002.mp4'\ncap = cv2.VideoCapture(video_path)\n\nx_center = 0\ny_center = 0\nframe_number = 0\nobject_detected = False\nconfidence_threshold = 0.8  # Set the confidence threshold\n\nwhile cap.isOpened():\n    ret, frame = cap.read()\n    if not ret:\n        break\n\n    frame_number += 1\n\n    # Run YOLOv8 detection\n    results = model(frame)\n\n    for r in results:\n        for box in r.boxes:\n            # Get the class label for the detected object\n            cls = int(box.cls[0].cpu().numpy())\n            class_name = model.names[cls]\n\n            # Check if the detected object is a \"bottle\" and has confidence > 0.8\n            confidence = box.conf[0].cpu().numpy()\n            if class_name == 'bottle' and confidence > confidence_threshold:\n                # Get the bounding box coordinates\n                x1, y1, x2, y2 = box.xyxy[0].cpu().numpy()\n\n                # Calculate the center coordinates\n                x_center = int((x1 + x2) / 2)\n                y_center = int((y1 + y2) / 2)\n                \n                # Print the first frame number and center coordinates\n                print(f\"First bottle detection at frame: {frame_number}\")\n                print(f\"Center coordinates: (x={x_center}, y={y_center}) with confidence {confidence:.2f}\")\n\n                object_detected = True\n                break  # Exit the loop after the first detection\n\n    if object_detected:\n        break  # Exit the main loop after the first detection\n\ncap.release()\n\n# If no bottle was detected with confidence > 0.8\nif not object_detected:\n    print(\"No requested Object detected in the video with confidence greater than 0.8.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"x_center:\",x_center)\nprint(\"y_center:\",y_center)\nprint(\"Frame No.:\",frame_number)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### clearing GPU cache","metadata":{}},{"cell_type":"code","source":"import torch\ntorch.cuda.empty_cache()\nprint(\"Done\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Video segmenting","metadata":{}},{"cell_type":"markdown","source":"### importing SAM-2 model (may take a while to download)","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/segment-anything-2.git\n%cd /kaggle/working/segment-anything-2\n%pip install -e .\n%cd /kaggle/working/segment-anything-2/checkpoints\n!bash /kaggle/working/segment-anything-2/checkpoints/download_ckpts.sh\n%cd /kaggle/working/segment-anything-2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport matplotlib.pyplot as plt\nfrom PIL import Image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use bfloat16 for the entire notebook\ntorch.autocast(device_type=\"cuda\", dtype=torch.float16).__enter__()\n\nif torch.cuda.get_device_properties(0).major >= 8:\n    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)\n    torch.backends.cuda.matmul.allow_tf32 = True\n    torch.backends.cudnn.allow_tf32 = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## video to frames","metadata":{}},{"cell_type":"code","source":"import cv2\nimport os\nimport shutil\n\ndef video_to_frames(video_path, output_folder):\n    # Ensure the output folder is clean\n    if os.path.exists(output_folder):\n        shutil.rmtree(output_folder)\n    os.makedirs(output_folder)\n    \n    # Open the video file\n    video_capture = cv2.VideoCapture(video_path)\n    \n    frame_count = 0\n    success = True\n\n    while success:\n        success, frame = video_capture.read()\n        if success:\n            # Save the frame with a consistent naming convention\n            frame_filename = os.path.join(output_folder, f\"{frame_count:05d}.jpg\")\n            cv2.imwrite(frame_filename, frame)\n            frame_count += 1\n\n    video_capture.release()\n    print(f\"Extracted {frame_count} frames to {output_folder}\")\n    return frame_count\n\n# Example usage\nvideo_path = \"/kaggle/input/shaolin-soccer/Untitled video - Made with Clipchamp.mp4\"\noutput_folder = \"/kaggle/working/output_frames\"\ntotal_frames = video_to_frames(video_path, output_folder)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## reordering Frames to video propagation\n","metadata":{}},{"cell_type":"code","source":"frame_number =0 ","metadata":{"execution":{"iopub.status.busy":"2024-08-23T05:45:01.624801Z","iopub.execute_input":"2024-08-23T05:45:01.625582Z","iopub.status.idle":"2024-08-23T05:45:01.636025Z","shell.execute_reply.started":"2024-08-23T05:45:01.625533Z","shell.execute_reply":"2024-08-23T05:45:01.634951Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"### (replace it with **frame_number** if using YOLO model)\n\n#### frame_number = frame_number","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\n\ndef reorder_frames(video_dir, ann_frame_idx, output_dir):\n    # Ensure the output directory is clean\n    if os.path.exists(output_dir):\n        shutil.rmtree(output_dir)\n    os.makedirs(output_dir)\n    \n    # Get and sort the list of frame filenames\n    frame_names = [\n        p for p in os.listdir(video_dir)\n        if os.path.splitext(p)[-1] in [\".jpg\", \".jpeg\", \".JPG\", \".JPEG\"]\n    ]\n    frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))\n    \n    total_frames = len(frame_names)\n    \n    # Copy and reorder the frames to the new directory\n    for i in range(total_frames):\n        if i >= ann_frame_idx:\n            new_idx = i - ann_frame_idx\n        else:\n            new_idx = total_frames - ann_frame_idx + i\n        old_path = os.path.join(video_dir, frame_names[i])\n        new_path = os.path.join(output_dir, f\"{new_idx:05d}.jpg\")\n        shutil.copy2(old_path, new_path)\n    \n    print(f\"Frames reordered and copied to {output_dir} successfully.\")\n    return len(os.listdir(output_dir))\n\n# Example usage\nreordered_dir = \"/kaggle/working/reordered_frames\"\nann_frame_idx = frame_number  # Frame index to start as 0\nreordered_count = reorder_frames(output_folder, ann_frame_idx, reordered_dir)\n\n# Verify total frame consistency\nif total_frames == reordered_count:\n    print(\"Frame count matches after reordering.\")\nelse:\n    print(f\"Frame count mismatch! Extracted: {total_frames}, Reordered: {reordered_count}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing Model and creating predictor","metadata":{}},{"cell_type":"code","source":"from sam2.build_sam import build_sam2_video_predictor\n\nsam2_checkpoint = \"/kaggle/working/segment-anything-2/checkpoints/sam2_hiera_base_plus.pt\"\nmodel_cfg = \"sam2_hiera_b+.yaml\"\n\npredictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## checking image where object is detected","metadata":{}},{"cell_type":"code","source":"frame_no = frame_number\n\ndef show_mask(mask, ax, obj_id=None, random_color=False):\n    if random_color:\n        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n    else:\n        cmap = plt.get_cmap(\"tab10\")\n        cmap_idx = 0 if obj_id is None else obj_id\n        color = np.array([*cmap(cmap_idx)[:3], 0.6])\n    h, w = mask.shape[-2:]\n    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n    ax.imshow(mask_image)\n\n\ndef show_points(coords, labels, ax, marker_size=200):\n    pos_points = coords[labels==1]\n    neg_points = coords[labels==0]\n    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n    \n# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`\nvideo_dir = \"/kaggle/working/reordered_frames\"\n\n# scan all the JPEG frame names in this directory\nframe_names = [\n    p for p in os.listdir(video_dir)\n    if os.path.splitext(p)[-1] in [\".jpg\", \".jpeg\", \".JPG\", \".JPEG\"]\n]\nframe_names.sort(key=lambda p: int(os.path.splitext(p)[0]))\n\n# take a look the first video frame\nframe_idx = frame_no\nplt.figure(figsize=(12, 8))\nplt.title(f\"frame {frame_idx}\")\nplt.imshow(Image.open(os.path.join(video_dir, frame_names[frame_idx])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_state = predictor.init_state(video_path=video_dir)\npredictor.reset_state(inference_state)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Masking the image object where object is detected in frame with coordinates","metadata":{}},{"cell_type":"code","source":"x_center= 1050\ny_center = 650","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### in case using Yolo model replace,\n\n### x_center =x_center\n### y_center =y_center","metadata":{}},{"cell_type":"code","source":"ann_frame_idx = 0  # the frame index we interact with\nann_obj_id = 1  # give a unique id to each object we interact with (it can be any integers)\nx = x_center\ny = y_center\n\npoints = np.array([[x,y]], dtype=np.float32)\nlabels = np.array([1], np.int32)\n_, out_obj_ids, out_mask_logits = predictor.add_new_points(\n    inference_state=inference_state,\n    frame_idx=ann_frame_idx,\n    obj_id=ann_obj_id,\n    points=points,\n    labels=labels,\n)\n\nplt.figure(figsize=(12, 8))\nplt.title(f\"frame {ann_frame_idx}\")\nplt.imshow(Image.open(os.path.join(video_dir, frame_names[ann_frame_idx])))\nshow_points(points, labels, plt.gca())\nshow_mask((out_mask_logits[0] > 0.0).cpu().numpy(), plt.gca(), obj_id=out_obj_ids[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Note: provide additional points if object not detected properly\n\n### in the format\n#### points = np.array([[x,y],[x1,y1],[x2,y2]], dtype=np.float32)\n#### labels = np.array([1,1,1], np.int32)\n\n#### in labels 1 indicate inclusive and 0 excluding point","metadata":{}},{"cell_type":"code","source":"def count_files_in_folder(folder_path):\n    \"\"\"\n    Count the number of files in a given folder.\n    \n    Args:\n    - folder_path (str): Path to the folder.\n    \n    Returns:\n    - int: Number of files in the folder.\n    \"\"\"\n    return len([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))])\n\n# Example usage\nfolder_path = \"/kaggle/working/reordered_frames\"  # Replace with your actual folder path\nnum_files = count_files_in_folder(folder_path)\nprint(f\"Number of files in the folder: {num_files}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mask generation\n### Propagating into Video with reordered Frames","metadata":{}},{"cell_type":"markdown","source":"### if Addition points are provided also change them in below code","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport shutil  # Importing shutil to remove directories\n\ndef apply_mask_to_image(frame, mask):\n    \"\"\"\n    Apply a mask to an image frame, setting non-mask areas to zero.\n    \"\"\"\n    h, w, _ = frame.shape\n    mask_resized = np.resize(mask, (h, w))  # Resize mask to match frame dimensions\n    mask_3d = np.repeat(mask_resized[:, :, np.newaxis], 3, axis=2)  # Expand mask dimensions for RGB channels\n    masked_frame = frame * mask_3d  # Apply the mask to the frame\n    return masked_frame\n\ndef show_mask(mask, ax, obj_id=None, random_color=False):\n    if random_color:\n        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n    else:\n        cmap = plt.get_cmap(\"tab10\")\n        cmap_idx = 0 if obj_id is None else obj_id\n        color = np.array([*cmap(cmap_idx)[:3], 0.6])\n    h, w = mask.shape[-2:]\n    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n    ax.imshow(mask_image)\n\ndef show_points(coords, labels, ax, marker_size=200):\n    pos_points = coords[labels == 1]\n    neg_points = coords[labels == 0]\n    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n\n# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`\nvideo_dir = \"/kaggle/working/reordered_frames\"\n\n# Scan all the JPEG frame names in this directory\nframe_names = [\n    p for p in os.listdir(video_dir)\n    if os.path.splitext(p)[-1] in [\".jpg\", \".jpeg\", \".JPG\", \".JPEG\"]\n]\nframe_names.sort(key=lambda p: int(os.path.splitext(p)[0]))\n\n# Initialize predictor and inference state\ninference_state = predictor.init_state(video_path=video_dir)\n\n# Reset the predictor state\npredictor.reset_state(inference_state)\n\n# Frame and object IDs\nann_frame_idx = 0  # frames are reordered\nann_obj_id = 1  # Give a unique ID to each object we interact with (can be any integer)\n\n# Add a 2nd positive click at (x, y) = (250, 220) to refine the mask\npoints = np.array([[x,y]], dtype=np.float32)\nlabels = np.array([1], np.int32)  # 1 means positive click, 0 means negative click\n_, out_obj_ids, out_mask_logits = predictor.add_new_points(\n    inference_state=inference_state,\n    frame_idx=ann_frame_idx,\n    obj_id=ann_obj_id,\n    points=points,\n    labels=labels,\n)\n\n# Run propagation throughout the video and collect the results in a dict\nvideo_segments = {}  # video_segments contains the per-frame segmentation results\nfor out_frame_idx, out_obj_ids, out_mask_logits in predictor.propagate_in_video(inference_state):\n    video_segments[out_frame_idx] = {\n        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()\n        for i, out_obj_id in enumerate(out_obj_ids)\n    }\n\n# Create an output directory for images\noutput_dir = '/kaggle/working/mask_segmentation_images'\nif not os.path.exists(output_dir):\n    os.makedirs(output_dir)\nelse:\n    # If the directory exists, clear its kaggle/workings\n    for filename in os.listdir(output_dir):\n        file_path = os.path.join(output_dir, filename)\n        try:\n            if os.path.isfile(file_path) or os.path.islink(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                shutil.rmtree(file_path)\n        except Exception as e:\n            print(f\"Failed to delete {file_path}. Reason: {e}\")\n\n# Render and save masked images every few frames\nvis_frame_stride = 1\nplt.close(\"all\")\nfor out_frame_idx in range(0, len(frame_names), vis_frame_stride):\n    frame = np.array(Image.open(os.path.join(video_dir, frame_names[out_frame_idx])))\n    masked_frame = frame.copy()  # Create a copy of the frame for modification\n    for out_obj_id, out_mask in video_segments[out_frame_idx].items():\n        masked_frame = apply_mask_to_image(masked_frame, out_mask)\n\n    # Convert masked frame to Image object for saving\n    masked_image = Image.fromarray(masked_frame.astype('uint8'))\n    masked_image.save(os.path.join(output_dir, f'frame_{out_frame_idx}.png'))\n\n    # Optional: Display the masked frame\n#     plt.figure(figsize=(6, 4))\n#     plt.title(f\"frame {out_frame_idx}\")\n#     plt.imshow(masked_frame)\n#     plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### we can also display the masked frame(s) by un-commenting the last 4 rows","metadata":{}},{"cell_type":"markdown","source":"## restore Original order of the video frames\n\n### this will restore the original order of the frames","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\n\ndef restore_original_order(video_dir, ann_frame_idx, output_dir):\n    \"\"\"\n    Restore the original order of frames from a directory and save them into a new directory.\n    \n    Args:\n    - video_dir (str): Directory containing the reordered frames.\n    - ann_frame_idx (int): The frame index used to start the reordering.\n    - output_dir (str): Directory to save the restored frames.\n    \"\"\"\n    # Ensure the output directory is clean\n    if os.path.exists(output_dir):\n        shutil.rmtree(output_dir)\n    os.makedirs(output_dir)\n    \n    # Get a list of all frame filenames in the original directory\n    frame_names = [\n        p for p in os.listdir(video_dir)\n        if p.endswith(\".png\") and p.startswith(\"frame_\")\n    ]\n    \n    # Ensure frames are sorted numerically by extracting the number from the filename\n    frame_names.sort(key=lambda p: int(p.split('_')[-1].split('.')[0]))\n\n    # Calculate total number of frames\n    total_frames = len(frame_names)\n\n    # Calculate the original frame indices\n    original_indices = {}\n    for i in range(total_frames):\n        if i < (total_frames - ann_frame_idx):\n            original_idx = i + ann_frame_idx\n        else:\n            original_idx = i - (total_frames - ann_frame_idx)\n        original_indices[frame_names[i]] = f\"frame_{original_idx:03d}.png\"\n    \n    # Copy and rename the files into the new directory\n    for old_name, new_name in original_indices.items():\n        old_path = os.path.join(video_dir, old_name)\n        new_path = os.path.join(output_dir, new_name)\n        shutil.copy2(old_path, new_path)\n    \n    print(f\"Frames restored to original order and saved to {output_dir} successfully.\")\n\n# Example usage\nvideo_dir = \"/kaggle/working/mask_segmentation_images\"  # Replace with your original frames directory\nann_frame_idx = 0  # The frame index used to start the reordering\noutput_dir = \"/kaggle/working/restored_frames\"  # Replace with your desired output folder path\nrestore_original_order(video_dir, ann_frame_idx, output_dir)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## converting mask Frames back to video","metadata":{}},{"cell_type":"code","source":"import cv2\nimport os\n\ndef frames_to_video(frames_folder, output_video_path, fps=30):\n    # Check if the output video file already exists and delete it\n    if os.path.exists(output_video_path):\n        try:\n            os.remove(output_video_path)\n            print(f\"Existing file {output_video_path} removed.\")\n        except Exception as e:\n            print(f\"Failed to remove {output_video_path}. Reason: {e}\")\n            return\n\n    # Get a list of frame files and sort them by name\n    frame_files = [f for f in os.listdir(frames_folder) if f.endswith('.png')]\n    frame_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))  # Sort by frame number\n\n    # Check if there are any frames to process\n    if not frame_files:\n        print(\"No frames found in the specified folder.\")\n        return\n\n    # Read the first frame to get the dimensions\n    first_frame_path = os.path.join(frames_folder, frame_files[0])\n    first_frame = cv2.imread(first_frame_path)\n    if first_frame is None:\n        print(f\"Failed to read the first frame at {first_frame_path}\")\n        return\n    height, width, _ = first_frame.shape\n\n    # Initialize the video writer\n    fourcc = cv2.VideoWriter_fourcc(*'mp4v')  # Codec for mp4 format\n    video_writer = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height))\n\n    # Write each frame to the video\n    for frame_file in frame_files:\n        frame_path = os.path.join(frames_folder, frame_file)\n        frame = cv2.imread(frame_path)\n        if frame is None:\n            print(f\"Failed to read frame at {frame_path}\")\n            continue\n        video_writer.write(frame)\n\n    # Release the video writer\n    video_writer.release()\n    print(f\"Video saved to {output_video_path}\")\n\n# Example usage\nframes_folder = r'/kaggle/working/restored_frames'  # Replace with the folder containing your frames\noutput_video_path = r\"/kaggle/working/mask_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inverse Mask Generation","metadata":{}},{"cell_type":"markdown","source":"### similarly in case of additional points make changes here also","metadata":{}},{"cell_type":"code","source":"def clear_output_directory(directory):\n    \"\"\"\n    Remove all files in the given directory.\n    \"\"\"\n    if os.path.exists(directory):\n        for file in os.listdir(directory):\n            file_path = os.path.join(directory, file)\n            try:\n                if os.path.isfile(file_path):\n                    os.unlink(file_path)\n            except Exception as e:\n                print(f\"Failed to delete {file_path}. Reason: {e}\")\n\ndef apply_inverse_mask_to_image(frame, mask):\n    \"\"\"\n    Apply the inverse of a mask to an image frame, setting mask areas to zero.\n    \"\"\"\n    h, w, _ = frame.shape\n    mask_resized = np.resize(mask, (h, w))  # Resize mask to match frame dimensions\n    inverse_mask = 1 - mask_resized  # Invert the mask\n    mask_3d = np.repeat(inverse_mask[:, :, np.newaxis], 3, axis=2)  # Expand mask dimensions for RGB channels\n    masked_frame = frame * mask_3d  # Apply the inverse mask to the frame\n    return masked_frame\n\ndef save_masked_image(masked_frame, out_frame_idx, output_dir):\n    \"\"\"\n    Save the masked image to the output directory.\n    \"\"\"\n    # Convert masked frame to Image object for saving\n    masked_image = Image.fromarray(masked_frame.astype('uint8'))\n    masked_image.save(os.path.join(output_dir, f'frame_{out_frame_idx}.png'))\n\ndef show_mask(mask, ax, obj_id=None, random_color=False):\n    if random_color:\n        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n    else:\n        cmap = plt.get_cmap(\"tab10\")\n        cmap_idx = 0 if obj_id is None else obj_id\n        color = np.array([*cmap(cmap_idx)[:3], 0.6])\n    h, w = mask.shape[-2:]\n    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n    ax.imshow(mask_image)\n\ndef show_points(coords, labels, ax, marker_size=200):\n    pos_points = coords[labels == 1]\n    neg_points = coords[labels == 0]\n    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n\n# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`\nvideo_dir = \"/kaggle/working/reordered_frames\"\n\n# Scan all the JPEG frame names in this directory\nframe_names = [\n    p for p in os.listdir(video_dir)\n    if os.path.splitext(p)[-1] in [\".jpg\", \".jpeg\", \".JPG\", \".JPEG\"]\n]\nframe_names.sort(key=lambda p: int(os.path.splitext(p)[0]))\n\n# Initialize predictor and inference state\ninference_state = predictor.init_state(video_path=video_dir)\n\n# Reset the predictor state\npredictor.reset_state(inference_state)\n\n# Frame and object IDs\nann_frame_idx = 0  # The frame index we interact with\nann_obj_id = 1  # Give a unique ID to each object we interact with (can be any integer)\n\n# Add a 2nd positive click at (x, y) = (250, 220) to refine the mask\npoints = np.array([[x,y]], dtype=np.float32)\nlabels = np.array([1], np.int32)  # 1 means positive click, 0 means negative click\n_, out_obj_ids, out_mask_logits = predictor.add_new_points(\n    inference_state=inference_state,\n    frame_idx=ann_frame_idx,\n    obj_id=ann_obj_id,\n    points=points,\n    labels=labels,\n)\n\n# Run propagation throughout the video and collect the results in a dict\nvideo_segments = {}  # video_segments contains the per-frame segmentation results\nfor out_frame_idx, out_obj_ids, out_mask_logits in predictor.propagate_in_video(inference_state):\n    video_segments[out_frame_idx] = {\n        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()\n        for i, out_obj_id in enumerate(out_obj_ids)\n    }\n\n# Create an output directory for images\noutput_dir = '/kaggle/working/inverse_segmentation_images'\nos.makedirs(output_dir, exist_ok=True)\n\n# Clear the output directory\nclear_output_directory(output_dir)\n\n# Render and save inverse masked images every few frames\nvis_frame_stride = 1\nplt.close(\"all\")\nfor out_frame_idx in range(0, len(frame_names), vis_frame_stride):\n    frame = np.array(Image.open(os.path.join(video_dir, frame_names[out_frame_idx])))\n    masked_frame = frame.copy()  # Create a copy of the frame for modification\n    for out_obj_id, out_mask in video_segments[out_frame_idx].items():\n        masked_frame = apply_inverse_mask_to_image(masked_frame, out_mask)\n\n    # Save the inverse masked frame\n    save_masked_image(masked_frame, out_frame_idx, output_dir)\n\n    # Optional: Display the inverse masked frame\n    # plt.figure(figsize=(6, 4))\n    # plt.title(f\"frame {out_frame_idx}\")\n    # plt.imshow(masked_frame)\n    # plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## restoring to original frames of inverse mask","metadata":{}},{"cell_type":"code","source":"video_dir = \"/kaggle/working/inverse_segmentation_images\"  # Replace with your original frames directory\nann_frame_idx = 0  # The frame index used to start the reordering\noutput_dir = \"/kaggle/working/inverse_restored_frames\"  # Replace with your desired output folder path\nrestore_original_order(video_dir, ann_frame_idx, output_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## converting inverse mask frames to video","metadata":{}},{"cell_type":"code","source":"frames_folder = r'/kaggle/working/inverse_restored_frames'  # Replace with the folder containing your frames\noutput_video_path = r\"/kaggle/working/inverse_mask_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Video mask Pixelation","metadata":{}},{"cell_type":"code","source":"def pixelate_area(image, mask, pixelation_level):\n    \"\"\"\n    Apply pixelation to the masked area of an image.\n\n    Parameters:\n    - image: NumPy array of the image to be pixelated.\n    - mask: Boolean NumPy array indicating the masked area.\n    - pixelation_level: Int, the size of the blocks used for pixelation.\n    \"\"\"\n    # Create a copy of the image to modify\n    pixelated_image = image.copy()\n\n    # Get image dimensions\n    h, w, _ = image.shape\n\n    # Loop through the masked area and apply pixelation\n    for y in range(0, h, pixelation_level):\n        for x in range(0, w, pixelation_level):\n            # Define the block area\n            block = (slice(y, min(y + pixelation_level, h)), slice(x, min(x + pixelation_level, w)))\n\n            # Check if the block is within the masked area\n            if np.any(mask[block]):\n                # Compute the mean color of the block\n                mean_color = image[block].mean(axis=(0, 1)).astype(int)\n\n                # Apply the mean color to the block\n                pixelated_image[block] = mean_color\n\n    return pixelated_image\n\ndef combine_pixelated_mask(masked_image_path, inverse_masked_image_path, save_path, pixelation_level=10):\n    \"\"\"\n    Combine the pixelated masked areas from the masked image with the inverse-masked image.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image.\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - save_path: String, path where the combined image will be saved.\n    - pixelation_level: Int, the size of the blocks used for pixelation.\n    \"\"\"\n    # Open images\n    masked_image = Image.open(masked_image_path).convert(\"RGBA\")\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Ensure images are the same size by resizing the inverse image\n    if masked_image.size != inverse_masked_image.size:\n        inverse_masked_image = inverse_masked_image.resize(masked_image.size)\n\n    # Convert images to numpy arrays\n    masked_array = np.array(masked_image)\n    inverse_masked_array = np.array(inverse_masked_image)\n\n    # Create a mask where the original mask was applied (non-zero areas in any color channel)\n    mask = np.any(masked_array[..., :3] > 0, axis=-1)\n\n    # Pixelate the masked area\n    pixelated_mask = pixelate_area(masked_array, mask, pixelation_level)\n\n    # Replace inverse-masked image values with pixelated masked image values where mask is true\n    combined_array = inverse_masked_array.copy()\n    combined_array[mask] = pixelated_mask[mask]\n\n    # Convert back to image\n    combined_image = Image.fromarray(combined_array)\n\n    # Save the combined image\n    combined_image.save(save_path)\n    print(f\"Combined image saved as {save_path}\")\n\n#     # Display the combined image\n#     plt.imshow(combined_image)\n#     plt.axis('off')\n#     plt.show()\n\n# Directory paths\nmasked_images_dir = \"/kaggle/working/restored_frames\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/pixelated_combined_images\"\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Get and sort the list of image files\nimage_files = [f for f in os.listdir(masked_images_dir) if f.startswith(\"frame_\") and f.endswith(\".png\")]\nimage_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))\n\n# Iterate over the sorted files\nfor image_name in image_files:\n    masked_image_path = os.path.join(masked_images_dir, image_name)\n    inverse_image_path = os.path.join(inverse_images_dir, image_name)\n    save_path = os.path.join(output_dir, f\"pixelated_combined_{image_name}\")\n\n    # Check if the corresponding inverse image exists before combining\n    if os.path.exists(inverse_image_path):\n        combine_pixelated_mask(masked_image_path, inverse_image_path, save_path, pixelation_level=20)\n    else:\n        print(f\"Warning: Missing inverse file for {image_name}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## converting frames of pixels to video","metadata":{}},{"cell_type":"code","source":"def frames_to_video(frames_folder, output_video_path, fps=30):\n    # Get a list of frame files and sort them by name\n    frame_files = [f for f in os.listdir(frames_folder) if f.endswith('.png')]\n\n    # Sort by frame number, assuming the filename format is \"frame_<number>.png\"\n    frame_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))\n\n    if not frame_files:\n        print(\"No frame files found in the specified directory.\")\n        return\n\n    # Read the first frame to get the dimensions\n    first_frame_path = os.path.join(frames_folder, frame_files[0])\n    first_frame = cv2.imread(first_frame_path)\n    if first_frame is None:\n        print(f\"Error reading the first frame: {first_frame_path}\")\n        return\n\n    height, width, _ = first_frame.shape\n\n    # Initialize the video writer\n    fourcc = cv2.VideoWriter_fourcc(*'mp4v')  # Codec for mp4 format\n    video_writer = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height))\n\n    # Write each frame to the video\n    for frame_file in frame_files:\n        frame_path = os.path.join(frames_folder, frame_file)\n        frame = cv2.imread(frame_path)\n        if frame is not None:\n            video_writer.write(frame)\n        else:\n            print(f\"Error reading frame: {frame_path}\")\n\n    # Release the video writer\n    video_writer.release()\n    print(f\"Video saved to {output_video_path}\")\n\n# Example usage\nframes_folder = '/kaggle/working/pixelated_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/pixelated_combined_images_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## side by side video of original with pixelated video.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport subprocess\nimport shutil\n\n# Directories for the input frames and output combined frames (switched)\ndir1 = '/kaggle/working/output_frames'  # Formerly dir2https://accounts.google.com/b/0/AddMailService\ndir2 = '/kaggle/working/pixelated_combined_images'  # Formerly dir1\noutput_dir = '/kaggle/working/combined_frames_pix'\nvideo_output = '/kaggle/working/pix_output_video.mp4'\n\n# Ensure the output directory exists and is empty\nif os.path.exists(output_dir):\n    shutil.rmtree(output_dir)  # Remove the directory and its contents\nos.makedirs(output_dir)  # Recreate the empty directory\n\n# Remove the previous video if it exists\nif os.path.exists(video_output):\n    os.remove(video_output)\n\n# Get sorted lists of the frames\nframes1 = sorted([f for f in os.listdir(dir1) if f.endswith('.jpg')])\nframes2 = sorted([f for f in os.listdir(dir2) if f.endswith('.png')])\n\n# Iterate over both directories and combine images\nfor idx, (f1, f2) in enumerate(zip(frames1, frames2), start=1):\n    img1 = Image.open(os.path.join(dir1, f1))\n    img2 = Image.open(os.path.join(dir2, f2))\n    \n    # Assuming both images have the same height, concatenate side by side\n    combined_img = Image.new('RGB', (img1.width + img2.width, img1.height))\n    combined_img.paste(img1, (0, 0))\n    combined_img.paste(img2, (img1.width, 0))\n    \n    # Save combined image with a sequential name like combined_frame_001.png\n    combined_img.save(os.path.join(output_dir, f\"combined_frame_{idx:03d}.png\"))\n\nprint(f\"Frames combined and saved in {output_dir}\")\n\n# List the files in the output directory to verify they exist\nprint(\"Files in output directory:\", os.listdir(output_dir))\n\n# Convert the combined frames into a video using ffmpeg\nsubprocess.run([\n    'ffmpeg', '-framerate', '30', '-i', \n    f'{output_dir}/combined_frame_%03d.png', '-c:v', \n    'libx264', '-pix_fmt', 'yuv420p', video_output\n])\n\nprint(f\"Video saved as {video_output}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked area Hue change in video","metadata":{}},{"cell_type":"code","source":"import matplotlib.colors as mcolors\n\ndef change_hue(image, mask, hue_shift):\n    \"\"\"\n    Change the hue of the masked area in an image.\n\n    Parameters:\n    - image: NumPy array of the image to be modified (in RGB).\n    - mask: Boolean NumPy array indicating the masked area.\n    - hue_shift: Float, amount to shift the hue (0 to 1 for a complete cycle).\n    \"\"\"\n    # Convert the image to float in the range [0, 1]\n    float_image = image.astype('float32') / 255.0\n\n    # Convert to HSV\n    hsv_image = mcolors.rgb_to_hsv(float_image)\n\n    # Change the hue in the masked area\n    hsv_image[..., 0][mask] = (hsv_image[..., 0][mask] + hue_shift) % 1.0\n\n    # Convert back to RGB\n    modified_float_image = mcolors.hsv_to_rgb(hsv_image)\n\n    # Scale back to [0, 255]\n    modified_image = (modified_float_image * 255).astype('uint8')\n\n    return modified_image\n\ndef combine_hue_modified_mask(masked_image_path, inverse_masked_image_path, save_path, hue_shift=0.1):\n    \"\"\"\n    Combine the hue-modified masked areas from the masked image with the inverse-masked image.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image.\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - save_path: String, path where the combined image will be saved.\n    - hue_shift: Float, amount to shift the hue (0 to 1 for a complete cycle).\n    \"\"\"\n    # Open images\n    masked_image = Image.open(masked_image_path).convert(\"RGBA\")\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Ensure images are the same size by resizing the inverse image\n    if masked_image.size != inverse_masked_image.size:\n        inverse_masked_image = inverse_masked_image.resize(masked_image.size)\n\n    # Convert images to numpy arrays\n    masked_array = np.array(masked_image)\n    inverse_masked_array = np.array(inverse_masked_image)\n\n    # Create a mask where the original mask was applied (non-zero areas in any color channel)\n    mask = np.any(masked_array[..., :3] > 0, axis=-1)\n\n    # Change the hue of the masked area\n    hue_modified_mask = change_hue(masked_array[..., :3], mask, hue_shift)\n\n    # Replace inverse-masked image values with hue-modified masked image values where mask is true\n    combined_array = inverse_masked_array.copy()\n    combined_array[mask] = np.dstack((hue_modified_mask, masked_array[..., 3]))[mask]  # Preserve alpha channel\n\n    # Convert back to image\n    combined_image = Image.fromarray(combined_array)\n\n    # Save the combined image\n    combined_image.save(save_path)\n    print(f\"Combined image saved as {save_path}\")\n\n#     # Display the combined image\n#     plt.imshow(combined_image)\n#     plt.axis('off')\n#     plt.show()\n\n# Directory paths\nmasked_images_dir = \"/kaggle/working/restored_frames\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/hue_combined_images\"\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Get and sort the list of image files\nimage_files = [f for f in os.listdir(masked_images_dir) if f.startswith(\"frame_\") and f.endswith(\".png\")]\nimage_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))\n\n# Iterate over the sorted files\nfor image_name in image_files:\n    masked_image_path = os.path.join(masked_images_dir, image_name)\n    inverse_image_path = os.path.join(inverse_images_dir, image_name)\n    save_path = os.path.join(output_dir, f\"hue_modified_combined_{image_name}\")\n\n    # Check if the corresponding inverse image exists before combining\n    if os.path.exists(inverse_image_path):\n        combine_hue_modified_mask(masked_image_path, inverse_image_path, save_path, hue_shift=0.25)\n    else:\n        print(f\"Warning: Missing inverse file for {image_name}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## converting back hue change to video","metadata":{}},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/hue_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/hue_combined_images_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## side by side video of original with Hue video.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport subprocess\nimport shutil\n\n# Directories for the input frames and output combined frames (switched)\ndir1 = '/kaggle/working/output_frames'  # Formerly dir2https://accounts.google.com/b/0/AddMailService\ndir2 = '/kaggle/working/hue_combined_images'  # Formerly dir1\noutput_dir = '/kaggle/working/hue_with_og_combined_frames'\nvideo_output = '/kaggle/working/hue_with_og_output_video.mp4'\n\n# Ensure the output directory exists and is empty\nif os.path.exists(output_dir):\n    shutil.rmtree(output_dir)  # Remove the directory and its contents\nos.makedirs(output_dir)  # Recreate the empty directory\n\n# Remove the previous video if it exists\nif os.path.exists(video_output):\n    os.remove(video_output)\n\n# Get sorted lists of the frames\nframes1 = sorted([f for f in os.listdir(dir1) if f.endswith('.jpg')])\nframes2 = sorted([f for f in os.listdir(dir2) if f.endswith('.png')])\n\n# Iterate over both directories and combine images\nfor idx, (f1, f2) in enumerate(zip(frames1, frames2), start=1):\n    img1 = Image.open(os.path.join(dir1, f1))\n    img2 = Image.open(os.path.join(dir2, f2))\n    \n    # Assuming both images have the same height, concatenate side by side\n    combined_img = Image.new('RGB', (img1.width + img2.width, img1.height))\n    combined_img.paste(img1, (0, 0))\n    combined_img.paste(img2, (img1.width, 0))\n    \n    # Save combined image with a sequential name like combined_frame_001.png\n    combined_img.save(os.path.join(output_dir, f\"combined_frame_{idx:03d}.png\"))\n\nprint(f\"Frames combined and saved in {output_dir}\")\n\n# List the files in the output directory to verify they exist\nprint(\"Files in output directory:\", os.listdir(output_dir))\n\n# Convert the combined frames into a video using ffmpeg\nsubprocess.run([\n    'ffmpeg', '-framerate', '30', '-i', \n    f'{output_dir}/combined_frame_%03d.png', '-c:v', \n    'libx264', '-pix_fmt', 'yuv420p', video_output\n])\n\nprint(f\"Video saved as {video_output}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mask replacement with another video","metadata":{}},{"cell_type":"markdown","source":"### replacement video Link required","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nimport cv2\n\ndef replace_area_with_frames(image, mask, replacement_frames, frame_idx):\n    \"\"\"\n    Replace the masked area of an image with a different video frame.\n\n    Parameters:\n    - image: NumPy array of the image to modify.\n    - mask: Boolean NumPy array indicating the masked area.\n    - replacement_frames: List of NumPy arrays, each representing a video frame to use as a replacement.\n    - frame_idx: Int, the index of the current frame in the replacement sequence.\n    \"\"\"\n    # Create a copy of the image to modify\n    modified_image = image.copy()\n\n    # Get the replacement frame, use the last one if index exceeds available frames\n    replacement_frame = replacement_frames[min(frame_idx, len(replacement_frames) - 1)]\n\n    # Resize the replacement frame to match the image size\n    replacement_frame_resized = cv2.resize(replacement_frame, (image.shape[1], image.shape[0]))\n\n    # Replace the masked area with the replacement frame\n    modified_image[mask] = replacement_frame_resized[mask]\n\n    return modified_image\n\ndef combine_mask_with_frames(masked_image_path, inverse_masked_image_path, replacement_frames, save_path, frame_idx):\n    \"\"\"\n    Combine the masked areas from the masked image with the inverse-masked image, using video frames to fill the masked area.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image.\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - replacement_frames: List of NumPy arrays, each representing a video frame to use as a replacement.\n    - save_path: String, path where the combined image will be saved.\n    - frame_idx: Int, the index of the current frame in the replacement sequence.\n    \"\"\"\n    # Open images\n    masked_image = Image.open(masked_image_path).convert(\"RGBA\")\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Ensure images are the same size by resizing the inverse image\n    if masked_image.size != inverse_masked_image.size:\n        inverse_masked_image = inverse_masked_image.resize(masked_image.size)\n\n    # Convert images to numpy arrays\n    masked_array = np.array(masked_image)\n    inverse_masked_array = np.array(inverse_masked_image)\n\n    # Create a mask where the original mask was applied (non-zero areas in any color channel)\n    mask = np.any(masked_array[..., :3] > 0, axis=-1)\n\n    # Replace the masked area with frames from the video\n    replaced_area = replace_area_with_frames(masked_array, mask, replacement_frames, frame_idx)\n\n    # Replace inverse-masked image values with the replaced area image values where mask is true\n    combined_array = inverse_masked_array.copy()\n    combined_array[mask] = replaced_area[mask]\n\n    # Convert back to image\n    combined_image = Image.fromarray(combined_array)\n\n    # Save the combined image\n    combined_image.save(save_path)\n    print(f\"Combined image saved as {save_path}\")\n\n# Directory paths\nmasked_images_dir = \"/kaggle/working/restored_frames\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/mask_replaced_combined_images\"\nreplacement_video_path = \"/kaggle/input/viedo-with-replacementy/Untitled video - Made with Clipchamp (1).mp4\" # input replacement video link\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Load the replacement video frames\nreplacement_frames = []\ncap = cv2.VideoCapture(replacement_video_path)\nwhile cap.isOpened():\n    ret, frame = cap.read()\n    if not ret:\n        break\n    replacement_frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA))\ncap.release()\n\n# Get and sort the list of image files\nimage_files = [f for f in os.listdir(masked_images_dir) if f.startswith(\"frame_\") and f.endswith(\".png\")]\nimage_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))\n\n# Iterate over the sorted files\nfor frame_idx, image_name in enumerate(image_files):\n    masked_image_path = os.path.join(masked_images_dir, image_name)\n    inverse_image_path = os.path.join(inverse_images_dir, image_name)\n    save_path = os.path.join(output_dir, f\"frame_combined_{image_name}\")\n\n    # Check if the corresponding inverse image exists before combining\n    if os.path.exists(inverse_image_path):\n        combine_mask_with_frames(masked_image_path, inverse_image_path, replacement_frames, save_path, frame_idx)\n    else:\n        print(f\"Warning: Missing inverse file for {image_name}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### replaced mask to video ","metadata":{}},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/mask_replaced_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/mask_replaced_combined_images_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## side by side video of original with mask replaced video.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport subprocess\nimport shutil\n\n# Directories for the input frames and output combined frames (switched)\ndir1 = '/kaggle/working/output_frames'  \ndir2 = '/kaggle/working/mask_replaced_combined_images'  \noutput_dir = '/kaggle/working/mask_replacement_with_orginal_combined_frames'\nvideo_output = '/kaggle/working/mask_replacement_with_orginal_output_video.mp4'\n\n# Ensure the output directory exists and is empty\nif os.path.exists(output_dir):\n    shutil.rmtree(output_dir)  # Remove the directory and its contents\nos.makedirs(output_dir)  # Recreate the empty directory\n\n# Remove the previous video if it exists\nif os.path.exists(video_output):\n    os.remove(video_output)\n\n# Get sorted lists of the frames\nframes1 = sorted([f for f in os.listdir(dir1) if f.endswith('.jpg')])\nframes2 = sorted([f for f in os.listdir(dir2) if f.endswith('.png')])\n\n# Iterate over both directories and combine images\nfor idx, (f1, f2) in enumerate(zip(frames1, frames2), start=1):\n    img1 = Image.open(os.path.join(dir1, f1))\n    img2 = Image.open(os.path.join(dir2, f2))\n    \n    # Assuming both images have the same height, concatenate side by side\n    combined_img = Image.new('RGB', (img1.width + img2.width, img1.height))\n    combined_img.paste(img1, (0, 0))\n    combined_img.paste(img2, (img1.width, 0))\n    \n    # Save combined image with a sequential name like combined_frame_001.png\n    combined_img.save(os.path.join(output_dir, f\"combined_frame_{idx:03d}.png\"))\n\nprint(f\"Frames combined and saved in {output_dir}\")\n\n# List the files in the output directory to verify they exist\nprint(\"Files in output directory:\", os.listdir(output_dir))\n\n# Convert the combined frames into a video using ffmpeg\nsubprocess.run([\n    'ffmpeg', '-framerate', '30', '-i', \n    f'{output_dir}/combined_frame_%03d.png', '-c:v', \n    'libx264', '-pix_fmt', 'yuv420p', video_output\n])\n\nprint(f\"Video saved as {video_output}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked area glow effect in video","metadata":{}},{"cell_type":"code","source":"from PIL import Image, ImageFilter\n\ndef apply_blur_to_masked_area(image, mask, blur_radius=10):\n    \"\"\"\n    Apply a blur effect to the masked area of an image.\n\n    Parameters:\n    - image: PIL Image object of the original image.\n    - mask: Boolean NumPy array indicating the masked area.\n    - blur_radius: Integer, the radius of the Gaussian blur for the blur effect.\n    \"\"\"\n    # Convert image to numpy array\n    image_array = np.array(image)\n\n    # Create a mask image\n    mask_image = Image.fromarray((mask * 255).astype('uint8'), mode='L')\n\n    # Apply a Gaussian blur to the mask image\n    blurred_mask_image = mask_image.filter(ImageFilter.GaussianBlur(radius=blur_radius))\n\n    # Convert the blurred mask to RGB\n    blurred_mask_image = blurred_mask_image.convert('RGB')\n    blurred_mask_array = np.array(blurred_mask_image)\n\n    # Create an image with the same dimensions as the original image\n    blurred_area = np.zeros_like(image_array[..., :3])\n    blurred_area[mask] = blurred_mask_array[mask]\n\n    # Combine the blurred area with the original image\n    combined_array = np.where(blurred_area > 0, blurred_area, image_array[..., :3])\n    combined_image = Image.fromarray(np.uint8(combined_array))\n\n    # Preserve the alpha channel from the original image\n    alpha_channel = image_array[..., 3]\n    combined_image = Image.fromarray(np.dstack((combined_array, alpha_channel)))\n\n    return combined_image\n\ndef combine_and_apply_blur(masked_image_path, inverse_masked_image_path, save_path, blur_radius):\n    \"\"\"\n    Apply a blur effect to the masked image and save the result.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image (used to extract the mask).\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - save_path: String, path where the final image will be saved.\n    - blur_radius: Integer, the radius of the Gaussian blur for the blur effect.\n    \"\"\"\n    # Open inverse-masked image\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Extract the mask from the masked image\n    masked_image = Image.open(masked_image_path).convert(\"L\")\n    mask = np.array(masked_image) > 0\n\n    # Apply blur effect to the masked area\n    blurred_image = apply_blur_to_masked_area(inverse_masked_image, mask, blur_radius)\n\n    # Save the final image\n    blurred_image.save(save_path)\n    print(f\"Final image with blur effect saved as {save_path}\")\n\n#     # Display the final image\n#     plt.imshow(blurred_image)\n#     plt.axis('off')\n#     plt.show()\n\n# Directory paths\nmasked_images_dir = \"/kaggle/working/restored_frames\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/blur_combined_images\"\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Get and sort the list of image files\nimage_files = [f for f in os.listdir(masked_images_dir) if f.startswith(\"frame_\") and f.endswith(\".png\")]\nimage_files.sort(key=lambda f: int(f.split('_')[-1].split('.')[0]))\n\n# Define blur radius\nblur_radius = 10\n\n# Iterate over the sorted files\nfor image_name in image_files:\n    masked_image_path = os.path.join(masked_images_dir, image_name)\n    inverse_image_path = os.path.join(inverse_images_dir, image_name)\n    save_path = os.path.join(output_dir, f\"blur_combined_{image_name}\")\n\n    # Check if the corresponding inverse image exists before combining\n    if os.path.exists(inverse_image_path):\n        combine_and_apply_blur(masked_image_path, inverse_image_path, save_path, blur_radius)\n    else:\n        print(f\"Warning: Missing inverse file for {image_name}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### converting glow effect frames into video ","metadata":{}},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/blur_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/blur_combined_images_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generative AI propagation in video","metadata":{}},{"cell_type":"code","source":"!pip install stability-sdk","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Single API mask video generation","metadata":{}},{"cell_type":"markdown","source":"### single API key code uses less stability AI credits can generate upto ~110 frames using 25 credits at below given configuration in code.\n\n### to generate API key from stability AI , signup on statbility ai platform (gives 25 $ free credit on new account) , copy API key and paste in the below code","metadata":{}},{"cell_type":"markdown","source":"#### Note: Due to generate high no. of frames quality is significantly poor for single API key","metadata":{}},{"cell_type":"code","source":"import os\nimport io\nimport warnings\nfrom PIL import Image\nfrom stability_sdk import client\nimport stability_sdk.interfaces.gooseai.generation.generation_pb2 as generation\n\n# Our Host URL should not be prepended with \"https\" nor should it have a trailing slash.\nos.environ['STABILITY_HOST'] = 'grpc.stability.ai:443'\n\n# Sign up for an account at the following link to get an API Key.\n# https://platform.stability.ai/\n\n# Click on the following link once you have created an account to be taken to your API Key.\n# https://platform.stability.ai/account/keys\n\n# Paste your API Key below.\n\nos.environ['STABILITY_KEY'] = 'sk-23mieeVXXXXXXXXXAegcZW3DZpGIz0M5'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set up our connection to the API.\nstability_api = client.StabilityInference(\n    key=os.environ['STABILITY_KEY'], # API Key reference.\n    verbose=True, # Print debug messages.\n    engine=\"stable-diffusion-xl-1024-v1-0\", # Set the engine to use for generation.\n    # Check out the following link for a list of available engines: https://platform.stability.ai/docs/features/api-parameters#engine\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport io\nimport warnings\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\ndef clear_output_directory(directory):\n    \"\"\"\n    Remove all files in the given directory.\n    \"\"\"\n    if os.path.exists(directory):\n        for file in os.listdir(directory):\n            file_path = os.path.join(directory, file)\n            try:\n                if os.path.isfile(file_path):\n                    os.unlink(file_path)\n            except Exception as e:\n                print(f\"Failed to delete {file_path}. Reason: {e}\")\n\ndef resize_image(image_path, output_path, max_size=1024):\n    \"\"\"\n    Resize an image if it exceeds the max_size dimension.\n    \"\"\"\n    # Open the image\n    image = Image.open(image_path)\n\n    # Get the current width and height of the image\n    width, height = image.size\n\n    # Calculate the scaling factor\n    if width > height:\n        scaling_factor = max_size / width\n    else:\n        scaling_factor = max_size / height\n\n    # Only resize if the image is larger than the max_size\n    if scaling_factor < 1:\n        # Calculate new dimensions\n        new_width = int(width * scaling_factor)\n        new_height = int(height * scaling_factor)\n\n        # Resize the image\n        image_resized = image.resize((new_width, new_height))\n\n        # Save the resized image\n        image_resized.save(output_path)\n        print(f\"Image resized to {new_width}x{new_height} and saved as {output_path}\")\n    else:\n        # Save the original image without resizing\n        image.save(output_path)\n        print(f\"Image is already within the size limits and saved as {output_path}\")\n\ndef generate_image_from_masked(input_image_path, output_image_path):\n    \"\"\"\n    Generate a new image from a masked image using an image-to-image model.\n    \"\"\"\n    # Open and possibly resize the image\n    resized_image_path = '/kaggle/working/temp_resized_image.jpg'\n    resize_image(input_image_path, resized_image_path)\n\n    # Open the resized image\n    img = Image.open(resized_image_path)\n\n    # Get the dimensions of the image\n    width, height = img.size\n\n    # Set up our initial generation parameters.\n    answers = stability_api.generate(\n        prompt=\"bottle with glowing effect holding magical potion, alphonse mucha and simon stalenhag style\",\n        seed = 69696969,\n        init_image=img,  # Assign our previously generated img as our Initial Image for transformation.\n        start_schedule=0.6,  # Set the strength of our prompt in relation to our initial image.\n        steps=30,  # Amount of inference steps performed on image generation. Defaults to 30.\n        cfg_scale=10.0,  # Influences how strongly your generation is guided to match your prompt.\n        width=width,  # Generation width\n        height=height,  # Generation height\n        sampler=generation.SAMPLER_DDIM,  # Sampler type\n        style_preset=\"comic-book\"  # Style preset\n    )\n\n    # Process the response and save the image\n    for resp in answers:\n        for artifact in resp.artifacts:\n            if artifact.finish_reason == generation.FILTER:\n                warnings.warn(\n                    \"Your request activated the API's safety filters and could not be processed.\"\n                    \"Please modify the prompt and try again.\")\n            if artifact.type == generation.ARTIFACT_IMAGE:\n                img2 = Image.open(io.BytesIO(artifact.binary))\n                img2.save(output_image_path)\n                print(f\"Generated image saved as {output_image_path}\")\n\n# Directory paths\nmasked_images_dir = '/kaggle/working/restored_frames'\noutput_gen_dir = '/kaggle/working/mask_gen'\nos.makedirs(output_gen_dir, exist_ok=True)\n\n# Clear the output directory\nclear_output_directory(output_gen_dir)\n\n# Iterate over each masked image and apply image-to-image generation\nfor masked_image_name in os.listdir(masked_images_dir):\n    masked_image_path = os.path.join(masked_images_dir, masked_image_name)\n    output_image_path = os.path.join(output_gen_dir, f\"gen_{masked_image_name}\")\n\n    # Generate new image from the masked image\n    generate_image_from_masked(masked_image_path, output_image_path)\n\n    # Optional: Display the generated image\n    out_img = Image.open(output_image_path)\n    plt.imshow(out_img)\n    plt.title(f\"Generated from {masked_image_name}\")\n    plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/mask_gen'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/mask_gen_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\n\ndef combine_masked_regions(masked_image_path, inverse_masked_image_path, save_path):\n    \"\"\"\n    Combine the original mask areas from the masked image with the inverse-masked image.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image.\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - save_path: String, path where the combined image will be saved.\n    \"\"\"\n    # Open images\n    masked_image = Image.open(masked_image_path).convert(\"RGBA\")\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Ensure images are the same size by resizing the inverse image\n    if masked_image.size != inverse_masked_image.size:\n        inverse_masked_image = inverse_masked_image.resize(masked_image.size)\n\n    # Convert images to numpy arrays\n    masked_array = np.array(masked_image)\n    inverse_masked_array = np.array(inverse_masked_image)\n\n    # Create a mask where the original mask was applied (non-zero areas in any color channel)\n    mask = np.any(masked_array[..., :3] > 30, axis=-1)\n\n    # Replace inverse-masked image values with masked image values where mask is true\n    combined_array = inverse_masked_array.copy()\n    combined_array[mask] = masked_array[mask]\n\n    # Convert back to image\n    combined_image = Image.fromarray(combined_array)\n\n    # Save the combined image\n    combined_image.save(save_path)\n    print(f\"Combined image saved as {save_path}\")\n\n#     # Display the combined image\n#     plt.imshow(combined_image)\n#     plt.axis('off')\n#     plt.show()\n\n# Define directory paths\nmasked_images_dir = \"/kaggle/working/mask_gen\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/Generative_combined_images\"\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Get lists of files in the masked directory\nmasked_images = sorted(os.listdir(masked_images_dir))\n\n# Process files with matching names based on pattern\nfor masked_image_name in masked_images:\n    if masked_image_name.startswith(\"gen_frame_\") and masked_image_name.endswith(\".png\"):\n        # Extract the index number from the masked image name\n        index = masked_image_name[len(\"gen_frame_\"):-len(\".png\")]\n\n        # Generate the corresponding inverse image name\n        inverse_image_name = f\"frame_{index}.png\"\n\n        masked_image_path = os.path.join(masked_images_dir, masked_image_name)\n        inverse_image_path = os.path.join(inverse_images_dir, inverse_image_name)\n        save_path = os.path.join(output_dir, f\"combined_frame_{index}.png\")\n\n        # Check if both files exist before combining\n        if os.path.exists(masked_image_path) and os.path.exists(inverse_image_path):\n            combine_masked_regions(masked_image_path, inverse_image_path, save_path)\n        else:\n            print(f\"Warning: Missing files for frame {index}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### frames to video ","metadata":{}},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/Generative_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/Generative_combined_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## generating using multiple APIs","metadata":{}},{"cell_type":"markdown","source":"### using Multiple keys with better output of image to image generation, the below code can generate ~ 50 frames per 25 credits or 1 free new signup. ","metadata":{}},{"cell_type":"code","source":"import os\nimport io\nimport warnings\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom stability_sdk import client\nimport stability_sdk.interfaces.gooseai.generation.generation_pb2 as generation\n\n# List of API keys\napi_keys = [\n    'sk-3GPp1EOphrXXXXXXXXXX3dmwrbji1iPK3',\n    'sk-6TygJFuBfiQWc7XXXXXXXXXXXqj8aMncmLYrYqpwE1Lv'\n    # Add more API keys here\n]\n\n# Directory paths\nmasked_images_dir = '/kaggle/working/restored_frames'\noutput_gen_dir = '/kaggle/working/HD_mask_gen'\n\nos.makedirs(output_gen_dir, exist_ok=True)\n\ndef initialize_stability_api(api_key):\n    \"\"\"\n    Initialize the Stability API client with the given API key.\n    \"\"\"\n    return client.StabilityInference(\n        key=api_key,  # API Key reference.\n        verbose=True,  # Print debug messages.\n        engine=\"stable-diffusion-xl-1024-v1-0\",  # Set the engine to use for generation.\n    )\n\ndef resize_image(image_path, output_path, max_size=1024):\n    \"\"\"\n    Resize an image if it exceeds the max_size dimension.\n    \"\"\"\n    # Open the image\n    image = Image.open(image_path)\n\n    # Get the current width and height of the image\n    width, height = image.size\n\n    # Calculate the scaling factor\n    if width > height:\n        scaling_factor = max_size / width\n    else:\n        scaling_factor = max_size / height\n\n    # Only resize if the image is larger than the max_size\n    if scaling_factor < 1:\n        # Calculate new dimensions\n        new_width = int(width * scaling_factor)\n        new_height = int(height * scaling_factor)\n\n        # Resize the image\n        image_resized = image.resize((new_width, new_height))\n\n        # Save the resized image\n        image_resized.save(output_path)\n        print(f\"Image resized to {new_width}x{new_height} and saved as {output_path}\")\n    else:\n        # Save the original image without resizing\n        image.save(output_path)\n        print(f\"Image is already within the size limits and saved as {output_path}\")\n\ndef generate_image_from_masked(api, input_image_path, output_image_path):\n    \"\"\"\n    Generate a new image from a masked image using an image-to-image model.\n    \"\"\"\n    # Open and possibly resize the image\n    resized_image_path = '/kaggle/working/temp_resized_image.jpg'\n    resize_image(input_image_path, resized_image_path)\n\n    # Open the resized image\n    img = Image.open(resized_image_path)\n\n    # Get the dimensions of the image\n    width, height = img.size\n\n    # Set up our initial generation parameters.\n    answers = api.generate(\n        prompt=\"soccer ball covered in flames,blazing fireball,eldenring fireball,flames, shiny golden\",\n        init_image=img,  # Assign our previously generated img as our Initial Image for transformation.\n        seed = 69696969,\n        start_schedule=0.6,  # Set the strength of our prompt in relation to our initial image.\n        steps=65,  # Amount of inference steps performed on image generation. Defaults to 30.\n        cfg_scale=10.0,  # Influences how strongly your generation is guided to match your prompt.\n        width=width,  # Generation width\n        height=height,  # Generation height\n        sampler=generation.SAMPLER_K_DPMPP_SDE,  # Sampler type\n        style_preset=\"fantasy-art\"  # Style preset\n    )\n\n    # Process the response and save the image\n    for resp in answers:\n        for artifact in resp.artifacts:\n            if artifact.finish_reason == generation.FILTER:\n                warnings.warn(\n                    \"Your request activated the API's safety filters and could not be processed.\"\n                    \"Please modify the prompt and try again.\")\n            if artifact.type == generation.ARTIFACT_IMAGE:\n                img2 = Image.open(io.BytesIO(artifact.binary))\n                img2.save(output_image_path)\n                print(f\"Generated image saved as {output_image_path}\")\n\n# Initialize the first Stability API client\nstability_api = initialize_stability_api(api_keys[0])\n\n# Iterate over each masked image and apply image-to-image generation\nfor i, masked_image_name in enumerate(os.listdir(masked_images_dir)):\n    # Change API key every 50 frames\n    if i > 0 and i % 50 == 0:\n        api_index = (i // 50) % len(api_keys)  # Calculate the API key index\n        stability_api = initialize_stability_api(api_keys[api_index])\n\n    masked_image_path = os.path.join(masked_images_dir, masked_image_name)\n    output_image_path = os.path.join(output_gen_dir, f\"gen_{masked_image_name}\")\n\n    # Generate new image from the masked image\n    generate_image_from_masked(stability_api, masked_image_path, output_image_path)\n\n    # Optional: Display the generated image\n    out_img = Image.open(output_image_path)\n    plt.imshow(out_img)\n    plt.title(f\"Generated from {masked_image_name}\")\n    plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/HD_mask_gen'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/HD_mask_gen_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\n\ndef combine_masked_regions(masked_image_path, inverse_masked_image_path, save_path):\n    \"\"\"\n    Combine the original mask areas from the masked image with the inverse-masked image.\n\n    Parameters:\n    - masked_image_path: String, path to the masked image.\n    - inverse_masked_image_path: String, path to the inverse-masked image.\n    - save_path: String, path where the combined image will be saved.\n    \"\"\"\n    # Open images\n    masked_image = Image.open(masked_image_path).convert(\"RGBA\")\n    inverse_masked_image = Image.open(inverse_masked_image_path).convert(\"RGBA\")\n\n    # Ensure images are the same size by resizing the inverse image\n    if masked_image.size != inverse_masked_image.size:\n        inverse_masked_image = inverse_masked_image.resize(masked_image.size)\n\n    # Convert images to numpy arrays\n    masked_array = np.array(masked_image)\n    inverse_masked_array = np.array(inverse_masked_image)\n\n    # Create a mask where the original mask was applied (non-zero areas in any color channel)\n    mask = np.any(masked_array[..., :3] > 30, axis=-1)\n\n    # Replace inverse-masked image values with masked image values where mask is true\n    combined_array = inverse_masked_array.copy()\n    combined_array[mask] = masked_array[mask]\n\n    # Convert back to image\n    combined_image = Image.fromarray(combined_array)\n\n    # Save the combined image\n    combined_image.save(save_path)\n    print(f\"Combined image saved as {save_path}\")\n\n#     # Display the combined image\n#     plt.imshow(combined_image)\n#     plt.axis('off')\n#     plt.show()\n\n# Define directory paths\nmasked_images_dir = \"/kaggle/working/HD_mask_gen\"\ninverse_images_dir = \"/kaggle/working/inverse_restored_frames\"\noutput_dir = \"/kaggle/working/HD_Generative_combined_images\"\n\n# Ensure the output directory exists\nos.makedirs(output_dir, exist_ok=True)\n\n# Get lists of files in the masked directory\nmasked_images = sorted(os.listdir(masked_images_dir))\n\n# Process files with matching names based on pattern\nfor masked_image_name in masked_images:\n    if masked_image_name.startswith(\"gen_frame_\") and masked_image_name.endswith(\".png\"):\n        # Extract the index number from the masked image name\n        index = masked_image_name[len(\"gen_frame_\"):-len(\".png\")]\n\n        # Generate the corresponding inverse image name\n        inverse_image_name = f\"frame_{index}.png\"\n\n        masked_image_path = os.path.join(masked_images_dir, masked_image_name)\n        inverse_image_path = os.path.join(inverse_images_dir, inverse_image_name)\n        save_path = os.path.join(output_dir, f\"combined_frame_{index}.png\")\n\n        # Check if both files exist before combining\n        if os.path.exists(masked_image_path) and os.path.exists(inverse_image_path):\n            combine_masked_regions(masked_image_path, inverse_image_path, save_path)\n        else:\n            print(f\"Warning: Missing files for frame {index}. Skipping combination.\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage\nframes_folder = '/kaggle/working/HD_Generative_combined_images'  # Replace with the folder containing your frames\noutput_video_path = \"/kaggle/working/HD_Generative_combined_output_video.mp4\"  # Desired output video file path\n\nframes_to_video(frames_folder, output_video_path, fps=30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## side by side video of original with Img2Img generated video.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport subprocess\nimport shutil\n\n# Directories for the input frames and output combined frames (switched)\ndir1 = '/kaggle/working/output_frames'  # Formerly dir2\ndir2 = '/kaggle/working/HD_Generative_combined_images'  # Formerly dir1\noutput_dir = '/kaggle/working/genai_with_replacement_combined_frames'\nvideo_output = '/kaggle/working/genai_with_replacement_output_video.mp4'\n\n# Ensure the output directory exists and is empty\nif os.path.exists(output_dir):\n    shutil.rmtree(output_dir)  # Remove the directory and its contents\nos.makedirs(output_dir)  # Recreate the empty directory\n\n# Remove the previous video if it exists\nif os.path.exists(video_output):\n    os.remove(video_output)\n\n# Get sorted lists of the frames\nframes1 = sorted([f for f in os.listdir(dir1) if f.endswith('.jpg')])\nframes2 = sorted([f for f in os.listdir(dir2) if f.endswith('.png')])\n\n# Iterate over both directories and combine images\nfor idx, (f1, f2) in enumerate(zip(frames1, frames2), start=1):\n    img1 = Image.open(os.path.join(dir1, f1))\n    img2 = Image.open(os.path.join(dir2, f2))\n    \n    # Resize the larger image to match the height of the smaller one while maintaining the aspect ratio\n    if img1.height > img2.height:\n        img1 = img1.resize((int(img1.width * (img2.height / img1.height)), img2.height), Image.LANCZOS)\n    elif img2.height > img1.height:\n        img2 = img2.resize((int(img2.width * (img1.height / img2.height)), img1.height), Image.LANCZOS)\n    \n    # Combine images side by side\n    combined_img = Image.new('RGB', (img1.width + img2.width, img1.height))\n    combined_img.paste(img1, (0, 0))\n    combined_img.paste(img2, (img1.width, 0))\n    \n    # Save combined image with a sequential name like combined_frame_001.png\n    combined_img.save(os.path.join(output_dir, f\"combined_frame_{idx:03d}.png\"))\n\nprint(f\"Frames combined and saved in {output_dir}\")\n\n# List the files in the output directory to verify they exist\nprint(\"Files in output directory:\", os.listdir(output_dir))\n\n# Convert the combined frames into a video using ffmpeg\nsubprocess.run([\n    'ffmpeg', '-framerate', '30', '-i', \n    f'{output_dir}/combined_frame_%03d.png', '-c:v', \n    'libx264', '-pix_fmt', 'yuv420p', video_output\n])\n\nprint(f\"Video saved as {video_output}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thank you!!!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}