data transformation added

config/config.yaml

@@ -4,4 +4,12 @@ data_ingestion:
   root_dir: artifacts/data_ingestion
   source_dir: C:\\mlops project\\archive.zip
   local_data_file : artifacts/data_ingestion/archive.zip
-  unzip_dir : artifacts/data_ingestion
+  unzip_dir : artifacts/data_ingestion
+
+
+data_transformation:
+  root_dir : artifacts/data_transformation
+  data_path_black : artifacts/data_ingestion/ab/ab/ab1.npy
+  data_path_grey : artifacts/data_ingestion/l/gray_scale.npy
+
+  

main.py

@@ -1,4 +1,5 @@
 from src.imagecolorization.pipeline.stage01_data_ingestion import DataIngestionPipeline
+from src.imagecolorization.pipeline.stage02_data_transformation import DataTransformationPipeline
 from src.imagecolorization.logging import logger
 
 STAGE_NAME = 'Data Ingestion Config'
@@ -8,6 +9,17 @@ try:
    data_ingestion = DataIngestionPipeline()
    data_ingestion.main()
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e
+     
+STAGE_NAME = 'Data Tranasformation Config'
+
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   data_transformation = DataTransformationPipeline()
+   data_transformation.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
         raise e

params.yaml

@@ -1 +1,5 @@
-key: val
+BATCH_SIZE : 1
+IMAGE_SIZE : [224,224,1]
+DATA_RANGE: 5000
+KERNEL_SIZE : 3
+p

research/data_transformation.ipynb

@@ -0,0 +1,287 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\mlops project\\\\image-colorization-mlops'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataTransformationConfig:\n",
+    "    root_dir : Path\n",
+    "    data_path_black : Path\n",
+    "    data_path_grey : Path\n",
+    "    BATCH_SIZE : int\n",
+    "    IMAGE_SIZE : list\n",
+    "    DATA_RANGE: int\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.imagecolorization.constants import *\n",
+    "from src.imagecolorization.utils.common import read_yaml, create_directories\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "        \n",
+    "    def get_data_transformation_config(self) -> DataTransformationConfig:\n",
+    "        config = self.config.data_transformation\n",
+    "        params = self.params\n",
+    "        \n",
+    "        data_transformation_cofig = DataTransformationConfig(\n",
+    "            root_dir= config.root_dir,\n",
+    "            data_path_black=config.data_path_black,\n",
+    "            data_path_grey=config.data_path_grey,\n",
+    "            BATCH_SIZE=params.BATCH_SIZE,\n",
+    "            IMAGE_SIZE=params.IMAGE_SIZE,\n",
+    "            DATA_RANGE=params.DATA_RANGE\n",
+    "        )\n",
+    "        \n",
+    "        return data_transformation_cofig"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch.utils.data import DataLoader, Dataset\n",
+    "from torchvision import transforms\n",
+    "\n",
+    "class ImageColorizationDataset(Dataset):\n",
+    "    def __init__(self, dataset, image_size, transform=None):\n",
+    "        self.dataset = dataset\n",
+    "        self.transform = transform\n",
+    "        self.image_size = tuple(image_size)  \n",
+    "    def __len__(self):\n",
+    "        return len(self.dataset[0])\n",
+    "    \n",
+    "    def __getitem__(self, idx):\n",
+    "        L = np.array(self.dataset[0][idx]).reshape(self.image_size + (1,))\n",
+    "        L = transforms.ToTensor()(L)\n",
+    "        \n",
+    "        ab = np.array(self.dataset[1][idx])\n",
+    "        ab = transforms.ToTensor()(ab)\n",
+    "        \n",
+    "        return ab, L\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch.utils.data import DataLoader\n",
+    "import gc\n",
+    "import os\n",
+    "from src.imagecolorization.logging import logger\n",
+    "\n",
+    "class DataTransformation:\n",
+    "    def __init__(self, config: DataTransformationConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "    def load_data(self):\n",
+    "        ab_df = np.load(self.config.data_path_black)[:self.config.DATA_RANGE]\n",
+    "        l_df = np.load(self.config.data_path_grey)[:self.config.DATA_RANGE]\n",
+    "        dataset = (l_df, ab_df)\n",
+    "        gc.collect()\n",
+    "        return dataset\n",
+    "    \n",
+    "    \n",
+    "    def get_dataloader(self, dataset):\n",
+    "        train_dataset = ImageColorizationDataset(\n",
+    "            dataset=dataset,\n",
+    "            image_size=self.config.IMAGE_SIZE\n",
+    "        )\n",
+    "        test_dataset = ImageColorizationDataset(\n",
+    "            dataset=dataset,\n",
+    "            image_size=self.config.IMAGE_SIZE\n",
+    "        )\n",
+    "        \n",
+    "        train_loader = DataLoader(\n",
+    "            train_dataset,\n",
+    "            batch_size=self.config.BATCH_SIZE,\n",
+    "            shuffle=True,\n",
+    "            pin_memory=True\n",
+    "        )\n",
+    "        test_loader = DataLoader(\n",
+    "            test_dataset,\n",
+    "            batch_size=self.config.BATCH_SIZE,\n",
+    "            shuffle=True,\n",
+    "            pin_memory=True\n",
+    "        )\n",
+    "        \n",
+    "        return train_loader, test_loader\n",
+    "    \n",
+    "    \n",
+    "    def save_dataloaders(self, train_loader, test_loader):\n",
+    "        train_loader_path = os.path.join(self.config.root_dir, 'train_loader.pt')\n",
+    "        test_loader_path = os.path.join(self.config.root_dir, 'test_loader.pt')\n",
+    "    \n",
+    "        torch.save(train_loader, train_loader_path)\n",
+    "        torch.save(test_loader, test_loader_path)\n",
+    "    \n",
+    "        logger.info(f\"Train Loader saved at: {train_loader_path}\")\n",
+    "        logger.info(f\"Test Loader saved at: {test_loader_path}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-18 13:20:34,232: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-08-18 13:20:34,234: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-08-18 13:20:34,235: INFO: common: created directory at: artifacts]\n"
+     ]
+    },
+    {
+     "ename": "BoxKeyError",
+     "evalue": "\"'ConfigBox' object has no attribute 'data_transformation'\"",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:503\u001b[0m, in \u001b[0;36mBox.__getitem__\u001b[1;34m(self, item, _ignore_default)\u001b[0m\n\u001b[0;32m    502\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 503\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getitem__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    504\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mKeyError\u001b[0m: 'data_transformation'",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m                               Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:536\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    535\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 536\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getitem__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_ignore_default\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    537\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:524\u001b[0m, in \u001b[0;36mBox.__getitem__\u001b[1;34m(self, item, _ignore_default)\u001b[0m\n\u001b[0;32m    523\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_default(item)\n\u001b[1;32m--> 524\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BoxKeyError(\u001b[38;5;28mstr\u001b[39m(err)) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m_exception_cause\u001b[39;00m(err)\n\u001b[0;32m    525\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m: \"'data_transformation'\"",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:538\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    537\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m--> 538\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mobject\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getattribute__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    539\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mAttributeError\u001b[0m: 'ConfigBox' object has no attribute 'data_transformation'",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m                               Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\config_box.py:28\u001b[0m, in \u001b[0;36mConfigBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 28\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getattr__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m:\n",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:552\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    551\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_default(item, attr\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m--> 552\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BoxKeyError(\u001b[38;5;28mstr\u001b[39m(err)) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m_exception_cause\u001b[39;00m(err)\n\u001b[0;32m    553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m value\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m: \"'ConfigBox' object has no attribute 'data_transformation'\"",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:503\u001b[0m, in \u001b[0;36mBox.__getitem__\u001b[1;34m(self, item, _ignore_default)\u001b[0m\n\u001b[0;32m    502\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 503\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getitem__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    504\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mKeyError\u001b[0m: 'data_transformation'",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m                               Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:536\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    535\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 536\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getitem__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_ignore_default\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    537\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:524\u001b[0m, in \u001b[0;36mBox.__getitem__\u001b[1;34m(self, item, _ignore_default)\u001b[0m\n\u001b[0;32m    523\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_default(item)\n\u001b[1;32m--> 524\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BoxKeyError(\u001b[38;5;28mstr\u001b[39m(err)) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m_exception_cause\u001b[39;00m(err)\n\u001b[0;32m    525\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m: \"'data_transformation'\"",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:538\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    537\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m--> 538\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mobject\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getattribute__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    539\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
+      "\u001b[1;31mAttributeError\u001b[0m: 'ConfigBox' object has no attribute 'data_transformation'",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m                               Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[7], line 9\u001b[0m\n\u001b[0;32m      7\u001b[0m     data_transformation\u001b[38;5;241m.\u001b[39msa\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m----> 9\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
+      "Cell \u001b[1;32mIn[7], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m      2\u001b[0m     config \u001b[38;5;241m=\u001b[39m ConfigurationManager()\n\u001b[1;32m----> 3\u001b[0m     data_transformation_config \u001b[38;5;241m=\u001b[39m \u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_data_transformation_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      4\u001b[0m     data_transformation \u001b[38;5;241m=\u001b[39m DataTransformation(config\u001b[38;5;241m=\u001b[39mdata_transformation_config)\n\u001b[0;32m      5\u001b[0m     data_transformation\u001b[38;5;241m.\u001b[39mload_data()\n",
+      "Cell \u001b[1;32mIn[4], line 17\u001b[0m, in \u001b[0;36mConfigurationManager.get_data_transformation_config\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_data_transformation_config\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataTransformationConfig:\n\u001b[1;32m---> 17\u001b[0m     config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata_transformation\u001b[49m\n\u001b[0;32m     18\u001b[0m     params \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparams\n\u001b[0;32m     20\u001b[0m     data_transformation_cofig \u001b[38;5;241m=\u001b[39m DataTransformationConfig(\n\u001b[0;32m     21\u001b[0m         root_dir\u001b[38;5;241m=\u001b[39m config\u001b[38;5;241m.\u001b[39mroot_dir,\n\u001b[0;32m     22\u001b[0m         data_path_black\u001b[38;5;241m=\u001b[39mconfig\u001b[38;5;241m.\u001b[39mdata_path_black,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     26\u001b[0m         DATA_RANGE\u001b[38;5;241m=\u001b[39mparams\u001b[38;5;241m.\u001b[39mDATA_RANGE\n\u001b[0;32m     27\u001b[0m     )\n",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\config_box.py:30\u001b[0m, in \u001b[0;36mConfigBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__getattr__\u001b[39m(item)\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m:\n\u001b[1;32m---> 30\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__getattr__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlower\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32mc:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\box\\box.py:552\u001b[0m, in \u001b[0;36mBox.__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m    550\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m BoxKeyError(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mitem\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: Does not exist and internal methods are never defaulted\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    551\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_default(item, attr\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m--> 552\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BoxKeyError(\u001b[38;5;28mstr\u001b[39m(err)) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m_exception_cause\u001b[39;00m(err)\n\u001b[0;32m    553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m value\n",
+      "\u001b[1;31mBoxKeyError\u001b[0m: \"'ConfigBox' object has no attribute 'data_transformation'\""
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_transformation_config = config.get_data_transformation_config()\n",
+    "    data_transformation = DataTransformation(config=data_transformation_config)\n",
+    "    data_transformation.load_data()\n",
+    "    data_transformation.get_dataloader()\n",
+    "    data_transformation.sa\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/imagecolorization/config/configuration.py

@@ -1,6 +1,7 @@
 from src.imagecolorization.constants import *
 from src.imagecolorization.utils.common import read_yaml, create_directories
-from src.imagecolorization.entity.config_entity import DataIngestionConfig
+from src.imagecolorization.entity.config_entity import (DataIngestionConfig,
+                                                        DataTransformationConfig)
 class ConfigurationManager:
     def __init__(
         self,
@@ -27,6 +28,22 @@ class ConfigurationManager:
         )
 
         return data_ingestion_config
+    
+    
+    def get_data_transformation_config(self) -> DataTransformationConfig:
+        config = self.config.data_transformation
+        params = self.params
+        
+        data_transformation_cofig = DataTransformationConfig(
+            root_dir= config.root_dir,
+            data_path_black=config.data_path_black,
+            data_path_grey=config.data_path_grey,
+            BATCH_SIZE=params.BATCH_SIZE,
+            IMAGE_SIZE=params.IMAGE_SIZE,
+            DATA_RANGE=params.DATA_RANGE
+        )
+        
+        return data_transformation_cofig
         
     
     

src/imagecolorization/conponents/data_tranformation.py

@@ -0,0 +1,90 @@
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+from torch.utils.data import DataLoader
+import gc
+import os
+from src.imagecolorization.logging import logger
+from src.imagecolorization.entity.config_entity import DataTransformationConfig
+
+
+class ImageColorizationDataset(Dataset):
+    def __init__(self, dataset, image_size, transform=None):
+        self.dataset = dataset
+        self.transform = transform
+        self.image_size = tuple(image_size)  
+    def __len__(self):
+        return len(self.dataset[0])
+    
+    def __getitem__(self, idx):
+        L = np.array(self.dataset[0][idx]).reshape(self.image_size + (1,))
+        L = transforms.ToTensor()(L)
+        
+        ab = np.array(self.dataset[1][idx])
+        ab = transforms.ToTensor()(ab)
+        
+        return ab, L
+    
+    
+class DataTransformation:
+    def __init__(self, config: DataTransformationConfig):
+        self.config = config
+        
+    def load_data(self):
+        ab_df = np.load(self.config.data_path_black)[:self.config.DATA_RANGE]
+        l_df = np.load(self.config.data_path_grey)[:self.config.DATA_RANGE]
+        dataset = (l_df, ab_df)
+        gc.collect()
+        return dataset
+    
+    
+    def get_dataloader(self, dataset):
+        train_dataset = ImageColorizationDataset(
+            dataset=dataset,
+            image_size=self.config.IMAGE_SIZE
+        )
+        test_dataset = ImageColorizationDataset(
+            dataset=dataset,
+            image_size=self.config.IMAGE_SIZE
+        )
+        
+        train_loader = DataLoader(
+            train_dataset,
+            batch_size=self.config.BATCH_SIZE,
+            shuffle=True,
+            pin_memory=True
+        )
+        test_loader = DataLoader(
+            test_dataset,
+            batch_size=self.config.BATCH_SIZE,
+            shuffle=True,
+            pin_memory=True
+        )
+        
+        return train_loader, test_loader
+    
+    
+    
+        
+        
+    
+    
+    def save_dataloaders(self, train_loader, test_loader):
+        # Ensure the directory exists
+        os.makedirs(self.config.root_dir, exist_ok=True)
+
+        train_loader_path = os.path.join(self.config.root_dir, 'train_loader.pt')
+        test_loader_path = os.path.join(self.config.root_dir, 'test_loader.pt')
+
+        try:
+            # Save the dataloaders
+            torch.save(train_loader, train_loader_path)
+            torch.save(test_loader, test_loader_path)
+
+            logger.info(f"Train Loader saved at: {train_loader_path}")
+            logger.info(f"Test Loader saved at: {test_loader_path}")
+        except Exception as e:
+            logger.error(f"Error saving dataloaders: {str(e)}")
+            raise e
+    

src/imagecolorization/entity/config_entity.py

@@ -6,4 +6,14 @@ class DataIngestionConfig:
     root_dir : Path
     source_dir : Path
     local_data_file: Path
-    unzip_dir : Path
+    unzip_dir : Path
+    
+    
+@dataclass(frozen=True)
+class DataTransformationConfig:
+    root_dir : Path
+    data_path_black : Path
+    data_path_grey : Path
+    BATCH_SIZE : int
+    IMAGE_SIZE : list
+    DATA_RANGE: int

src/imagecolorization/pipeline/stage02_data_transformation.py

@@ -0,0 +1,15 @@
+from src.imagecolorization.conponents.data_tranformation import DataTransformation
+from src.imagecolorization.config.configuration import ConfigurationManager
+
+
+class DataTransformationPipeline:
+    def __init__(sefl):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        data_transformation_config = config.get_data_transformation_config()
+        data_transformation = DataTransformation(config=data_transformation_config)
+        dataset = data_transformation.load_data()
+        train_loader, test_loader = data_transformation.get_dataloader(dataset)
+        data_transformation.save_dataloaders(train_loader, test_loader)