model evaluation added

config/config.yaml

@@ -19,5 +19,13 @@ model_trainer:
   root_dir : artifacts/trained_model
   test_data_path : artifacts/data_transformation/test_dataset.pt
   train_data_path : artifacts/data_transformation/train_dataset.pt
+
+model_evaluation:
+  test_data: 'artifacts/data_transformation/test_dataset.pt'
+  generator_model: 'artifacts/trained_model/cwgan_generator_final.pt'
+  critic_model: 'artifacts/trained_model/cwgan_critic_final.pt'
+  
+
+
   
   

lightning_logs/version_37/hparams.yaml

@@ -0,0 +1,7 @@
+display_step: 10
+in_channels: 1
+lambda_gp: 10
+lambda_r1: 10
+lambda_recon: 100
+learning_rate: 0.0002
+out_channels: 2

lightning_logs/version_38/hparams.yaml

@@ -0,0 +1,7 @@
+display_step: 10
+in_channels: 1
+lambda_gp: 10
+lambda_r1: 10
+lambda_recon: 100
+learning_rate: 0.0002
+out_channels: 2

lightning_logs/version_39/hparams.yaml

@@ -0,0 +1,7 @@
+display_step: 10
+in_channels: 1
+lambda_gp: 10
+lambda_r1: 10
+lambda_recon: 100
+learning_rate: 0.0002
+out_channels: 2

lightning_logs/version_40/hparams.yaml

@@ -0,0 +1,7 @@
+display_step: 10
+in_channels: 1
+lambda_gp: 10
+lambda_r1: 10
+lambda_recon: 100
+learning_rate: 0.0002
+out_channels: 2

main.py

@@ -2,9 +2,10 @@ from src.imagecolorization.pipeline.stage01_data_ingestion import DataIngestionP
 from src.imagecolorization.pipeline.stage02_data_transformation import DataTransformationPipeline
 from src.imagecolorization.pipeline.stage_03_model_building import ModelBuildingPipeline
 from src.imagecolorization.pipeline.stage_04_model_trainer import ModelTrainerPipeline
+from src.imagecolorization.pipeline.stage_05_model_evaluation import ModelEvaluationPipeLine
 from src.imagecolorization.logging import logger
 
-STAGE_NAME = 'Data Ingestion Config'
+STAGE_NAME = 'Data Ingestion Stage'
 
 try:
    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
@@ -15,7 +16,7 @@ except Exception as e:
         logger.exception(e)
         raise e
      
-STAGE_NAME = 'Data Tranasformation Config'
+STAGE_NAME = 'Data Tranasformation Stage'
 
 try:
    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
@@ -27,7 +28,7 @@ except Exception as e:
         raise e
      
      
-STAGE_NAME = 'Model Building Config'
+STAGE_NAME = 'Model Building Stage'
 
 try:
    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
@@ -39,7 +40,7 @@ except Exception as e:
         raise e
      
      
-STAGE_NAME = 'Model Training Config'
+STAGE_NAME = 'Model Training Stage'
 
 try:
    logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
@@ -48,4 +49,18 @@ try:
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
-        raise e
+        raise e
+
+
+
+STAGE_NAME = 'Model Evaluation Stage'
+
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   model_trianer = ModelEvaluationPipeLine()
+   model_trianer.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e
+

params.yaml

@@ -1,7 +1,7 @@
 # Data Parameters
 BATCH_SIZE: 1
 IMAGE_SIZE: [224, 224, 1]
-DATA_RANGE: 5000
+DATA_RANGE: 3000
 
 # Convolutional Layer Parameters
 KERNEL_SIZE_RES: 3
@@ -14,7 +14,7 @@ SCALE_FACTOR: 2
 DIM: 1
 
 # Dropout Parameters
-DROPOUT_RATE: 0.2
+DROPOUT_RATE: 0.3
 
 # Generator Parameters
 KERNEL_SIZE_GENERATOR: 1
@@ -25,7 +25,7 @@ OUTPUT_CHANNELS: 2
 IN_CHANNELS: 3
 
 # model train
-LEARNING_RATE : 2e-4
+LEARNING_RATE : 2e-5
 LAMBDA_RECON : 100
-DISPLAY_STEP : 10
-EPOCH : 1
+DISPLAY_STEP : 11
+EPOCH : 2

research/model_evaluation.ipynb

@@ -0,0 +1,507 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\mlops project\\\\image-colorization-mlops'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class ModelEvalutaionConfig:\n",
+    "    test_data : Path\n",
+    "    generator_model : Path\n",
+    "    critic_model : Path\n",
+    "    all_params: dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.imagecolorization.constants import *\n",
+    "from src.imagecolorization.utils.common import read_yaml, create_directories, save_json\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(self, config_filepath=CONFIG_FILE_PATH, params_filepath=PARAMS_FILE_PATH):\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "        \n",
+    "    def get_model_evaluation_config(self) -> ModelEvalutaionConfig:\n",
+    "        config = self.config.model_evaluation \n",
+    "        params = self.params\n",
+    "\n",
+    "        model_evaluation_config = ModelEvalutaionConfig(\n",
+    "            \n",
+    "            test_data=config.test_data,\n",
+    "            generator_model=config.generator_model,\n",
+    "            critic_model=config.critic_model,\n",
+    "            all_params = params\n",
+    "            \n",
+    "        )\n",
+    "\n",
+    "        return model_evaluation_config\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from torch.utils.data import DataLoader\n",
+    "import mlflow\n",
+    "import dagshub\n",
+    "from tqdm.notebook import tqdm\n",
+    "import json\n",
+    "import os\n",
+    "import logging\n",
+    "from src.imagecolorization.conponents.model_building import Generator, Critic\n",
+    "from src.imagecolorization.conponents.model_trainer import CWGAN\n",
+    "import torch\n",
+    "from torch import nn, optim\n",
+    "from torchvision import transforms\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from torch.autograd import Variable\n",
+    "from torchvision import models\n",
+    "from torch.nn import functional as F\n",
+    "import torch.utils.data\n",
+    "from torchvision.models.inception import inception_v3\n",
+    "from scipy.stats import entropy\n",
+    "import pytorch_lightning as pl\n",
+    "from torchsummary import summary\n",
+    "from src.imagecolorization.conponents.model_building import Generator, Critic\n",
+    "from src.imagecolorization.conponents.data_tranformation import ImageColorizationDataset\n",
+    "from src.imagecolorization.logging import logger\n",
+    "import gc\n",
+    "import numpy as np\n",
+    "\n",
+    "logger = logging.getLogger(__name__)\n",
+    "\n",
+    "import torch\n",
+    "from torch.utils.data import DataLoader\n",
+    "import mlflow\n",
+    "import dagshub\n",
+    "from tqdm.notebook import tqdm\n",
+    "import json\n",
+    "import os\n",
+    "import logging\n",
+    "from torchvision.models.inception import inception_v3\n",
+    "from torch.nn import functional as F\n",
+    "import numpy as np\n",
+    "from torchvision import transforms\n",
+    "\n",
+    "logger = logging.getLogger(__name__)\n",
+    "\n",
+    "class FID:\n",
+    "    def __init__(self, device):\n",
+    "        self.device = device\n",
+    "        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)\n",
+    "        self.inception.eval()\n",
+    "        self.resize = transforms.Resize((299, 299))\n",
+    "\n",
+    "    def convert_to_three_channels(self, images):\n",
+    "        if images.shape[1] == 2:\n",
+    "            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel\n",
+    "        return images\n",
+    "\n",
+    "    def preprocess_images(self, images):\n",
+    "        images = self.convert_to_three_channels(images)\n",
+    "        images = images.to(self.device)\n",
+    "        images = self.resize(images)\n",
+    "        return images\n",
+    "\n",
+    "    def calculate_fid(self, real_images, generated_images):\n",
+    "        batch_size = 32\n",
+    "        real_features_list = []\n",
+    "        generated_features_list = []\n",
+    "\n",
+    "        for i in range(0, len(real_images), batch_size):\n",
+    "            real_batch = self.preprocess_images(real_images[i:i+batch_size])\n",
+    "            generated_batch = self.preprocess_images(generated_images[i:i+batch_size])\n",
+    "\n",
+    "            with torch.no_grad():\n",
+    "                real_features = self.inception(real_batch).view(real_batch.size(0), -1)\n",
+    "                generated_features = self.inception(generated_batch).view(generated_batch.size(0), -1)\n",
+    "\n",
+    "            real_features_list.append(real_features.cpu())\n",
+    "            generated_features_list.append(generated_features.cpu())\n",
+    "\n",
+    "        real_features = torch.cat(real_features_list, dim=0)\n",
+    "        generated_features = torch.cat(generated_features_list, dim=0)\n",
+    "\n",
+    "        mu_diff = real_features.mean(dim=0) - generated_features.mean(dim=0)\n",
+    "        sigma_diff = real_features.std(dim=0) - generated_features.std(dim=0)\n",
+    "\n",
+    "        fid = mu_diff.pow(2).sum() + sigma_diff.pow(2).sum()\n",
+    "        return fid.item()\n",
+    "\n",
+    "class InceptionScore:\n",
+    "    def __init__(self, device):\n",
+    "        self.device = device\n",
+    "        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)\n",
+    "        self.inception.eval()\n",
+    "        self.resize = transforms.Resize((299, 299))\n",
+    "\n",
+    "    def convert_to_three_channels(self, images):\n",
+    "        if images.shape[1] == 2:  # If the input has 2 channels\n",
+    "            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel\n",
+    "        return images\n",
+    "\n",
+    "    def preprocess_images(self, images):\n",
+    "        images = self.convert_to_three_channels(images)\n",
+    "        images = images.to(self.device)\n",
+    "        images = self.resize(images)\n",
+    "        return images\n",
+    "\n",
+    "    def calculate_is(self, images):\n",
+    "        batch_size = 1\n",
+    "        splits = 10\n",
+    "        preds = []\n",
+    "\n",
+    "        for i in range(0, len(images), batch_size):\n",
+    "            batch = self.preprocess_images(images[i:i+batch_size])\n",
+    "            with torch.no_grad():\n",
+    "                pred = F.softmax(self.inception(batch), dim=1)\n",
+    "            preds.append(pred.cpu().numpy())\n",
+    "\n",
+    "        preds = np.concatenate(preds, axis=0)\n",
+    "        n_images = preds.shape[0]\n",
+    "\n",
+    "        split_scores = []\n",
+    "        for k in range(splits):\n",
+    "            part = preds[k * (n_images // splits): (k + 1) * (n_images // splits), :]\n",
+    "            py = np.mean(part, axis=0)\n",
+    "            scores = []\n",
+    "            for i in range(part.shape[0]):\n",
+    "                pyx = part[i, :]\n",
+    "                scores.append(entropy(pyx, py))\n",
+    "            split_scores.append(np.exp(np.mean(scores)))\n",
+    "\n",
+    "        return np.mean(split_scores), np.std(split_scores)\n",
+    "\n",
+    "class ModelEvaluation:\n",
+    "    def __init__(self, config):\n",
+    "        self.config = config\n",
+    "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "        self.generator = None\n",
+    "        self.critic = None\n",
+    "\n",
+    "    def load_model(self):\n",
+    "        self.generator = Generator(input_channel=1, output_channel=2).to(self.device)\n",
+    "        self.critic = Critic(in_channels=3).to(self.device)\n",
+    "\n",
+    "        self.generator.load_state_dict(torch.load(self.config.generator_model))\n",
+    "        self.critic.load_state_dict(torch.load(self.config.critic_model))\n",
+    "\n",
+    "        self.generator.eval()\n",
+    "        self.critic.eval()\n",
+    "\n",
+    "        logger.info(\"Model loaded successfully.\")\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        self.test_dataset = torch.load(self.config.test_data)\n",
+    "        self.test_dataloader = DataLoader(\n",
+    "            self.test_dataset, \n",
+    "            batch_size=self.config.all_params.BATCH_SIZE, \n",
+    "            shuffle=True,\n",
+    "        )\n",
+    "\n",
+    "    def evaluate_model(self):\n",
+    "        is_calculator = InceptionScore(self.device)\n",
+    "        fid_calculator = FID(self.device)\n",
+    "\n",
+    "        all_preds = []\n",
+    "        all_real = []\n",
+    "\n",
+    "        with torch.no_grad():\n",
+    "            for batch in tqdm(self.test_dataloader, desc=\"Evaluating\", unit=\"batch\"):\n",
+    "                real, condition = batch\n",
+    "                real, condition = real.to(self.device), condition.to(self.device)\n",
+    "                fake = self.generator(condition)\n",
+    "                all_preds.append(fake.cpu())\n",
+    "                all_real.append(real.cpu())\n",
+    "\n",
+    "        all_preds = torch.cat(all_preds, dim=0)\n",
+    "        all_real = torch.cat(all_real, dim=0)\n",
+    "\n",
+    "        print(\"Calculating Inception Score for real images...\")\n",
+    "        mean_real_is, std_real_is = is_calculator.calculate_is(all_real)\n",
+    "        print(\"Calculating Inception Score for generated images...\")\n",
+    "        mean_fake_is, std_fake_is = is_calculator.calculate_is(all_preds)\n",
+    "\n",
+    "        print(\"Calculating Fréchet Inception Distance...\")\n",
+    "        fid_value = fid_calculator.calculate_fid(all_real, all_preds)\n",
+    "\n",
+    "        results = {\n",
+    "            \"inception_score_real\": {\"mean\": float(mean_real_is), \"std\": float(std_real_is)},\n",
+    "            \"inception_score_fake\": {\"mean\": float(mean_fake_is), \"std\": float(std_fake_is)},\n",
+    "            \"fid\": float(fid_value)\n",
+    "        }\n",
+    "        return results\n",
+    "\n",
+    "    def save_scores(self, results):\n",
+    "        save_json(path=Path('scores.json'), data=results)\n",
+    "\n",
+    "    def log_to_mlflow(self, results):\n",
+    "        dagshub.init(repo_owner='HAKIM-ML', repo_name='image-colorization-mlops', mlflow=True)\n",
+    "\n",
+    "        with mlflow.start_run():\n",
+    "            # Log all parameters\n",
+    "            for key, value in self.config.all_params.items():\n",
+    "                mlflow.log_param(key, value)\n",
+    "\n",
+    "            # Log metrics\n",
+    "            mlflow.log_metric('inception_score_real_mean', results['inception_score_real']['mean'])\n",
+    "            mlflow.log_metric('inception_score_fake_mean', results['inception_score_fake']['mean'])\n",
+    "            mlflow.log_metric('fid', results['fid'])\n",
+    "\n",
+    "            # Log the JSON file as an artifact\n",
+    "            mlflow.log_artifact('scores.json')\n",
+    "\n",
+    "    def run(self):\n",
+    "        self.load_model()\n",
+    "        self.load_data()\n",
+    "        results = self.evaluate_model()\n",
+    "        self.save_scores(results)\n",
+    "        self.log_to_mlflow(results)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-27 19:55:58,492: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-08-27 19:55:58,497: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-08-27 19:55:58,498: INFO: common: created directory at: artifacts]\n",
+      "[2024-08-27 19:55:59,527: INFO: 1629019639: Model loaded successfully.]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:144: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  self.generator.load_state_dict(torch.load(self.config.generator_model))\n",
+      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:145: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  self.critic.load_state_dict(torch.load(self.config.critic_model))\n",
+      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:153: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  self.test_dataset = torch.load(self.config.test_data)\n",
+      "c:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
+      "  warnings.warn(\n",
+      "c:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=Inception_V3_Weights.IMAGENET1K_V1`. You can also use `weights=Inception_V3_Weights.DEFAULT` to get the most up-to-date weights.\n",
+      "  warnings.warn(msg)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e0c3c8dcf08846e2ad26bd6966770dfa",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Evaluating:   0%|          | 0/5000 [00:00<?, ?batch/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Calculating Inception Score for real images...\n",
+      "Calculating Inception Score for generated images...\n",
+      "Calculating Fréchet Inception Distance...\n",
+      "[2024-08-27 20:03:48,138: INFO: common: Json file saved at: scores.json]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Accessing as HAKIM-ML\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Accessing as HAKIM-ML\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-27 20:03:55,975: INFO: helpers: Accessing as HAKIM-ML]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Initialized MLflow to track repo <span style=\"color: #008000; text-decoration-color: #008000\">\"HAKIM-ML/image-colorization-mlops\"</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Initialized MLflow to track repo \u001b[32m\"HAKIM-ML/image-colorization-mlops\"\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-27 20:04:04,162: INFO: helpers: Initialized MLflow to track repo \"HAKIM-ML/image-colorization-mlops\"]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Repository HAKIM-ML/image-colorization-mlops initialized!\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Repository HAKIM-ML/image-colorization-mlops initialized!\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-27 20:04:04,166: INFO: helpers: Repository HAKIM-ML/image-colorization-mlops initialized!]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024/08/27 20:04:24 INFO mlflow.tracking._tracking_service.client: 🏃 View run grandiose-rat-650 at: https://dagshub.com/HAKIM-ML/image-colorization-mlops.mlflow/#/experiments/0/runs/593e3211953d43359f8810e4d3b21738.\n",
+      "2024/08/27 20:04:24 INFO mlflow.tracking._tracking_service.client: 🧪 View experiment at: https://dagshub.com/HAKIM-ML/image-colorization-mlops.mlflow/#/experiments/0.\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "try:\n",
+    "    config_manager = ConfigurationManager()\n",
+    "    model_evaluation_config = config_manager.get_model_evaluation_config()\n",
+    "    model_evaluation = ModelEvaluation(config=model_evaluation_config)\n",
+    "    model_evaluation.run()\n",
+    "except Exception as e:\n",
+    "    logger.exception(\"An error occurred during model evaluation\")\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scores.json

@@ -0,0 +1,11 @@
+{
+    "inception_score_real": {
+        "mean": 3.2541100236429847,
+        "std": 0.2093519219339589
+    },
+    "inception_score_fake": {
+        "mean": 1.2316137280208515,
+        "std": 0.011668683233882338
+    },
+    "fid": 340.08624267578125
+}

src/imagecolorization/config/configuration.py

@@ -3,7 +3,8 @@ from src.imagecolorization.utils.common import read_yaml, create_directories
 from src.imagecolorization.entity.config_entity import (DataIngestionConfig,
                                                         DataTransformationConfig,
                                                         ModelBuildingConfig,
-                                                        ModelTrainerConfig)
+                                                        ModelTrainerConfig,
+                                                        ModelEvalutaionConfig)
 class ConfigurationManager:
     def __init__(
         self,
@@ -91,6 +92,22 @@ class ConfigurationManager:
             BATCH_SIZE= params.BATCH_SIZE
         )
         return model_trainer_config
+    
+    
+    def get_model_evaluation_config(self) -> ModelEvalutaionConfig:
+        config = self.config.model_evaluation 
+        params = self.params
+
+        model_evaluation_config = ModelEvalutaionConfig(
+            
+            test_data=config.test_data,
+            generator_model=config.generator_model,
+            critic_model=config.critic_model,
+            all_params = params
+            
+        )
+
+        return model_evaluation_config
 
         
     

src/imagecolorization/conponents/model_building.py

@@ -131,8 +131,8 @@ class ModelBuilding:
 
     def get_generator(self):
         return Generator(
-            input_channel=self.config.INPUT_CHANNELS,  # corrected argument name
-            output_channel=self.config.OUTPUT_CHANNELS,  # corrected argument name
+            input_channel=self.config.INPUT_CHANNELS,  
+            output_channel=self.config.OUTPUT_CHANNELS,  
             dropout_rate=self.config.DROPOUT_RATE
         ).to(self.device)
 

src/imagecolorization/conponents/model_evaluation.py

@@ -0,0 +1,221 @@
+import torch
+from torch.utils.data import DataLoader
+import mlflow
+import dagshub
+from tqdm.notebook import tqdm
+import json
+import os
+import logging
+from src.imagecolorization.conponents.model_building import Generator, Critic
+from src.imagecolorization.conponents.model_trainer import CWGAN
+import torch
+from torch import nn, optim
+from torchvision import transforms
+from torch.utils.data import Dataset, DataLoader
+from torch.autograd import Variable
+from torchvision import models
+from torch.nn import functional as F
+import torch.utils.data
+from torchvision.models.inception import inception_v3
+from scipy.stats import entropy
+import pytorch_lightning as pl
+from torchsummary import summary
+from src.imagecolorization.conponents.model_building import Generator, Critic
+from src.imagecolorization.conponents.data_tranformation import ImageColorizationDataset
+from src.imagecolorization.logging import logger
+import gc
+import numpy as np
+from src.imagecolorization.config.configuration import ConfigurationManager
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+import torch
+from torch.utils.data import DataLoader
+import mlflow
+import dagshub
+from tqdm.notebook import tqdm
+import json
+import os
+import logging
+from torchvision.models.inception import inception_v3
+from torch.nn import functional as F
+import numpy as np
+from torchvision import transforms
+from src.imagecolorization.utils.common import save_json
+
+logger = logging.getLogger(__name__)
+
+class FID:
+    def __init__(self, device):
+        self.device = device
+        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)
+        self.inception.eval()
+        self.resize = transforms.Resize((299, 299))
+
+    def convert_to_three_channels(self, images):
+        if images.shape[1] == 2:
+            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel
+        return images
+
+    def preprocess_images(self, images):
+        images = self.convert_to_three_channels(images)
+        images = images.to(self.device)
+        images = self.resize(images)
+        return images
+
+    def calculate_fid(self, real_images, generated_images):
+        batch_size = 32
+        real_features_list = []
+        generated_features_list = []
+
+        for i in range(0, len(real_images), batch_size):
+            real_batch = self.preprocess_images(real_images[i:i+batch_size])
+            generated_batch = self.preprocess_images(generated_images[i:i+batch_size])
+
+            with torch.no_grad():
+                real_features = self.inception(real_batch).view(real_batch.size(0), -1)
+                generated_features = self.inception(generated_batch).view(generated_batch.size(0), -1)
+
+            real_features_list.append(real_features.cpu())
+            generated_features_list.append(generated_features.cpu())
+
+        real_features = torch.cat(real_features_list, dim=0)
+        generated_features = torch.cat(generated_features_list, dim=0)
+
+        mu_diff = real_features.mean(dim=0) - generated_features.mean(dim=0)
+        sigma_diff = real_features.std(dim=0) - generated_features.std(dim=0)
+
+        fid = mu_diff.pow(2).sum() + sigma_diff.pow(2).sum()
+        return fid.item()
+
+class InceptionScore:
+    def __init__(self, device):
+        self.device = device
+        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)
+        self.inception.eval()
+        self.resize = transforms.Resize((299, 299))
+
+    def convert_to_three_channels(self, images):
+        if images.shape[1] == 2:  # If the input has 2 channels
+            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel
+        return images
+
+    def preprocess_images(self, images):
+        images = self.convert_to_three_channels(images)
+        images = images.to(self.device)
+        images = self.resize(images)
+        return images
+
+    def calculate_is(self, images):
+        batch_size = 1
+        splits = 10
+        preds = []
+
+        for i in range(0, len(images), batch_size):
+            batch = self.preprocess_images(images[i:i+batch_size])
+            with torch.no_grad():
+                pred = F.softmax(self.inception(batch), dim=1)
+            preds.append(pred.cpu().numpy())
+
+        preds = np.concatenate(preds, axis=0)
+        n_images = preds.shape[0]
+
+        split_scores = []
+        for k in range(splits):
+            part = preds[k * (n_images // splits): (k + 1) * (n_images // splits), :]
+            py = np.mean(part, axis=0)
+            scores = []
+            for i in range(part.shape[0]):
+                pyx = part[i, :]
+                scores.append(entropy(pyx, py))
+            split_scores.append(np.exp(np.mean(scores)))
+
+        return np.mean(split_scores), np.std(split_scores)
+
+class ModelEvaluation:
+    def __init__(self, config):
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.generator = None
+        self.critic = None
+
+    def load_model(self):
+        self.generator = Generator(input_channel=1, output_channel=2).to(self.device)
+        self.critic = Critic(in_channels=3).to(self.device)
+
+        self.generator.load_state_dict(torch.load(self.config.generator_model))
+        self.critic.load_state_dict(torch.load(self.config.critic_model))
+
+        self.generator.eval()
+        self.critic.eval()
+
+        logger.info("Model loaded successfully.")
+
+    def load_data(self):
+        self.test_dataset = torch.load(self.config.test_data)
+        self.test_dataloader = DataLoader(
+            self.test_dataset, 
+            batch_size=self.config.all_params.BATCH_SIZE, 
+            shuffle=True,
+        )
+
+    def evaluate_model(self):
+        is_calculator = InceptionScore(self.device)
+        fid_calculator = FID(self.device)
+
+        all_preds = []
+        all_real = []
+
+        with torch.no_grad():
+            for batch in tqdm(self.test_dataloader, desc="Evaluating", unit="batch"):
+                real, condition = batch
+                real, condition = real.to(self.device), condition.to(self.device)
+                fake = self.generator(condition)
+                all_preds.append(fake.cpu())
+                all_real.append(real.cpu())
+
+        all_preds = torch.cat(all_preds, dim=0)
+        all_real = torch.cat(all_real, dim=0)
+
+        print("Calculating Inception Score for real images...")
+        mean_real_is, std_real_is = is_calculator.calculate_is(all_real)
+        print("Calculating Inception Score for generated images...")
+        mean_fake_is, std_fake_is = is_calculator.calculate_is(all_preds)
+
+        print("Calculating Fréchet Inception Distance...")
+        fid_value = fid_calculator.calculate_fid(all_real, all_preds)
+
+        results = {
+            "inception_score_real": {"mean": float(mean_real_is), "std": float(std_real_is)},
+            "inception_score_fake": {"mean": float(mean_fake_is), "std": float(std_fake_is)},
+            "fid": float(fid_value)
+        }
+        return results
+
+    def save_scores(self, results):
+        save_json(path=Path('scores.json'), data=results)
+
+    def log_to_mlflow(self, results):
+        dagshub.init(repo_owner='HAKIM-ML', repo_name='image-colorization-mlops', mlflow=True)
+
+        with mlflow.start_run():
+            # Log all parameters
+            for key, value in self.config.all_params.items():
+                mlflow.log_param(key, value)
+
+            # Log metrics
+            mlflow.log_metric('inception_score_real_mean', results['inception_score_real']['mean'])
+            mlflow.log_metric('inception_score_fake_mean', results['inception_score_fake']['mean'])
+            mlflow.log_metric('fid', results['fid'])
+
+            # Log the JSON file as an artifact
+            mlflow.log_artifact('scores.json')
+
+    def run(self):
+        self.load_model()
+        self.load_data()
+        results = self.evaluate_model()
+        self.save_scores(results)
+        self.log_to_mlflow(results)

src/imagecolorization/entity/config_entity.py

@@ -47,4 +47,12 @@ class ModelTrainerConfig:
     INPUT_CHANNELS: int
     OUTPUT_CHANNELS: int
     EPOCH: int
-    BATCH_SIZE : int
+    BATCH_SIZE : int
+    
+    
+@dataclass(frozen=True)
+class ModelEvalutaionConfig:
+    test_data : Path
+    generator_model : Path
+    critic_model : Path
+    all_params: dict

src/imagecolorization/pipeline/stage_05_model_evaluation.py

@@ -0,0 +1,17 @@
+from src.imagecolorization.conponents.model_evaluation import ModelEvaluation
+from src.imagecolorization.config.configuration import ConfigurationManager
+
+
+class ModelEvaluationPipeLine:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config_manager = ConfigurationManager()
+        model_evaluation_config = config_manager.get_model_evaluation_config()
+        model_evaluation = ModelEvaluation(config=model_evaluation_config)
+        model_evaluation.run()
+        
+
+
+