research/model_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\mlops project\\\\image-colorization-mlops'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "from pathlib import Path\n",
    "\n",
    "@dataclass(frozen=True)\n",
    "class ModelEvalutaionConfig:\n",
    "    test_data : Path\n",
    "    generator_model : Path\n",
    "    critic_model : Path\n",
    "    all_params: dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.imagecolorization.constants import *\n",
    "from src.imagecolorization.utils.common import read_yaml, create_directories, save_json\n",
    "\n",
    "class ConfigurationManager:\n",
    "    def __init__(self, config_filepath=CONFIG_FILE_PATH, params_filepath=PARAMS_FILE_PATH):\n",
    "        self.config = read_yaml(config_filepath)\n",
    "        self.params = read_yaml(params_filepath)\n",
    "        create_directories([self.config.artifacts_root])\n",
    "        \n",
    "        \n",
    "    def get_model_evaluation_config(self) -> ModelEvalutaionConfig:\n",
    "        config = self.config.model_evaluation \n",
    "        params = self.params\n",
    "\n",
    "        model_evaluation_config = ModelEvalutaionConfig(\n",
    "            \n",
    "            test_data=config.test_data,\n",
    "            generator_model=config.generator_model,\n",
    "            critic_model=config.critic_model,\n",
    "            all_params = params\n",
    "            \n",
    "        )\n",
    "\n",
    "        return model_evaluation_config\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "import mlflow\n",
    "import dagshub\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "import os\n",
    "import logging\n",
    "from src.imagecolorization.conponents.model_building import Generator, Critic\n",
    "from src.imagecolorization.conponents.model_trainer import CWGAN\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.autograd import Variable\n",
    "from torchvision import models\n",
    "from torch.nn import functional as F\n",
    "import torch.utils.data\n",
    "from torchvision.models.inception import inception_v3\n",
    "from scipy.stats import entropy\n",
    "import pytorch_lightning as pl\n",
    "from torchsummary import summary\n",
    "from src.imagecolorization.conponents.model_building import Generator, Critic\n",
    "from src.imagecolorization.conponents.data_tranformation import ImageColorizationDataset\n",
    "from src.imagecolorization.logging import logger\n",
    "import gc\n",
    "import numpy as np\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "import mlflow\n",
    "import dagshub\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "import os\n",
    "import logging\n",
    "from torchvision.models.inception import inception_v3\n",
    "from torch.nn import functional as F\n",
    "import numpy as np\n",
    "from torchvision import transforms\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "class FID:\n",
    "    def __init__(self, device):\n",
    "        self.device = device\n",
    "        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)\n",
    "        self.inception.eval()\n",
    "        self.resize = transforms.Resize((299, 299))\n",
    "\n",
    "    def convert_to_three_channels(self, images):\n",
    "        if images.shape[1] == 2:\n",
    "            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel\n",
    "        return images\n",
    "\n",
    "    def preprocess_images(self, images):\n",
    "        images = self.convert_to_three_channels(images)\n",
    "        images = images.to(self.device)\n",
    "        images = self.resize(images)\n",
    "        return images\n",
    "\n",
    "    def calculate_fid(self, real_images, generated_images):\n",
    "        batch_size = 32\n",
    "        real_features_list = []\n",
    "        generated_features_list = []\n",
    "\n",
    "        for i in range(0, len(real_images), batch_size):\n",
    "            real_batch = self.preprocess_images(real_images[i:i+batch_size])\n",
    "            generated_batch = self.preprocess_images(generated_images[i:i+batch_size])\n",
    "\n",
    "            with torch.no_grad():\n",
    "                real_features = self.inception(real_batch).view(real_batch.size(0), -1)\n",
    "                generated_features = self.inception(generated_batch).view(generated_batch.size(0), -1)\n",
    "\n",
    "            real_features_list.append(real_features.cpu())\n",
    "            generated_features_list.append(generated_features.cpu())\n",
    "\n",
    "        real_features = torch.cat(real_features_list, dim=0)\n",
    "        generated_features = torch.cat(generated_features_list, dim=0)\n",
    "\n",
    "        mu_diff = real_features.mean(dim=0) - generated_features.mean(dim=0)\n",
    "        sigma_diff = real_features.std(dim=0) - generated_features.std(dim=0)\n",
    "\n",
    "        fid = mu_diff.pow(2).sum() + sigma_diff.pow(2).sum()\n",
    "        return fid.item()\n",
    "\n",
    "class InceptionScore:\n",
    "    def __init__(self, device):\n",
    "        self.device = device\n",
    "        self.inception = inception_v3(pretrained=True, transform_input=False).to(self.device)\n",
    "        self.inception.eval()\n",
    "        self.resize = transforms.Resize((299, 299))\n",
    "\n",
    "    def convert_to_three_channels(self, images):\n",
    "        if images.shape[1] == 2:  # If the input has 2 channels\n",
    "            images = torch.cat((images, images[:, :1, :, :]), dim=1)  # Duplicate one channel\n",
    "        return images\n",
    "\n",
    "    def preprocess_images(self, images):\n",
    "        images = self.convert_to_three_channels(images)\n",
    "        images = images.to(self.device)\n",
    "        images = self.resize(images)\n",
    "        return images\n",
    "\n",
    "    def calculate_is(self, images):\n",
    "        batch_size = 1\n",
    "        splits = 10\n",
    "        preds = []\n",
    "\n",
    "        for i in range(0, len(images), batch_size):\n",
    "            batch = self.preprocess_images(images[i:i+batch_size])\n",
    "            with torch.no_grad():\n",
    "                pred = F.softmax(self.inception(batch), dim=1)\n",
    "            preds.append(pred.cpu().numpy())\n",
    "\n",
    "        preds = np.concatenate(preds, axis=0)\n",
    "        n_images = preds.shape[0]\n",
    "\n",
    "        split_scores = []\n",
    "        for k in range(splits):\n",
    "            part = preds[k * (n_images // splits): (k + 1) * (n_images // splits), :]\n",
    "            py = np.mean(part, axis=0)\n",
    "            scores = []\n",
    "            for i in range(part.shape[0]):\n",
    "                pyx = part[i, :]\n",
    "                scores.append(entropy(pyx, py))\n",
    "            split_scores.append(np.exp(np.mean(scores)))\n",
    "\n",
    "        return np.mean(split_scores), np.std(split_scores)\n",
    "\n",
    "class ModelEvaluation:\n",
    "    def __init__(self, config):\n",
    "        self.config = config\n",
    "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        self.generator = None\n",
    "        self.critic = None\n",
    "\n",
    "    def load_model(self):\n",
    "        self.generator = Generator(input_channel=1, output_channel=2).to(self.device)\n",
    "        self.critic = Critic(in_channels=3).to(self.device)\n",
    "\n",
    "        self.generator.load_state_dict(torch.load(self.config.generator_model))\n",
    "        self.critic.load_state_dict(torch.load(self.config.critic_model))\n",
    "\n",
    "        self.generator.eval()\n",
    "        self.critic.eval()\n",
    "\n",
    "        logger.info(\"Model loaded successfully.\")\n",
    "\n",
    "    def load_data(self):\n",
    "        self.test_dataset = torch.load(self.config.test_data)\n",
    "        self.test_dataloader = DataLoader(\n",
    "            self.test_dataset, \n",
    "            batch_size=self.config.all_params.BATCH_SIZE, \n",
    "            shuffle=True,\n",
    "        )\n",
    "\n",
    "    def evaluate_model(self):\n",
    "        is_calculator = InceptionScore(self.device)\n",
    "        fid_calculator = FID(self.device)\n",
    "\n",
    "        all_preds = []\n",
    "        all_real = []\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for batch in tqdm(self.test_dataloader, desc=\"Evaluating\", unit=\"batch\"):\n",
    "                real, condition = batch\n",
    "                real, condition = real.to(self.device), condition.to(self.device)\n",
    "                fake = self.generator(condition)\n",
    "                all_preds.append(fake.cpu())\n",
    "                all_real.append(real.cpu())\n",
    "\n",
    "        all_preds = torch.cat(all_preds, dim=0)\n",
    "        all_real = torch.cat(all_real, dim=0)\n",
    "\n",
    "        print(\"Calculating Inception Score for real images...\")\n",
    "        mean_real_is, std_real_is = is_calculator.calculate_is(all_real)\n",
    "        print(\"Calculating Inception Score for generated images...\")\n",
    "        mean_fake_is, std_fake_is = is_calculator.calculate_is(all_preds)\n",
    "\n",
    "        print(\"Calculating Fréchet Inception Distance...\")\n",
    "        fid_value = fid_calculator.calculate_fid(all_real, all_preds)\n",
    "\n",
    "        results = {\n",
    "            \"inception_score_real\": {\"mean\": float(mean_real_is), \"std\": float(std_real_is)},\n",
    "            \"inception_score_fake\": {\"mean\": float(mean_fake_is), \"std\": float(std_fake_is)},\n",
    "            \"fid\": float(fid_value)\n",
    "        }\n",
    "        return results\n",
    "\n",
    "    def save_scores(self, results):\n",
    "        save_json(path=Path('scores.json'), data=results)\n",
    "\n",
    "    def log_to_mlflow(self, results):\n",
    "        dagshub.init(repo_owner='HAKIM-ML', repo_name='image-colorization-mlops', mlflow=True)\n",
    "\n",
    "        with mlflow.start_run():\n",
    "            # Log all parameters\n",
    "            for key, value in self.config.all_params.items():\n",
    "                mlflow.log_param(key, value)\n",
    "\n",
    "            # Log metrics\n",
    "            mlflow.log_metric('inception_score_real_mean', results['inception_score_real']['mean'])\n",
    "            mlflow.log_metric('inception_score_fake_mean', results['inception_score_fake']['mean'])\n",
    "            mlflow.log_metric('fid', results['fid'])\n",
    "\n",
    "            # Log the JSON file as an artifact\n",
    "            mlflow.log_artifact('scores.json')\n",
    "\n",
    "    def run(self):\n",
    "        self.load_model()\n",
    "        self.load_data()\n",
    "        results = self.evaluate_model()\n",
    "        self.save_scores(results)\n",
    "        self.log_to_mlflow(results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-08-27 19:55:58,492: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
      "[2024-08-27 19:55:58,497: INFO: common: yaml file: params.yaml loaded successfully]\n",
      "[2024-08-27 19:55:58,498: INFO: common: created directory at: artifacts]\n",
      "[2024-08-27 19:55:59,527: INFO: 1629019639: Model loaded successfully.]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:144: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  self.generator.load_state_dict(torch.load(self.config.generator_model))\n",
      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:145: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  self.critic.load_state_dict(torch.load(self.config.critic_model))\n",
      "C:\\Users\\azizu\\AppData\\Local\\Temp\\ipykernel_54388\\1629019639.py:153: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  self.test_dataset = torch.load(self.config.test_data)\n",
      "c:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "c:\\Users\\azizu\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=Inception_V3_Weights.IMAGENET1K_V1`. You can also use `weights=Inception_V3_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0c3c8dcf08846e2ad26bd6966770dfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/5000 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating Inception Score for real images...\n",
      "Calculating Inception Score for generated images...\n",
      "Calculating Fréchet Inception Distance...\n",
      "[2024-08-27 20:03:48,138: INFO: common: Json file saved at: scores.json]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Accessing as HAKIM-ML\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Accessing as HAKIM-ML\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-08-27 20:03:55,975: INFO: helpers: Accessing as HAKIM-ML]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Initialized MLflow to track repo <span style=\"color: #008000; text-decoration-color: #008000\">\"HAKIM-ML/image-colorization-mlops\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Initialized MLflow to track repo \u001b[32m\"HAKIM-ML/image-colorization-mlops\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-08-27 20:04:04,162: INFO: helpers: Initialized MLflow to track repo \"HAKIM-ML/image-colorization-mlops\"]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Repository HAKIM-ML/image-colorization-mlops initialized!\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Repository HAKIM-ML/image-colorization-mlops initialized!\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-08-27 20:04:04,166: INFO: helpers: Repository HAKIM-ML/image-colorization-mlops initialized!]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/08/27 20:04:24 INFO mlflow.tracking._tracking_service.client: 🏃 View run grandiose-rat-650 at: https://dagshub.com/HAKIM-ML/image-colorization-mlops.mlflow/#/experiments/0/runs/593e3211953d43359f8810e4d3b21738.\n",
      "2024/08/27 20:04:24 INFO mlflow.tracking._tracking_service.client: 🧪 View experiment at: https://dagshub.com/HAKIM-ML/image-colorization-mlops.mlflow/#/experiments/0.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "try:\n",
    "    config_manager = ConfigurationManager()\n",
    "    model_evaluation_config = config_manager.get_model_evaluation_config()\n",
    "    model_evaluation = ModelEvaluation(config=model_evaluation_config)\n",
    "    model_evaluation.run()\n",
    "except Exception as e:\n",
    "    logger.exception(\"An error occurred during model evaluation\")\n",
    "    raise e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}