data validation added

config/config.yaml

@@ -5,4 +5,10 @@ data_ingestion:
   root_dir: artifacts/data_ingestion
   source_URL: https://github.com/entbappy/Branching-tutorial/raw/master/summarizer-data.zip
   local_data_file: artifacts/data_ingestion/data.zip
-  unzip_dir: artifacts/data_ingestion
+  unzip_dir: artifacts/data_ingestion
+
+
+data_validation:
+  root_dir: artifacts/data_validation
+  STATUS_FILE: artifacts/data_validation/status.txt
+  ALL_REQUIRED_FILES: ["train", "test", "validation"]

main.py

@@ -1,4 +1,5 @@
 from textsummarizer.pipeline.stage_01_data_ingestion import DataIngestionPipeline
+from textsummarizer.pipeline.stage_02_data_validation import DataValidationPipeline
 from textsummarizer.logging import logger
 
 STAGE_NAME = "Data Ingestion stage"
@@ -7,6 +8,17 @@ try:
    data_ingestion = DataIngestionPipeline()
    data_ingestion.main()
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e
+     
+     
+STAGE_NAME = "Data Validation stage"
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   data_ingestion = DataValidationPipeline()
+   data_ingestion.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
         raise e

research/data_ingestoin.ipynb

@@ -0,0 +1,200 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\mlops projects\\\\text-summarization'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "@dataclass(frozen=True)\n",
+    "class DataIngestionConfig:\n",
+    "    root_dir : Path\n",
+    "    source_URL : str\n",
+    "    local_data_file : Path\n",
+    "    unzip_dir : Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textsummarizer.constants import *\n",
+    "from textsummarizer.utils.common import read_yaml, create_directories\n",
+    "\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "\n",
+    "    \n",
+    "\n",
+    "    def get_data_ingestion_config(self) -> DataIngestionConfig:\n",
+    "        config = self.config.data_ingestion\n",
+    "\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        data_ingestion_config = DataIngestionConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            source_URL=config.source_URL,\n",
+    "            local_data_file=config.local_data_file,\n",
+    "            unzip_dir=config.unzip_dir \n",
+    "        )\n",
+    "\n",
+    "        return data_ingestion_config\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import urllib.request as request\n",
+    "import zipfile\n",
+    "from textsummarizer.logging import logger\n",
+    "from textsummarizer.utils.common import get_size"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DataIngestion:\n",
+    "    def __init__(self, config: DataIngestionConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "        \n",
+    "    def download_file(self):\n",
+    "        if not os.path.exists(self.config.local_data_file):\n",
+    "            filename, header = request.urlretrieve(\n",
+    "                url=self.config.source_URL,\n",
+    "                filename = self.config.local_data_file\n",
+    "            )\n",
+    "            logger.info(f'{filename} download! with following info: \\n{header}')\n",
+    "            \n",
+    "        else:\n",
+    "            logger.info(f\"File already exist size {get_size(Path(self.config.local_data_file))}\")\n",
+    "            \n",
+    "            \n",
+    "        \n",
+    "            \n",
+    "            \n",
+    "    def extract_zip_file(self):\n",
+    "        \"\"\"\n",
+    "        zip_file_path: str\n",
+    "        Extracts the zip file into the data directory\n",
+    "        Function returns None\n",
+    "        \"\"\"\n",
+    "        unzip_path = self.config.unzip_dir\n",
+    "        os.makedirs(unzip_path, exist_ok=True)\n",
+    "        with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:\n",
+    "            zip_ref.extractall(unzip_path)\n",
+    "            \n",
+    "            \n",
+    "                    \n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-08-11 15:50:51,008: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-08-11 15:50:51,011: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-08-11 15:50:51,012: INFO: common: created directory at: artifacts]\n",
+      "[2024-08-11 15:50:51,014: INFO: common: created directory at: artifacts/data_ingestion]\n",
+      "[2024-08-11 15:50:51,016: INFO: 4172299431: File already exist size ~ 7718 KB]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_ingestion_config = config.get_data_ingestion_config()\n",
+    "    data_ingestion = DataIngestion(config=data_ingestion_config)\n",
+    "    data_ingestion.download_file()\n",
+    "    data_ingestion.extract_zip_file()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

research/data_validation.ipynb

@@ -0,0 +1,70 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'c:\\\\mlops projects\\\\text-summarization'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "@dataclass(frozen=True)\n",
+    "class DataValidationConfig:\n",
+    "    root_dir : Path\n",
+    "    STATUS_FILE : Path\n",
+    "    ALL_REQUIRED_FILES : "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/textsummarizer/config/configuration.py

@@ -1,6 +1,7 @@
 from textsummarizer.constants import *
 from textsummarizer.utils.common import read_yaml, create_directories
-from textsummarizer.entity.config_entity import DataIngestionConfig
+from textsummarizer.entity.config_entity import (DataIngestionConfig,
+                                                 DataValidationConfig)
 
 class ConfigurationManager:
     def __init__(
@@ -28,4 +29,18 @@ class ConfigurationManager:
         )
 
         return data_ingestion_config
+    
+    
+    def get_data_validation_config(self) -> DataValidationConfig:
+        config = self.config.data_validation
+
+        create_directories([config.root_dir])
+
+        data_validation_config = DataValidationConfig(
+            root_dir=config.root_dir,
+            STATUS_FILE=config.STATUS_FILE,
+            ALL_REQUIRED_FILES=config.ALL_REQUIRED_FILES,
+        )
+
+        return data_validation_config
         

src/textsummarizer/conponents/data_validation.py

@@ -0,0 +1,30 @@
+import os
+from textsummarizer.logging import logger
+from textsummarizer.entity.config_entity import DataValidationConfig
+
+class DataValiadtion:
+    def __init__(self, config: DataValidationConfig):
+        self.config = config
+
+
+    
+    def validate_all_files_exist(self)-> bool:
+        try:
+            validation_status = None
+
+            all_files = os.listdir(os.path.join("artifacts","data_ingestion","samsum_dataset"))
+
+            for file in all_files:
+                if file not in self.config.ALL_REQUIRED_FILES:
+                    validation_status = False
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+                else:
+                    validation_status = True
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+
+            return validation_status
+        
+        except Exception as e:
+            raise e

src/textsummarizer/entity/config_entity.py

@@ -9,3 +9,9 @@ class DataIngestionConfig:
     unzip_dir : Path
     
     
+    
+@dataclass(frozen=True)
+class DataValidationConfig:
+    root_dir : Path
+    STATUS_FILE : str
+    ALL_REQUIRED_FILES : list

src/textsummarizer/pipeline/stage_02_data_validation.py

@@ -0,0 +1,14 @@
+from textsummarizer.config.configuration import ConfigurationManager
+from textsummarizer.entity.config_entity import DataValidationConfig
+from textsummarizer.conponents.data_validation import DataValiadtion
+
+
+class DataValidationPipeline:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config  = ConfigurationManager()
+        data_validataion_config = config.get_data_validation_config()
+        data_validation = DataValiadtion(config=data_validataion_config)
+        data_validation.validate_all_files_exist()