added the appilication

.gitignore

@@ -0,0 +1,166 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# leave the dataset file folder
+artifacts/
+venv_text_summarizaition/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Pavithra Devi M
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -0,0 +1,49 @@
+import os
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import Response
+from starlette.responses import RedirectResponse
+from textSummarizer.pipeline.prediction import PredictionPipeline
+
+text:str = "What is Text Summarization?"
+
+app = FastAPI()
+
+@app.get("/", tags=["authentication"])
+async def index():
+    """
+    The main page.
+    """
+    return "The API is UP and running."
+
+
+@app.get("/train")
+async def training():
+    """
+    The training page.
+    """
+    try:
+        os.system("python main.py")
+        return Response("Training successful !!")
+
+    except Exception as e:
+        return Response(f"Error Occurred! {e}")
+
+
+@app.post("/predict")
+async def predict_route(text):
+    """
+    The prediction api call.
+    """
+    try:
+
+        obj = PredictionPipeline()
+        text = obj.predict(text)
+        return text
+    except Exception as e:
+        raise e
+
+
+if __name__=="__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8080)

check_code/01_data_dowanloading.ipynb

@@ -0,0 +1,179 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/home/pavithra/projects/Text-summarization-nlp'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "%pwd\n",
+    "os.chdir(\"../\")\n",
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataIngestionConfig:\n",
+    "    dataset_name: str\n",
+    "    arrow_dataset_dir: str\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from box import ConfigBox\n",
+    "from pathlib import Path\n",
+    "from src.TextSummarizer.constants import file_path\n",
+    "from src.TextSummarizer.utils.general import read_yaml, create_directories\n",
+    "\n",
+    "# Create a config manager.\n",
+    "class ConfigManager:\n",
+    "    \"\"\"\n",
+    "    Class to manage the configuration files.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self) -> None:\n",
+    "        self.config: ConfigBox = read_yaml(Path(file_path.CONFIG_FILE_PATH))\n",
+    "        self.params: ConfigBox = read_yaml(Path(file_path.PARAMS_FILE_PATH))\n",
+    "\n",
+    "        create_directories(path_to_directories=[self.config.artifacts_root])\n",
+    "\n",
+    "    def get_data_ingestion_config(self) -> DataIngestionConfig:\n",
+    "        \"\"\"\n",
+    "        Get the config which is needed to download the data files.\n",
+    "        \"\"\"\n",
+    "        config: ConfigBox = self.config.data_ingestion\n",
+    "\n",
+    "        data_ingestion_config: DataIngestionConfig = DataIngestionConfig(\n",
+    "            dataset_name=config.dataset_name,\n",
+    "            arrow_dataset_dir=config.arrow_dataset_dir,\n",
+    "        )\n",
+    "\n",
+    "        return data_ingestion_config\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from datasets import load_dataset\n",
+    "\n",
+    "# test_dataset = load_dataset(\"alexfabbri/multi_news\")\n",
+    "\n",
+    "\n",
+    "# test_dataset.save_to_disk(\"multi_news_arrow_dataset\")\n",
+    "# from datasets import load_from_disk\n",
+    "# arrow_datasets_reloaded = load_from_disk(\"multi_news_arrow_dataset\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "class DataIngestionComponent:\n",
+    "    \"\"\"\n",
+    "    A Class which is responsible for data ingestion.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self, config: DataIngestionConfig) -> None:\n",
+    "        self.config = config\n",
+    "\n",
+    "    def save_dataset(self):\n",
+    "        \"\"\"\n",
+    "        Load the dataset.\n",
+    "        \"\"\"\n",
+    "        test_dataset = load_dataset(self.config.dataset_name)\n",
+    "        test_dataset.save_to_disk(self.config.arrow_dataset_dir)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: 'config/config.yaml'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[1;32m/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb Cell 6\u001b[0m line \u001b[0;36m7\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m     data_ingestion\u001b[39m.\u001b[39msave_dataset()\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m     \u001b[39mraise\u001b[39;00m err\n",
+      "\u001b[1;32m/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb Cell 6\u001b[0m line \u001b[0;36m2\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m     config: ConfigManager \u001b[39m=\u001b[39m ConfigManager()\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m     data_ingestion_config: DataIngestionConfig \u001b[39m=\u001b[39m config\u001b[39m.\u001b[39mget_data_ingestion_config()\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m     data_ingestion \u001b[39m=\u001b[39m DataIngestionComponent(config\u001b[39m=\u001b[39mdata_ingestion_config)\n",
+      "\u001b[1;32m/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb Cell 6\u001b[0m line \u001b[0;36m1\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig: ConfigBox \u001b[39m=\u001b[39m read_yaml(Path(file_path\u001b[39m.\u001b[39;49mCONFIG_FILE_PATH))\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mparams: ConfigBox \u001b[39m=\u001b[39m read_yaml(Path(file_path\u001b[39m.\u001b[39mPARAMS_FILE_PATH))\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/pavithra/projects/Text-summarization-nlp/check_code/01_data_dowanloading.ipynb#W5sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m     create_directories(path_to_directories\u001b[39m=\u001b[39m[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39martifacts_root])\n",
+      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/ensure/main.py:872\u001b[0m, in \u001b[0;36mWrappedFunctionReturn.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    869\u001b[0m         msg \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mArgument \u001b[39m\u001b[39m{arg}\u001b[39;00m\u001b[39m of type \u001b[39m\u001b[39m{valt}\u001b[39;00m\u001b[39m to \u001b[39m\u001b[39m{f}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mdoes not match annotation type \u001b[39m\u001b[39m{t}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m    870\u001b[0m         \u001b[39mraise\u001b[39;00m EnsureError(msg\u001b[39m.\u001b[39mformat(arg\u001b[39m=\u001b[39marg, f\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mf, t\u001b[39m=\u001b[39mtempl, valt\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m(value)))\n\u001b[0;32m--> 872\u001b[0m return_val \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mf(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    873\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(return_val, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mreturn_templ):\n\u001b[1;32m    874\u001b[0m     msg \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mReturn value of \u001b[39m\u001b[39m{f}\u001b[39;00m\u001b[39m of type \u001b[39m\u001b[39m{valt}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mdoes not match annotation type \u001b[39m\u001b[39m{t}\u001b[39;00m\u001b[39m\"\u001b[39m\n",
+      "File \u001b[0;32m~/projects/Text-summarization-nlp/src/TextSummarizer/utils/general.py:33\u001b[0m, in \u001b[0;36mread_yaml\u001b[0;34m(path_to_yaml)\u001b[0m\n\u001b[1;32m     31\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39myaml file: \u001b[39m\u001b[39m{\u001b[39;00mpath_to_yaml\u001b[39m}\u001b[39;00m\u001b[39m is empty.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     32\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exp:\n\u001b[0;32m---> 33\u001b[0m     \u001b[39mraise\u001b[39;00m exp\n",
+      "File \u001b[0;32m~/projects/Text-summarization-nlp/src/TextSummarizer/utils/general.py:26\u001b[0m, in \u001b[0;36mread_yaml\u001b[0;34m(path_to_yaml)\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m     20\u001b[0m \u001b[39mRead yaml file and return as Dictionary.\u001b[39;00m\n\u001b[1;32m     21\u001b[0m \n\u001b[1;32m     22\u001b[0m \u001b[39m:param path_to_yaml: Path to yaml file.\u001b[39;00m\n\u001b[1;32m     23\u001b[0m \u001b[39m:return: A ConfigBox dictionary object containing the the yaml file contents.\u001b[39;00m\n\u001b[1;32m     24\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m     25\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 26\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39;49m(path_to_yaml) \u001b[39mas\u001b[39;00m yaml_file:\n\u001b[1;32m     27\u001b[0m         content \u001b[39m=\u001b[39m yaml\u001b[39m.\u001b[39msafe_load(yaml_file)\n\u001b[1;32m     28\u001b[0m         backend_logger\u001b[39m.\u001b[39minfo(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39myaml file: \u001b[39m\u001b[39m{\u001b[39;00mpath_to_yaml\u001b[39m}\u001b[39;00m\u001b[39m loaded successfully\u001b[39m\u001b[39m\"\u001b[39m)\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'config/config.yaml'"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config: ConfigManager = ConfigManager()\n",
+    "    data_ingestion_config: DataIngestionConfig = config.get_data_ingestion_config()\n",
+    "    data_ingestion = DataIngestionComponent(config=data_ingestion_config)\n",
+    "    data_ingestion.save_dataset()\n",
+    "except Exception as err:\n",
+    "    raise err"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

check_code/02_data_validation.ipynb

@@ -0,0 +1,150 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/home/pavithra/projects/Text-summarization-nlp'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "%pwd\n",
+    "os.chdir(\"../\")\n",
+    "\n",
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataValidationConfig:\n",
+    "    root_dir: str\n",
+    "    status_file: str\n",
+    "    all_required_folders: list\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.TextSummarizer.constants import file_path\n",
+    "from src.TextSummarizer.utils.general import read_yaml, create_directories\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = file_path.CONFIG_FILE_PATH,\n",
+    "        params_filepath = file_path.PARAMS_FILE_PATH):\n",
+    "\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def get_data_validation_config(self) -> DataValidationConfig:\n",
+    "        config = self.config.data_validation\n",
+    "\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        data_validation_config = DataValidationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            status_file=config.status_file,\n",
+    "            all_required_folders=config.all_required_folders,\n",
+    "        )\n",
+    "\n",
+    "        return data_validation_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DataValidation:\n",
+    "    def __init__(self, config: DataValidationConfig):\n",
+    "        self.config = config\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def validate_all_files_exist(self)-> bool:\n",
+    "        try:\n",
+    "            validation_status: bool | None = None\n",
+    "\n",
+    "            all_folder = os.listdir(os.path.join(\"artifacts\",\"data\"))\n",
+    "\n",
+    "            for folder in all_folder:\n",
+    "                if folder not in self.config.all_required_folders:\n",
+    "                    validation_status = False\n",
+    "                    with open(self.config.status_file, \"w\") as f:\n",
+    "                        f.write(f\"Validation status: {validation_status}\")\n",
+    "                else:\n",
+    "                    validation_status = True\n",
+    "                    with open(self.config.status_file, \"w\") as f:\n",
+    "                        f.write(f\"Validation status: {validation_status}\")\n",
+    "\n",
+    "            return validation_status\n",
+    "\n",
+    "        except Exception as exp:\n",
+    "            raise exp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_validation_config = config.get_data_validation_config()\n",
+    "    data_validation = DataValiadtion(config=data_validation_config)\n",
+    "    data_validation.validate_all_files_exist()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

check_code/03_data_transformation.ipynb

@@ -0,0 +1,132 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "%pwd\n",
+    "os.chdir(\"../\")\n",
+    "\n",
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataTransformationConfig:\n",
+    "    root_dir: str\n",
+    "    data_path: str\n",
+    "    tokenizer_name: str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from box import ConfigBox\n",
+    "from pathlib import Path\n",
+    "from src.TextSummarizer.constants import file_path\n",
+    "from src.TextSummarizer.utils.general import read_yaml, create_directories\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    \"\"\"\n",
+    "    Class to manage the configuration files.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self) -> None:\n",
+    "        self.config: ConfigBox = read_yaml(Path(file_path.CONFIG_FILE_PATH))\n",
+    "        self.params: ConfigBox = read_yaml(Path(file_path.PARAMS_FILE_PATH))\n",
+    "\n",
+    "        create_directories(path_to_directories=[self.config.artifacts_root])\n",
+    "\n",
+    "    def get_data_transformation_config(self) -> DataTransformationConfig:\n",
+    "        config = self.config.data_transformation\n",
+    "\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        data_transformation_config = DataTransformationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            data_path=config.data_path,\n",
+    "            tokenizer_name = config.tokenizer_name\n",
+    "        )\n",
+    "\n",
+    "        return data_transformation_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "import os\n",
+    "from src.TextSummarizer.logger import backend_logger\n",
+    "from transformers import AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk\n",
+    "\n",
+    "\n",
+    "class DataTransformation:\n",
+    "    def __init__(self, config: DataTransformationConfig):\n",
+    "        self.config = config\n",
+    "        self.tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name)\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def convert_examples_to_features(self,example_batch):\n",
+    "        input_encodings = self.tokenizer(example_batch['dialogue'] , max_length = 800, truncation = True )\n",
+    "\n",
+    "        with self.tokenizer.as_target_tokenizer():\n",
+    "            target_encodings = self.tokenizer(example_batch['summary'], max_length = 128, truncation = True )\n",
+    "\n",
+    "        return {\n",
+    "            'input_ids' : input_encodings['input_ids'],\n",
+    "            'attention_mask': input_encodings['attention_mask'],\n",
+    "            'labels': target_encodings['input_ids']\n",
+    "        }\n",
+    "\n",
+    "\n",
+    "    def convert(self):\n",
+    "        dataset = load_from_disk(self.config.data_path)\n",
+    "        dataset = dataset.map(self.convert_examples_to_features, batched = True)\n",
+    "        dataset.save_to_disk(os.path.join(self.config.root_dir,\"dataset\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_transformation_config = config.get_data_transformation_config()\n",
+    "    data_transformation = DataTransformation(config=data_transformation_config)\n",
+    "    data_transformation.convert()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

check_code/04_model_training.ipynb

@@ -0,0 +1,184 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "%pwd\n",
+    "os.chdir(\"../\")\n",
+    "\n",
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class ModelTrainerConfig:\n",
+    "    root_dir: str\n",
+    "    data_path: str\n",
+    "    model_ckpt: str\n",
+    "    num_train_epochs: int\n",
+    "    warmup_steps: int\n",
+    "    per_device_train_batch_size: int\n",
+    "    weight_decay: float\n",
+    "    logging_steps: int\n",
+    "    evaluation_strategy: str\n",
+    "    eval_steps: int\n",
+    "    save_steps: float\n",
+    "    gradient_accumulation_steps: int"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from box import ConfigBox\n",
+    "from pathlib import Path\n",
+    "from src.TextSummarizer.constants import file_path\n",
+    "from src.TextSummarizer.utils.general import read_yaml, create_directories\n",
+    "\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "\n",
+    "    def __init__(self) -> None:\n",
+    "        self.config: ConfigBox = read_yaml(Path(file_path.CONFIG_FILE_PATH))\n",
+    "        self.params: ConfigBox = read_yaml(Path(file_path.PARAMS_FILE_PATH))\n",
+    "\n",
+    "        create_directories(path_to_directories=[self.config.artifacts_root])\n",
+    "\n",
+    "    def get_model_trainer_config(self) -> ModelTrainerConfig:\n",
+    "        config = self.config.model_trainer\n",
+    "        params = self.params.TrainingArguments\n",
+    "\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        model_trainer_config = ModelTrainerConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            data_path=config.data_path,\n",
+    "            model_ckpt = config.model_ckpt,\n",
+    "            num_train_epochs = params.num_train_epochs,\n",
+    "            warmup_steps = params.warmup_steps,\n",
+    "            per_device_train_batch_size = params.per_device_train_batch_size,\n",
+    "            weight_decay = params.weight_decay,\n",
+    "            logging_steps = params.logging_steps,\n",
+    "            evaluation_strategy = params.evaluation_strategy,\n",
+    "            eval_steps = params.evaluation_strategy,\n",
+    "            save_steps = params.save_steps,\n",
+    "            gradient_accumulation_steps = params.gradient_accumulation_steps\n",
+    "        )\n",
+    "\n",
+    "        return model_trainer_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import TrainingArguments, Trainer\n",
+    "from transformers import DataCollatorForSeq2Seq\n",
+    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk\n",
+    "import torch\n",
+    "\n",
+    "\n",
+    "class ModelTrainer:\n",
+    "    def __init__(self, config: ModelTrainerConfig):\n",
+    "        self.config = config\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def train(self):\n",
+    "        device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(self.config.model_ckpt)\n",
+    "        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_ckpt).to(device)\n",
+    "        seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)\n",
+    "\n",
+    "        #loading data\n",
+    "        dataset = load_from_disk(self.config.data_path)\n",
+    "\n",
+    "        # trainer_args = TrainingArguments(\n",
+    "        #     output_dir=self.config.root_dir, num_train_epochs=self.config.num_train_epochs, warmup_steps=self.config.warmup_steps,\n",
+    "        #     per_device_train_batch_size=self.config.per_device_train_batch_size, per_device_eval_batch_size=self.config.per_device_train_batch_size,\n",
+    "        #     weight_decay=self.config.weight_decay, logging_steps=self.config.logging_steps,\n",
+    "        #     evaluation_strategy=self.config.evaluation_strategy, eval_steps=self.config.eval_steps, save_steps=1e6,\n",
+    "        #     gradient_accumulation_steps=self.config.gradient_accumulation_steps\n",
+    "        # )\n",
+    "\n",
+    "\n",
+    "        trainer_args = TrainingArguments(\n",
+    "            output_dir=self.config.root_dir,\n",
+    "            num_train_epochs=1,\n",
+    "            warmup_steps=500,\n",
+    "            per_device_train_batch_size=1,\n",
+    "            per_device_eval_batch_size=1,\n",
+    "            weight_decay=0.01,\n",
+    "            logging_steps=10,\n",
+    "            evaluation_strategy='steps',\n",
+    "            eval_steps=500,\n",
+    "            save_steps=1e6,\n",
+    "            gradient_accumulation_steps=16\n",
+    "        )\n",
+    "\n",
+    "        trainer = Trainer(\n",
+    "            model=model_pegasus,\n",
+    "            args=trainer_args,\n",
+    "            tokenizer=tokenizer,\n",
+    "            data_collator=seq2seq_data_collator,\n",
+    "            train_dataset=dataset[\"train\"],\n",
+    "            eval_dataset=dataset[\"validation\"])\n",
+    "\n",
+    "        # trainer.train()\n",
+    "\n",
+    "        ## Save model\n",
+    "        model_pegasus.save_pretrained(\"multi-news-model\")\n",
+    "\n",
+    "        ## Save tokenizer\n",
+    "        tokenizer.save_pretrained(\"tokenizer\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    model_trainer_config = config.get_model_trainer_config()\n",
+    "    model_trainer_config = ModelTrainer(config=model_trainer_config)\n",
+    "    model_trainer_config.train()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

check_code/05_model_evaluation.ipynb

@@ -0,0 +1,177 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "%pwd\n",
+    "os.chdir(\"../\")\n",
+    "\n",
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class ModelEvaluationConfig:\n",
+    "    root_dir: str\n",
+    "    data_path: str\n",
+    "    model_path: str\n",
+    "    tokenizer_path: str\n",
+    "    metric_file_name: str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from box import ConfigBox\n",
+    "from pathlib import Path\n",
+    "from src.TextSummarizer.constants import file_path\n",
+    "from src.TextSummarizer.utils.general import read_yaml, create_directories\n",
+    "\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "\n",
+    "    def __init__(self) -> None:\n",
+    "        self.config: ConfigBox = read_yaml(Path(file_path.CONFIG_FILE_PATH))\n",
+    "        self.params: ConfigBox = read_yaml(Path(file_path.PARAMS_FILE_PATH))\n",
+    "\n",
+    "        create_directories(path_to_directories=[self.config.artifacts_root])\n",
+    "\n",
+    "    def get_model_evaluation_config(self) -> ModelEvaluationConfig:\n",
+    "        config = self.config.model_evaluation\n",
+    "\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        model_evaluation_config = ModelEvaluationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            data_path=config.data_path,\n",
+    "            model_path = config.model_path,\n",
+    "            tokenizer_path = config.tokenizer_path,\n",
+    "            metric_file_name = config.metric_file_name\n",
+    "\n",
+    "        )\n",
+    "\n",
+    "        return model_evaluation_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
+    "from datasets import load_dataset, load_from_disk, load_metric\n",
+    "import torch\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "class ModelEvaluation:\n",
+    "    def __init__(self, config: ModelEvaluationConfig):\n",
+    "        self.config = config\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def generate_batch_sized_chunks(self,list_of_elements, batch_size):\n",
+    "        \"\"\"split the dataset into smaller batches that we can process simultaneously\n",
+    "        Yield successive batch-sized chunks from list_of_elements.\"\"\"\n",
+    "        for i in range(0, len(list_of_elements), batch_size):\n",
+    "            yield list_of_elements[i : i + batch_size]\n",
+    "\n",
+    "\n",
+    "    def calculate_metric_on_test_ds(self,dataset, metric, model, tokenizer,\n",
+    "                               batch_size=16, device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
+    "                               column_text=\"article\",\n",
+    "                               column_summary=\"highlights\"):\n",
+    "        article_batches = list(self.generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
+    "        target_batches = list(self.generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
+    "\n",
+    "        for article_batch, target_batch in tqdm(\n",
+    "            zip(article_batches, target_batches), total=len(article_batches)):\n",
+    "\n",
+    "            inputs = tokenizer(article_batch, max_length=1024,  truncation=True,\n",
+    "                            padding=\"max_length\", return_tensors=\"pt\")\n",
+    "\n",
+    "            summaries = model.generate(input_ids=inputs[\"input_ids\"].to(device),\n",
+    "                            attention_mask=inputs[\"attention_mask\"].to(device),\n",
+    "                            length_penalty=0.8, num_beams=8, max_length=128)\n",
+    "            ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''\n",
+    "\n",
+    "            # Finally, we decode the generated texts,\n",
+    "            # replace the  token, and add the decoded texts with the references to the metric.\n",
+    "            decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,\n",
+    "                                    clean_up_tokenization_spaces=True)\n",
+    "                for s in summaries]\n",
+    "\n",
+    "            decoded_summaries = [d.replace(\"\", \" \") for d in decoded_summaries]\n",
+    "\n",
+    "\n",
+    "            metric.add_batch(predictions=decoded_summaries, references=target_batch)\n",
+    "\n",
+    "        #  Finally compute and return the ROUGE scores.\n",
+    "        score = metric.compute()\n",
+    "        return score\n",
+    "\n",
+    "\n",
+    "    def evaluate(self):\n",
+    "        device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_path)\n",
+    "        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path).to(device)\n",
+    "\n",
+    "        #loading data\n",
+    "        dataset_samsum_pt = load_from_disk(self.config.data_path)\n",
+    "\n",
+    "\n",
+    "        rouge_names = [\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"]\n",
+    "\n",
+    "        rouge_metric = load_metric('rouge')\n",
+    "\n",
+    "        score = self.calculate_metric_on_test_ds(\n",
+    "        dataset_samsum_pt['test'][0:10], rouge_metric, model_pegasus, tokenizer, batch_size = 2, column_text = 'dialogue', column_summary= 'summary'\n",
+    "            )\n",
+    "\n",
+    "        rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names )\n",
+    "\n",
+    "        df = pd.DataFrame(rouge_dict, index = ['pegasus'] )\n",
+    "        df.to_csv(self.config.metric_file_name, index=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    model_evaluation_config = config.get_model_evaluation_config()\n",
+    "    model_evaluation_config = ModelEvaluation(config=model_evaluation_config)\n",
+    "    model_evaluation_config.evaluate()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

check_code/predict.ipynb

@@ -0,0 +1,504 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6fe4263eb5e743e7968cf6f1b140a744",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# pavithra-devi/google_pegasus_multi_news_model\n",
+    "\n",
+    "\n",
+    "from huggingface_hub import notebook_login\n",
+    "\n",
+    "notebook_login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "df05cc7954324819a2ba5645e881ea49",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/20.1k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9ca03f3fecbe48f994a272a75a06e779",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "spiece.model:   0%|          | 0.00/1.91M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ba543a91b4b9494e8009f60e2cd80fc2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/6.60M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7911d674ade7471e95a8c9041d69adb1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "special_tokens_map.json:   0%|          | 0.00/1.77k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6d1b306a35f24cd8a76415bb399a74c5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/1.32k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "04440e9bef144c4197f8e11b00a21b61",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/2.28G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2e5f5ed6ff8644cd92f3690374361820",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/275 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForSeq2SeqLM\n",
+    "model_name = \"pavithra-devi/pegasus-multi-news\"\n",
+    "\n",
+    "# Load model directly\n",
+    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"pavithra-devi/pegasus-multi-news\")\n",
+    "model = AutoModelForSeq2SeqLM.from_pretrained(\"pavithra-devi/pegasus-multi-news\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "\n",
+    "summarizer = pipeline(\"summarization\", model=\"pavithra-devi/pegasus-multi-news\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'summary_text': '– A recent Supreme Court decision gives states the ability to opt out of the law\\'s expansion of Medicaid, the federal insurance program for poor, disabled and elderly Americans, confers \"incredible power\" on the states and their governors, says political scientist Thad Kousser, co-author of The Power of American Governors. \"No matter who wins the presidency, national politics is going to be stalemated on the Affordable Care Act,\" says Kousser, of the University of California-Berkeley. Just look at what happened when the Obama administration in 2010 offered federal stimulus money to states to begin building a high-speed rail network. Three'}]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "text = \"\"\" While the occupant of the governor's office is historically far less important than the party that controls the state legislature, top state officials in coming years are expected to wield significant influence in at least one major area.\n",
+    "\n",
+    " And that's health care, says political scientist Thad Kousser, co-author of The Power of American Governors.\n",
+    "\n",
+    " \"No matter who wins the presidency, national politics is going to be stalemated on the Affordable Care Act,\" says Kousser, of the University of California-Berkeley.\n",
+    "\n",
+    " A recent U.S. Supreme Court decision giving states the ability to opt out of the law's expansion of Medicaid, the federal insurance program for poor, disabled and elderly Americans, confers \"incredible power\" on the states and their governors, Kousser says.\n",
+    "\n",
+    " Just look at what happened when the Obama administration in 2010 offered federal stimulus money to states to begin building a high-speed rail network. Three Republican governors, including Rick Scott of Florida and Scott Walker of Wisconsin, rejected a share of the money citing debt and deficit concerns.\n",
+    "\n",
+    " \"A [Mitt] Romney victory would dramatically empower Republican governors,\" Kousser says. \"\"\"\n",
+    "\n",
+    "summarizer(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "[{'summary_text': '– A recent Supreme Court decision gives states the ability to opt out of the law\\'s expansion of Medicaid,\n",
+    "   the federal insurance program for poor, disabled and elderly Americans, confers \"incredible power\" on the states and their governors,\n",
+    "   says political scientist Thad Kousser, co-author of The Power of American Governors. \"No matter who wins the presidency, national politics\n",
+    "     is going to be stalemated on the Affordable Care Act,\" says Kousser, of the University of California-Berkeley. Just look at what happened when the\n",
+    "     Obama administration in 2010 offered federal stimulus money to states to begin building a high-speed rail network. Three'}]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['T_destination',\n",
+       " '__annotations__',\n",
+       " '__call__',\n",
+       " '__class__',\n",
+       " '__delattr__',\n",
+       " '__dict__',\n",
+       " '__dir__',\n",
+       " '__doc__',\n",
+       " '__eq__',\n",
+       " '__format__',\n",
+       " '__ge__',\n",
+       " '__getattr__',\n",
+       " '__getattribute__',\n",
+       " '__getstate__',\n",
+       " '__gt__',\n",
+       " '__hash__',\n",
+       " '__init__',\n",
+       " '__init_subclass__',\n",
+       " '__le__',\n",
+       " '__lt__',\n",
+       " '__module__',\n",
+       " '__ne__',\n",
+       " '__new__',\n",
+       " '__reduce__',\n",
+       " '__reduce_ex__',\n",
+       " '__repr__',\n",
+       " '__setattr__',\n",
+       " '__setstate__',\n",
+       " '__sizeof__',\n",
+       " '__str__',\n",
+       " '__subclasshook__',\n",
+       " '__weakref__',\n",
+       " '_apply',\n",
+       " '_assisted_decoding',\n",
+       " '_auto_class',\n",
+       " '_autoset_attn_implementation',\n",
+       " '_backward_compatibility_gradient_checkpointing',\n",
+       " '_backward_hooks',\n",
+       " '_backward_pre_hooks',\n",
+       " '_beam_sample',\n",
+       " '_beam_search',\n",
+       " '_buffers',\n",
+       " '_call_impl',\n",
+       " '_check_and_enable_flash_attn_2',\n",
+       " '_check_and_enable_sdpa',\n",
+       " '_compiled_call_impl',\n",
+       " '_constrained_beam_search',\n",
+       " '_contrastive_search',\n",
+       " '_convert_head_mask_to_5d',\n",
+       " '_copy_lm_head_original_to_resized',\n",
+       " '_create_repo',\n",
+       " '_dispatch_accelerate_model',\n",
+       " '_expand_inputs_for_generation',\n",
+       " '_extract_past_from_model_output',\n",
+       " '_forward_hooks',\n",
+       " '_forward_hooks_always_called',\n",
+       " '_forward_hooks_with_kwargs',\n",
+       " '_forward_pre_hooks',\n",
+       " '_forward_pre_hooks_with_kwargs',\n",
+       " '_from_config',\n",
+       " '_get_backward_hooks',\n",
+       " '_get_backward_pre_hooks',\n",
+       " '_get_cache',\n",
+       " '_get_candidate_generator',\n",
+       " '_get_decoder_start_token_id',\n",
+       " '_get_files_timestamps',\n",
+       " '_get_initial_cache_position',\n",
+       " '_get_logits_processor',\n",
+       " '_get_logits_warper',\n",
+       " '_get_name',\n",
+       " '_get_no_split_modules',\n",
+       " '_get_resized_embeddings',\n",
+       " '_get_resized_lm_head',\n",
+       " '_get_stopping_criteria',\n",
+       " '_greedy_search',\n",
+       " '_group_beam_search',\n",
+       " '_has_unfinished_sequences',\n",
+       " '_hf_peft_config_loaded',\n",
+       " '_hook_rss_memory_post_forward',\n",
+       " '_hook_rss_memory_pre_forward',\n",
+       " '_init_weights',\n",
+       " '_initialize_weights',\n",
+       " '_is_full_backward_hook',\n",
+       " '_is_hf_initialized',\n",
+       " '_is_quantized_training_enabled',\n",
+       " '_keep_in_fp32_modules',\n",
+       " '_keep_in_fp32_modules',\n",
+       " '_keys_to_ignore_on_load_missing',\n",
+       " '_keys_to_ignore_on_load_unexpected',\n",
+       " '_keys_to_ignore_on_save',\n",
+       " '_load_from_state_dict',\n",
+       " '_load_pretrained_model',\n",
+       " '_load_pretrained_model_low_mem',\n",
+       " '_load_state_dict_post_hooks',\n",
+       " '_load_state_dict_pre_hooks',\n",
+       " '_maybe_initialize_input_ids_for_generation',\n",
+       " '_maybe_warn_non_full_backward_hook',\n",
+       " '_merge_criteria_processor_list',\n",
+       " '_modules',\n",
+       " '_named_members',\n",
+       " '_no_split_modules',\n",
+       " '_non_persistent_buffers_set',\n",
+       " '_parameters',\n",
+       " '_prepare_attention_mask_for_generation',\n",
+       " '_prepare_decoder_input_ids_for_generation',\n",
+       " '_prepare_encoder_decoder_kwargs_for_generation',\n",
+       " '_prepare_generated_length',\n",
+       " '_prepare_generation_config',\n",
+       " '_prepare_model_inputs',\n",
+       " '_prepare_special_tokens',\n",
+       " '_register_load_state_dict_pre_hook',\n",
+       " '_register_state_dict_hook',\n",
+       " '_reorder_cache',\n",
+       " '_replicate_for_data_parallel',\n",
+       " '_resize_final_logits_bias',\n",
+       " '_resize_token_embeddings',\n",
+       " '_sample',\n",
+       " '_save_to_state_dict',\n",
+       " '_set_default_torch_dtype',\n",
+       " '_set_gradient_checkpointing',\n",
+       " '_skip_keys_device_placement',\n",
+       " '_slow_forward',\n",
+       " '_state_dict_hooks',\n",
+       " '_state_dict_pre_hooks',\n",
+       " '_supports_cache_class',\n",
+       " '_supports_flash_attn_2',\n",
+       " '_supports_quantized_cache',\n",
+       " '_supports_sdpa',\n",
+       " '_supports_static_cache',\n",
+       " '_temporary_reorder_cache',\n",
+       " '_tie_encoder_decoder_weights',\n",
+       " '_tie_or_clone_weights',\n",
+       " '_tied_weights_keys',\n",
+       " '_update_model_kwargs_for_generation',\n",
+       " '_upload_modified_files',\n",
+       " '_validate_assistant',\n",
+       " '_validate_generated_length',\n",
+       " '_validate_model_class',\n",
+       " '_validate_model_kwargs',\n",
+       " '_version',\n",
+       " '_wrapped_call_impl',\n",
+       " 'active_adapter',\n",
+       " 'active_adapters',\n",
+       " 'add_adapter',\n",
+       " 'add_memory_hooks',\n",
+       " 'add_model_tags',\n",
+       " 'add_module',\n",
+       " 'apply',\n",
+       " 'base_model',\n",
+       " 'base_model_prefix',\n",
+       " 'bfloat16',\n",
+       " 'buffers',\n",
+       " 'call_super_init',\n",
+       " 'can_generate',\n",
+       " 'children',\n",
+       " 'compile',\n",
+       " 'compute_transition_scores',\n",
+       " 'config',\n",
+       " 'config_class',\n",
+       " 'cpu',\n",
+       " 'create_extended_attention_mask_for_decoder',\n",
+       " 'cuda',\n",
+       " 'dequantize',\n",
+       " 'device',\n",
+       " 'disable_adapters',\n",
+       " 'disable_input_require_grads',\n",
+       " 'double',\n",
+       " 'dtype',\n",
+       " 'dummy_inputs',\n",
+       " 'dump_patches',\n",
+       " 'enable_adapters',\n",
+       " 'enable_input_require_grads',\n",
+       " 'estimate_tokens',\n",
+       " 'eval',\n",
+       " 'extra_repr',\n",
+       " 'final_logits_bias',\n",
+       " 'float',\n",
+       " 'floating_point_ops',\n",
+       " 'forward',\n",
+       " 'framework',\n",
+       " 'from_pretrained',\n",
+       " 'generate',\n",
+       " 'generation_config',\n",
+       " 'get_adapter_state_dict',\n",
+       " 'get_buffer',\n",
+       " 'get_decoder',\n",
+       " 'get_encoder',\n",
+       " 'get_extended_attention_mask',\n",
+       " 'get_extra_state',\n",
+       " 'get_head_mask',\n",
+       " 'get_input_embeddings',\n",
+       " 'get_memory_footprint',\n",
+       " 'get_output_embeddings',\n",
+       " 'get_parameter',\n",
+       " 'get_position_embeddings',\n",
+       " 'get_submodule',\n",
+       " 'gradient_checkpointing_disable',\n",
+       " 'gradient_checkpointing_enable',\n",
+       " 'half',\n",
+       " 'init_weights',\n",
+       " 'invert_attention_mask',\n",
+       " 'ipu',\n",
+       " 'is_gradient_checkpointing',\n",
+       " 'is_parallelizable',\n",
+       " 'lm_head',\n",
+       " 'load_adapter',\n",
+       " 'load_state_dict',\n",
+       " 'main_input_name',\n",
+       " 'model',\n",
+       " 'model_tags',\n",
+       " 'modules',\n",
+       " 'name_or_path',\n",
+       " 'named_buffers',\n",
+       " 'named_children',\n",
+       " 'named_modules',\n",
+       " 'named_parameters',\n",
+       " 'num_parameters',\n",
+       " 'parameters',\n",
+       " 'post_init',\n",
+       " 'prepare_decoder_input_ids_from_labels',\n",
+       " 'prepare_inputs_for_generation',\n",
+       " 'prune_heads',\n",
+       " 'push_to_hub',\n",
+       " 'register_backward_hook',\n",
+       " 'register_buffer',\n",
+       " 'register_for_auto_class',\n",
+       " 'register_forward_hook',\n",
+       " 'register_forward_pre_hook',\n",
+       " 'register_full_backward_hook',\n",
+       " 'register_full_backward_pre_hook',\n",
+       " 'register_load_state_dict_post_hook',\n",
+       " 'register_module',\n",
+       " 'register_parameter',\n",
+       " 'register_state_dict_pre_hook',\n",
+       " 'requires_grad_',\n",
+       " 'reset_memory_hooks_state',\n",
+       " 'resize_position_embeddings',\n",
+       " 'resize_token_embeddings',\n",
+       " 'retrieve_modules_from_names',\n",
+       " 'reverse_bettertransformer',\n",
+       " 'save_pretrained',\n",
+       " 'set_adapter',\n",
+       " 'set_extra_state',\n",
+       " 'set_input_embeddings',\n",
+       " 'set_output_embeddings',\n",
+       " 'share_memory',\n",
+       " 'state_dict',\n",
+       " 'supports_gradient_checkpointing',\n",
+       " 'tie_weights',\n",
+       " 'to',\n",
+       " 'to_bettertransformer',\n",
+       " 'to_empty',\n",
+       " 'train',\n",
+       " 'training',\n",
+       " 'type',\n",
+       " 'warn_if_padding_and_no_attention_mask',\n",
+       " 'warnings_issued',\n",
+       " 'xpu',\n",
+       " 'zero_grad']"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenized_data ="
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv_text_summarizaition",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

main.py

@@ -0,0 +1,74 @@
+"""
+The file which does the all work. Calling all the pipeline to do the training.
+"""
+
+from src.TextSummarizer.logger import backend_logger
+from src.TextSummarizer.pipeline.step_01_data_ingestion import DataIngestionPipeline
+from src.TextSummarizer.pipeline.step_02_data_validation import DataValidationPipeline
+from src.TextSummarizer.pipeline.step_03_data_transformation import (
+    DataTransformationPipeline,
+)
+from src.TextSummarizer.pipeline.step_04_train_model import ModelTrainerPipeline
+from src.TextSummarizer.pipeline.step_05_model_evaluation import ModelEvaluationPipeline
+
+stage_name_01: str = "Stage 1: Data Integration Stage"
+stage_name_02: str = "Stage 2: Data Validation Stage"
+stage_name_03: str = "Stage 3: Data Transformation Stage"
+stage_name_04: str = "Stage 4: Model training Stage"
+stage_name_05: str = "Stage 5: Model Evaluation Stage"
+
+
+line_msg: str = "="*100
+
+try:
+    backend_logger.info(line_msg)
+    backend_logger.info(f"Stage {stage_name_01} started")
+    DataIngestionPipeline().run()
+    backend_logger.info(f"Stage {stage_name_01} completed.")
+    backend_logger.info(line_msg)
+except Exception as err:
+    backend_logger.error(f"Data ingestion pipeline failed. Reason: {err}")
+
+
+try:
+    backend_logger.info(line_msg)
+    backend_logger.info(f"Stage {stage_name_02} started")
+    DataValidationPipeline().run()
+    backend_logger.info(f"Stage {stage_name_02} completed.")
+    backend_logger.info(line_msg)
+except Exception as err:
+    backend_logger.error(f"Data validation pipeline failed. Reason: {err}")
+
+
+try:
+    backend_logger.info(line_msg)
+    backend_logger.info(f"Stage {stage_name_03} started")
+    DataTransformationPipeline().run()
+    backend_logger.info(f"Stage {stage_name_03} completed.")
+    backend_logger.info(line_msg)
+except Exception as err:
+    backend_logger.error(f"Data Transformation pipeline failed. Reason: {err}")
+
+
+# For the device limitations issues, i have trained the model on online and stored the model in huggingface profile.
+# We can skip the training and model evaluation steps while running locally.
+
+
+try:
+    backend_logger.info(line_msg)
+    backend_logger.info(f"Stage {stage_name_04} started")
+    ModelTrainerPipeline().run()
+    backend_logger.info(f"Stage {stage_name_04} completed.")
+    backend_logger.info(line_msg)
+except Exception as err:
+    backend_logger.error(f"Data data training pipeline failed. Reason: {err}")
+
+
+try:
+    backend_logger.info(line_msg)
+    backend_logger.info(f"Stage {stage_name_05} started")
+    ModelEvaluationPipeline().run()
+    backend_logger.info(f"Stage {stage_name_05} completed.")
+    backend_logger.info(line_msg)
+except Exception as err:
+    backend_logger.error(f"Model evaluation pipeline failed. Reason: {err}")

project_structure.py

@@ -0,0 +1,52 @@
+"""
+Used to create the whole project structure.
+"""
+
+import logging
+import os
+from pathlib import Path
+
+logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s]: %(message)s:')
+
+project_name: str = "TextSummarizer"
+
+list_of_files = [
+    ".github/workflows/.gitkeep",
+    f"src/{project_name}/__init__.py",
+    f"src/{project_name}/components/__init__.py",
+    f"src/{project_name}/utils/__init__.py",
+    f"src/{project_name}/utils/general.py",
+    f"src/{project_name}/logger/__init__.py",
+    f"src/{project_name}/config/__init__.py",
+    f"src/{project_name}/config/configuration.py",
+    f"src/{project_name}/pipeline/__init__.py",
+    f"src/{project_name}/entity/__init__.py",
+    f"src/{project_name}/constants/__init__.py",
+    f"src/{project_name}/exception/__init__.py",
+    "config/config.yaml",
+    "params/params.yaml",
+    "params.yaml",
+    "app.py",
+    "main.py",
+    "Dockerfile",
+    "requirements.txt",
+    "setup.py",
+    "check_code/trials.ipynb",
+
+]
+
+
+for filepath in list_of_files:
+    filepath = Path(filepath)
+    filedir, filename = os.path.split(filepath)
+
+    if filedir != "":
+        os.makedirs(filedir, exist_ok=True)
+        logging.info("Creating directory:{filedir} for the file {filename}")
+
+    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
+        with open(filepath,'w') as f:
+            logging.info("Creating empty file: {filepath}")
+
+    else:
+        logging.info("{filename} is already exists")

requirements.txt

@@ -0,0 +1,21 @@
+transformers
+transformers[sentencepiece]
+datasets
+sacrebleu
+rouge_score
+py7zr
+pandas
+nltk
+tqdm
+PyYAML
+matplotlib
+torch
+notebook
+boto3
+mypy-boto3-s3
+python-box==6.0.2
+ensure==1.0.2
+fastapi==0.78.0
+uvicorn==0.18.3
+Jinja2==3.1.2
+-e .

setup.py

@@ -0,0 +1,36 @@
+"""
+Make the project installable via pip. the project meta data are present here.
+"""
+
+import setuptools
+from setuptools import setup
+
+# Read the readme file, which will be our long description.
+with open("README.md", "r", encoding="utf-8") as file:
+    long_description = file.read()
+
+version: str = "1.0.0"
+repo_name: str = "Text-summarization-nlp"
+git_hub_username: str = "pavi-ninjaac"
+author_name: str = "Pavithra Devi M"
+author_email = "[email protected]"
+package_name: str = "TextSummarizer"
+
+
+# setup the project.
+setup(
+    name=package_name,
+    version=version,
+    author=author_name,
+    author_email=author_email,
+    description="A small package for text summarization",
+    long_description=long_description,
+    long_description_content="text/markdown",
+    url=f"https://github.com/{git_hub_username}/{repo_name}",
+    project_urls={
+        "Bug Tracker": f"https://github.com/{git_hub_username}/{repo_name}/issues",
+    },
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.8"
+)

src/TextSummarizer/components/data_ingestion.py

@@ -0,0 +1,25 @@
+import os
+
+from datasets import load_dataset
+
+from src.TextSummarizer.entity import entities
+
+
+class DataIngestionComponent:
+    """
+    A Class which is responsible for data ingestion.
+    """
+
+    def __init__(self, config: entities.DataIngestionConfig) -> None:
+        self.config = config
+
+    def save_dataset(self):
+        """
+        Load the dataset.
+        """
+        # if the dataset is already loaded then don't call it.
+        if os.path.exists(self.config.arrow_dataset_dir):
+            return
+
+        test_dataset = load_dataset(self.config.dataset_name)
+        test_dataset.save_to_disk(self.config.arrow_dataset_dir)

src/TextSummarizer/components/data_transformation.py

@@ -0,0 +1,45 @@
+"""
+Module which represents to the data validation step.
+"""
+
+import os
+
+from datasets import load_dataset, load_from_disk
+from transformers import AutoTokenizer
+
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class DataTransformation:
+    def __init__(self, config: entities.DataTransformationConfig):
+        self.config = config
+        self.tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name)
+
+    def convert_examples_to_features(self,example_batch):
+        """
+        Convert the examples to features.
+        """
+        input_encodings = self.tokenizer(example_batch['document'] , max_length = 800, truncation = True )
+
+        with self.tokenizer.as_target_tokenizer():
+            target_encodings = self.tokenizer(example_batch['summary'], max_length = 128, truncation = True )
+
+        return {
+            'input_ids' : input_encodings['input_ids'],
+            'attention_mask': input_encodings['attention_mask'],
+            'labels': target_encodings['input_ids']
+        }
+
+    def convert(self):
+        """
+        Tokenzie the dataset and store it to the disk.
+        """
+        backend_logger.info("Converting text to tokens....")
+        # Check if the dataset folder already exists.
+        if os.path.exists(os.path.join(self.config.root_dir,"dataset")):
+            return
+        dataset = load_from_disk(self.config.data_path)
+        dataset = dataset.map(self.convert_examples_to_features, batched = True)
+        dataset.save_to_disk(os.path.join(self.config.root_dir,"dataset"))
+        backend_logger.info("Converted text to tokens.")

src/TextSummarizer/components/data_validation.py

@@ -0,0 +1,36 @@
+import os
+
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class DataValidation:
+    def __init__(self, config: entities.DataValidationConfig):
+        self.config = config
+
+    def validate_all_files_exist(self) -> bool:
+        """
+        Check if all the required folders are present.
+        """
+        try:
+            validation_status: bool | None = None
+
+            all_folder = os.listdir(os.path.join("artifacts","data"))
+
+            for folder in all_folder:
+                print(folder)
+                if folder not in self.config.all_required_folders:
+                    validation_status = False
+                    with open(self.config.status_file, "w") as f:
+                        backend_logger.info("Writing the data validation status as False")
+                        f.write(f"Validation status: {validation_status}")
+                else:
+                    validation_status = True
+                    with open(self.config.status_file, "w") as f:
+                        backend_logger.info("Writing the data validation status as True")
+                        f.write(f"Validation status: {validation_status}")
+
+            return validation_status
+
+        except Exception as exp:
+            raise exp

src/TextSummarizer/components/model_evaluation.py

@@ -0,0 +1,87 @@
+import os
+
+import pandas as pd
+import torch
+from datasets import load_dataset, load_from_disk, load_metric
+from tqdm import tqdm
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class ModelEvaluation:
+    def __init__(self, config: entities.ModelEvaluationConfig):
+        self.config = config
+
+    def generate_batch_sized_chunks(self,list_of_elements, batch_size):
+        """split the dataset into smaller batches that we can process simultaneously
+        Yield successive batch-sized chunks from list_of_elements."""
+        for i in range(0, len(list_of_elements), batch_size):
+            yield list_of_elements[i : i + batch_size]
+
+    def calculate_metric_on_test_ds(self,
+                                    dataset,
+                                    metric,
+                                    model,
+                                    tokenizer,
+                                    batch_size=16,
+                                    device="cuda" if torch.cuda.is_available() else "cpu",
+                                    column_text="article",
+                                    column_summary="highlights"):
+        """
+        Calculate the metrics.
+        """
+        article_batches = list(self.generate_batch_sized_chunks(dataset[column_text], batch_size))
+        target_batches = list(self.generate_batch_sized_chunks(dataset[column_summary], batch_size))
+
+        for article_batch, target_batch in tqdm(
+            zip(article_batches, target_batches), total=len(article_batches)):
+
+            inputs = tokenizer(article_batch, max_length=1024,  truncation=True,
+                            padding="max_length", return_tensors="pt")
+
+            summaries = model.generate(input_ids=inputs["input_ids"].to(device),
+                            attention_mask=inputs["attention_mask"].to(device),
+                            length_penalty=0.8, num_beams=8, max_length=128)
+            ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''
+
+            # Finally, we decode the generated texts,
+            # replace the  token, and add the decoded texts with the references to the metric.
+            decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,
+                                 clean_up_tokenization_spaces=True)
+                                 for s in summaries]
+
+            decoded_summaries = [d.replace("", " ") for d in decoded_summaries]
+
+
+            metric.add_batch(predictions=decoded_summaries, references=target_batch)
+
+        #  Finally compute and return the ROUGE scores.
+        score = metric.compute()
+        return score
+
+    def run(self):
+        """
+        Run the model evaluation step.
+        """
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_path)
+        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path).to(device)
+
+        #loading data
+        dataset_samsum_pt = load_from_disk(self.config.data_path)
+
+
+        rouge_names = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
+
+        rouge_metric = load_metric('rouge')
+
+        score = self.calculate_metric_on_test_ds(
+        dataset_samsum_pt['test'][0:10], rouge_metric, model_pegasus, tokenizer, batch_size = 2, column_text = 'dialogue', column_summary= 'summary'
+            )
+
+        rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names )
+
+        df = pd.DataFrame(rouge_dict, index = ['pegasus'] )
+        df.to_csv(self.config.metric_file_name, index=False)

src/TextSummarizer/components/train_model.py

@@ -0,0 +1,70 @@
+import torch
+from datasets import load_from_disk
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    Trainer,
+    TrainingArguments,
+)
+
+from src.TextSummarizer.entity import entities
+
+
+class ModelTrainer:
+    """
+    Train a model.
+    """
+    def __init__(self, config: entities.ModelTrainerConfig):
+        self.config = config
+
+    def train(self):
+        """
+        Train the model.
+        """
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        tokenizer = AutoTokenizer.from_pretrained(self.config.model_ckpt)
+        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_ckpt).to(device)
+        seq2seq_data_collator = DataCollatorForSeq2Seq(tokenizer, model=model_pegasus)
+
+        #loading data
+        dataset = load_from_disk(self.config.data_path)
+
+        # trainer_args = TrainingArguments(
+        #     output_dir=self.config.root_dir, num_train_epochs=self.config.num_train_epochs, warmup_steps=self.config.warmup_steps,
+        #     per_device_train_batch_size=self.config.per_device_train_batch_size, per_device_eval_batch_size=self.config.per_device_train_batch_size,
+        #     weight_decay=self.config.weight_decay, logging_steps=self.config.logging_steps,
+        #     evaluation_strategy=self.config.evaluation_strategy, eval_steps=self.config.eval_steps, save_steps=1e6,
+        #     gradient_accumulation_steps=self.config.gradient_accumulation_steps
+        # )
+
+
+        trainer_args = TrainingArguments(
+            output_dir=self.config.root_dir,
+            num_train_epochs=1,
+            warmup_steps=500,
+            per_device_train_batch_size=1,
+            per_device_eval_batch_size=1,
+            weight_decay=0.01,
+            logging_steps=10,
+            evaluation_strategy='steps',
+            eval_steps=500,
+            save_steps=1e6,
+            gradient_accumulation_steps=16
+        )
+
+        trainer = Trainer(
+            model=model_pegasus,
+            args=trainer_args,
+            tokenizer=tokenizer,
+            data_collator=seq2seq_data_collator,
+            train_dataset=dataset["train"],
+            eval_dataset=dataset["validation"])
+
+        # trainer.train()
+
+        ## Save model
+        model_pegasus.save_pretrained(self.config.model_path)
+
+        ## Save tokenizer
+        tokenizer.save_pretrained(self.config.tokenizer_path)

src/TextSummarizer/config/config.yaml

@@ -0,0 +1,34 @@
+artifacts_root: artifacts
+
+
+data_ingestion:
+  dataset_name: "alexfabbri/multi_news"
+  arrow_dataset_dir: artifacts/data
+
+
+data_validation:
+  root_dir: artifacts/data_validation
+  status_file: artifacts/data_validation/status.txt
+  all_required_folders: ["train", "test", "validation", "dataset_dict.json"]
+
+
+data_transformation:
+  root_dir: artifacts/data_transformation
+  data_path: artifacts/data
+  tokenizer_name: google/pegasus-cnn_dailymail
+
+
+model_trainer:
+  root_dir: artifacts/model_trainer
+  data_path: artifacts/data_transformation/dataset
+  model_ckpt: google/pegasus-cnn_dailymail
+  model_path: artifacts/model_trainer/pegasus-samsum-model
+  tokenizer_path: artifacts/model_trainer/tokenizer
+
+model_evaluation:
+  root_dir: artifacts/model_evaluation
+  data_path: artifacts/data_transformation/dataset
+  model_path: artifacts/model_trainer/pegasus-samsum-model
+  tokenizer_path: artifacts/model_trainer/tokenizer
+  metric_file_name: artifacts/model_evaluation/metrics.csv
+  hub_model_name: pavithra-devi/pegasus-multi-news

src/TextSummarizer/config/config_manager.py

@@ -0,0 +1,117 @@
+"""
+Configuration manager to get and set all the configuration.
+"""
+
+from pathlib import Path
+
+from box import ConfigBox
+
+from src.TextSummarizer.constants import file_path
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.utils.general import create_directories, read_yaml
+
+
+# Create a config manager.
+class ConfigManager:
+    """
+    Class to manage the configuration files.
+    """
+
+    def __init__(self) -> None:
+        self.config: ConfigBox = read_yaml(Path(file_path.CONFIG_FILE_PATH))
+        self.params: ConfigBox = read_yaml(Path(file_path.PARAMS_FILE_PATH))
+
+        create_directories(path_to_directories=[self.config.artifacts_root])
+
+    def get_data_ingestion_config(self) -> entities.DataIngestionConfig:
+        """
+        Get the config which is needed to download the data files.
+        """
+        config: ConfigBox = self.config.data_ingestion
+
+        data_ingestion_config: entities.DataIngestionConfig = entities.DataIngestionConfig(
+            dataset_name=config.dataset_name,
+            arrow_dataset_dir=config.arrow_dataset_dir,
+        )
+
+        return data_ingestion_config
+
+    def get_data_validation_config(self) -> entities.DataValidationConfig:
+        """
+        Get the config which is needed to validate the data files.
+        """
+        config = self.config.data_validation
+
+        create_directories([config.root_dir])
+
+        data_validation_config = entities.DataValidationConfig(
+            root_dir=config.root_dir,
+            status_file=config.status_file,
+            all_required_folders=config.all_required_folders,
+        )
+
+        return data_validation_config
+
+    def get_data_transformation_config(self) -> entities.DataTransformationConfig:
+        """
+        Get teh data transformation configurations.
+        """
+        config = self.config.data_transformation
+
+        create_directories([config.root_dir])
+
+        data_transformation_config = entities.DataTransformationConfig(
+            root_dir=config.root_dir,
+            data_path=config.data_path,
+            tokenizer_name = config.tokenizer_name
+        )
+
+        return data_transformation_config
+
+    def get_model_trainer_config(self) -> entities.ModelTrainerConfig:
+        """
+        Get the configuration which is needed to train the model.
+        """
+        config = self.config.model_trainer
+        params = self.params.TrainingArguments
+
+        create_directories([config.root_dir])
+
+        model_trainer_config = entities.ModelTrainerConfig(
+            root_dir=config.root_dir,
+            data_path=config.data_path,
+            model_path= config.model_path,
+            tokenizer_path= config.tokenizer_path,
+            model_ckpt = config.model_ckpt,
+            num_train_epochs = params.num_train_epochs,
+            warmup_steps = params.warmup_steps,
+            per_device_train_batch_size = params.per_device_train_batch_size,
+            weight_decay = params.weight_decay,
+            logging_steps = params.logging_steps,
+            evaluation_strategy = params.evaluation_strategy,
+            eval_steps = params.evaluation_strategy,
+            save_steps = params.save_steps,
+            gradient_accumulation_steps = params.gradient_accumulation_steps
+        )
+
+        return model_trainer_config
+
+    def get_model_evaluation_config(self) -> entities.ModelEvaluationConfig:
+        """
+        Get the model evaluation configuration.
+        """
+        config = self.config.model_evaluation
+
+        create_directories([config.root_dir])
+
+        model_evaluation_config = entities.ModelEvaluationConfig(
+            root_dir=config.root_dir,
+            data_path=config.data_path,
+            model_path = config.model_path,
+            tokenizer_path = config.tokenizer_path,
+            metric_file_name = config.metric_file_name,
+            hub_model_name=config.hub_model_name
+
+        )
+
+        return model_evaluation_config

src/TextSummarizer/config/params.yaml

@@ -0,0 +1,10 @@
+TrainingArguments:
+  num_train_epochs: 1
+  warmup_steps: 500
+  per_device_train_batch_size: 1
+  weight_decay: 0.01
+  logging_steps: 10
+  evaluation_strategy: steps
+  eval_steps: 500
+  save_steps: 1e6
+  gradient_accumulation_steps: 16

src/TextSummarizer/constants/file_path.py

@@ -0,0 +1,8 @@
+"""
+Constants which is related to the file paths are present here.
+"""
+
+from typing import Final
+
+CONFIG_FILE_PATH: Final[str] = "src/TextSummarizer/config/config.yaml"
+PARAMS_FILE_PATH: Final[str] = "src/TextSummarizer/config/params.yaml"

src/TextSummarizer/entity/entities.py

@@ -0,0 +1,69 @@
+"""
+All the class return types are present here.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass()
+class DataIngestionConfig:
+    """
+    The return type of the data ingestion config function.
+    """
+    dataset_name: str
+    arrow_dataset_dir: str
+
+
+@dataclass()
+class DataValidationConfig:
+    """
+    Return type of the data validation config function.
+    """
+    root_dir: str
+    status_file: str
+    all_required_folders: list
+
+
+@dataclass(frozen=True)
+class DataTransformationConfig:
+    """
+    Return type of the data transformation config function.
+    """
+    root_dir: str
+    data_path: str
+    tokenizer_name: str
+
+
+@dataclass(frozen=True)
+class ModelTrainerConfig:
+    """
+    Return type of the model trainer config function.
+    """
+    root_dir: str
+    data_path: str
+    model_ckpt: str
+    model_path: str
+    tokenizer_path: str
+    num_train_epochs: int
+    warmup_steps: int
+    per_device_train_batch_size: int
+    weight_decay: float
+    logging_steps: int
+    evaluation_strategy: str
+    eval_steps: int
+    save_steps: float
+    gradient_accumulation_steps: int
+
+
+@dataclass(frozen=True)
+class ModelEvaluationConfig:
+    """
+    Return type of the model evaluation config function.
+    """
+    root_dir: str
+    data_path: str
+    model_path: str
+    tokenizer_path: str
+    metric_file_name: str
+    hub_model_name: str

src/TextSummarizer/logger/__init__.py

@@ -0,0 +1,43 @@
+"""
+Creating the logger needed for the project.
+"""
+
+import logging
+import os
+
+logging_str: str = "[%(asctime)s - %(levelname)s - %(module)s.py - %(name)s]: %(message)s"
+
+# Create the log directory.
+log_dir: str = "logs"
+backend_log_filename: str = "back_end.log"
+frontend_log_filename: str = "front_end.log"
+backend_log_filepath: str = os.path.join(log_dir, backend_log_filename)
+frontend_log_filepath: str = os.path.join(log_dir, frontend_log_filename)
+
+os.makedirs(log_dir, exist_ok=True)
+
+# Set the logging config.
+logging.basicConfig(
+    level=logging.DEBUG,
+    format=logging_str,
+    # handlers=[
+    #     logging.FileHandler(log_filepath),
+    #     logging.StreamHandler(sys.stdout)
+    # ]
+)
+
+# Get the logger.
+backend_file_handler = logging.FileHandler(backend_log_filepath)
+frontend_file_handler = logging.FileHandler(frontend_log_filepath)
+
+# add the formettor for the handler,
+backend_file_handler.setFormatter(logging.Formatter(logging_str))
+frontend_file_handler.setFormatter(logging.Formatter(logging_str))
+
+backend_logger = logging.getLogger("backend")
+frontend_logger = logging.getLogger("frontend")
+
+
+# add the handlers.
+backend_logger.addHandler(backend_file_handler)
+frontend_logger.addHandler(frontend_file_handler)

src/TextSummarizer/pipeline/prediction.py

@@ -0,0 +1,25 @@
+from transformers import pipeline
+
+from src.TextSummarizer.config.config_manager import ConfigManager
+
+
+class PredictionPipeline:
+    def __init__(self):
+        self.config = ConfigManager().get_model_evaluation_config()
+
+    def predict(self,text):
+        """
+        Predict the tex summarization for the given text.
+        """
+        gen_kwargs = {"length_penalty": 0.8, "num_beams":8, "max_length": 128}
+
+        summarizer = pipeline("summarization", model=self.config.hub_model_name)
+
+        print("document:")
+        print(text)
+
+        output = summarizer(text, **gen_kwargs)[0]["summary_text"]
+        print("\nModel Summary:")
+        print(output)
+
+        return output

src/TextSummarizer/pipeline/step_01_data_ingestion.py

@@ -0,0 +1,25 @@
+"""
+The data ingestion pipeline.
+"""
+
+from src.TextSummarizer.components.data_ingestion import DataIngestionComponent
+from src.TextSummarizer.config.config_manager import ConfigManager
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class DataIngestionPipeline:
+    """
+    The data ingestion pipeline.
+    """
+
+    def run(self):
+        """
+        The main function of the data ingestion pipeline.
+        """
+        backend_logger.info("Starting the data ingestion pipeline.")
+        config: ConfigManager = ConfigManager()
+        data_ingestion_config: entities.DataIngestionConfig = config.get_data_ingestion_config()
+        data_ingestion = DataIngestionComponent(config=data_ingestion_config)
+        data_ingestion.save_dataset()
+        backend_logger.info("Finished the data ingestion pipeline.")

src/TextSummarizer/pipeline/step_02_data_validation.py

@@ -0,0 +1,25 @@
+"""
+The data validation pipeline.
+"""
+
+from src.TextSummarizer.components.data_validation import DataValidation
+from src.TextSummarizer.config.config_manager import ConfigManager
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class DataValidationPipeline:
+    """
+    The data validation pipeline.
+    """
+
+    def run(self):
+        """
+        The main function of the data validation pipeline.
+        """
+        backend_logger.info("Starting the data validation pipeline.")
+        config: ConfigManager = ConfigManager()
+        data_ingestion_config: entities.DataIngestionConfig = config.get_data_validation_config()
+        data_ingestion = DataValidation(config=data_ingestion_config)
+        data_ingestion.validate_all_files_exist()
+        backend_logger.info("Finished the data validation pipeline.")

src/TextSummarizer/pipeline/step_03_data_transformation.py

@@ -0,0 +1,25 @@
+"""
+The data Transformation pipeline.
+"""
+
+from src.TextSummarizer.components.data_transformation import DataTransformation
+from src.TextSummarizer.config.config_manager import ConfigManager
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class DataTransformationPipeline:
+    """
+    The data Transformation pipeline.
+    """
+
+    def run(self):
+        """
+        The main function of the data Transformation pipeline.
+        """
+        backend_logger.info("Starting the data Transformation pipeline.")
+        config: ConfigManager = ConfigManager()
+        data_transformation_config: entities.DataTransformationConfig = config.get_data_transformation_config()
+        data_transformation = DataTransformation(config=data_transformation_config)
+        data_transformation.convert()
+        backend_logger.info("Finished the data Transformation pipeline.")

src/TextSummarizer/pipeline/step_04_train_model.py

@@ -0,0 +1,25 @@
+"""
+The data Transformation pipeline.
+"""
+
+from src.TextSummarizer.components.train_model import ModelTrainer
+from src.TextSummarizer.config.config_manager import ConfigManager
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class ModelTrainerPipeline:
+    """
+    The data Transformation pipeline.
+    """
+
+    def run(self):
+        """
+        The main function of the train model pipeline.
+        """
+        backend_logger.info("Starting the train model pipeline.")
+        config: ConfigManager = ConfigManager()
+        train_model_config: entities.DataTransformationConfig = config.get_model_trainer_config()
+        train_model = ModelTrainer(config=train_model_config)
+        train_model.train()
+        backend_logger.info("Finished the train model pipeline.")

src/TextSummarizer/pipeline/step_05_model_evaluation.py

@@ -0,0 +1,25 @@
+"""
+The data Transformation pipeline.
+"""
+
+from src.TextSummarizer.components.model_evaluation import ModelEvaluation
+from src.TextSummarizer.config.config_manager import ConfigManager
+from src.TextSummarizer.entity import entities
+from src.TextSummarizer.logger import backend_logger
+
+
+class ModelEvaluationPipeline:
+    """
+    The model evaluation pipeline.
+    """
+
+    def run(self):
+        """
+        The main function of the model evaluation pipeline.
+        """
+        backend_logger.info("Starting the  model evaluation pipeline.")
+        config = ConfigManager()
+        model_evaluation_config = config.get_model_evaluation_config()
+        model_evaluation_config = ModelEvaluation(config=model_evaluation_config)
+        model_evaluation_config.evaluate()
+        backend_logger.info("Finished the model evaluation pipeline.")

src/TextSummarizer/utils/general.py

@@ -0,0 +1,56 @@
+"""
+The functions used throughout the project is present here.
+"""
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+from box import ConfigBox
+from box.exceptions import BoxValueError
+from ensure import ensure_annotations
+
+from src.TextSummarizer.logger import backend_logger
+
+
+@ensure_annotations
+def read_yaml(path_to_yaml: Path) -> ConfigBox:
+    """
+    Read yaml file and return as Dictionary.
+
+    :param path_to_yaml: Path to yaml file.
+    :return: A ConfigBox dictionary object containing the the yaml file contents.
+    """
+    try:
+        with open(path_to_yaml) as yaml_file:
+            content = yaml.safe_load(yaml_file)
+            backend_logger.info(f"yaml file: {path_to_yaml} loaded successfully")
+            return ConfigBox(content)
+    except BoxValueError:
+        raise ValueError(f"yaml file: {path_to_yaml} is empty.")
+    except Exception as exp:
+        raise exp
+
+
+def create_directories(path_to_directories: list) -> None:
+    """
+    create list of directories.
+
+    :params path_to_directories: list of path of directories.
+    """
+    for path in path_to_directories:
+        os.makedirs(path, exist_ok=True)
+        backend_logger.info(f"created directory at: {path}")
+
+
+@ensure_annotations
+def get_size(path: Path) -> str:
+    """
+    Get the file size in KB.
+
+    :param path: Path of the file.
+    :returns: Size in KB.
+    """
+    size_in_kb = round(os.path.getsize(path)/1024)
+    return f"~ {size_in_kb} KB"