{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":9585160,"sourceType":"datasetVersion","datasetId":5845201}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from datasets import load_dataset","metadata":{"execution":{"iopub.status.busy":"2024-10-13T03:59:54.293786Z","iopub.execute_input":"2024-10-13T03:59:54.294192Z","iopub.status.idle":"2024-10-13T03:59:56.257624Z","shell.execute_reply.started":"2024-10-13T03:59:54.294138Z","shell.execute_reply":"2024-10-13T03:59:56.256580Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"!pip install sentencepiece\n!pip install transformers\n!pip install datasets evaluate\n!pip install sacrebleu","metadata":{"execution":{"iopub.status.busy":"2024-10-13T03:59:56.261528Z","iopub.execute_input":"2024-10-13T03:59:56.261989Z","iopub.status.idle":"2024-10-13T04:00:47.093610Z","shell.execute_reply.started":"2024-10-13T03:59:56.261955Z","shell.execute_reply":"2024-10-13T04:00:47.092408Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (0.2.0)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.45.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.15.1)\nRequirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.25.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.2)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2024.5.15)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.32.3)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.5)\nRequirement already satisfied: tokenizers<0.21,>=0.20 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.20.0)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.66.4)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.6.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2024.8.30)\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (3.0.1)\nCollecting evaluate\n Downloading evaluate-0.4.3-py3-none-any.whl.metadata (9.2 kB)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from datasets) (3.15.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.26.4)\nRequirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (16.1.0)\nRequirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.2.2)\nRequirement already satisfied: requests>=2.32.2 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.32.3)\nRequirement already satisfied: tqdm>=4.66.3 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.66.4)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets) (2024.6.1)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.9.5)\nRequirement already satisfied: huggingface-hub>=0.22.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.25.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0.2)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.22.0->datasets) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (2024.8.30)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nDownloading evaluate-0.4.3-py3-none-any.whl (84 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.0/84.0 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hInstalling collected packages: evaluate\nSuccessfully installed evaluate-0.4.3\nCollecting sacrebleu\n Downloading sacrebleu-2.4.3-py3-none-any.whl.metadata (51 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.8/51.8 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting portalocker (from sacrebleu)\n Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\nRequirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (2024.5.15)\nRequirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (1.26.4)\nRequirement already satisfied: colorama in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.4.6)\nRequirement already satisfied: lxml in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (5.3.0)\nDownloading sacrebleu-2.4.3-py3-none-any.whl (103 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.0/104.0 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading portalocker-2.10.1-py3-none-any.whl (18 kB)\nInstalling collected packages: portalocker, sacrebleu\nSuccessfully installed portalocker-2.10.1 sacrebleu-2.4.3\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install wandb -qqq\nimport wandb\nwandb.login()","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:00:47.095135Z","iopub.execute_input":"2024-10-13T04:00:47.095477Z","iopub.status.idle":"2024-10-13T04:01:19.743885Z","shell.execute_reply.started":"2024-10-13T04:00:47.095441Z","shell.execute_reply":"2024-10-13T04:01:19.742777Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:","output_type":"stream"},{"output_type":"stream","name":"stdin","text":" ········································\n"},{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n","output_type":"stream"},{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"# Initialize wandb.run first\nwandb.init()\n\n# If cell outputs wandb.run, you'll see live graphs\nwandb.run","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:19.746153Z","iopub.execute_input":"2024-10-13T04:01:19.746742Z","iopub.status.idle":"2024-10-13T04:01:23.136029Z","shell.execute_reply.started":"2024-10-13T04:01:19.746702Z","shell.execute_reply":"2024-10-13T04:01:23.135072Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msarch7040\u001b[0m (\u001b[33msarch7040-\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011112782366666983, max=1.0…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3c66afed2b82401bbe5179218d4ef737"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":"Tracking run with wandb version 0.18.3"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":"Run data is saved locally in /kaggle/working/wandb/run-20241013_040119-uj3tsimu"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":"Syncing run grateful-snow-4 to Weights & Biases (docs)
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":" View project at https://wandb.ai/sarch7040-/uncategorized"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"","text/html":" View run at https://wandb.ai/sarch7040-/uncategorized/runs/uj3tsimu"},"metadata":{}},{"execution_count":4,"output_type":"execute_result","data":{"text/html":"","text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"from transformers import AutoTokenizer, M2M100ForConditionalGeneration,Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq,pipeline\nfrom huggingface_hub import notebook_login\nfrom datasets import load_dataset\nimport evaluate\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:23.137721Z","iopub.execute_input":"2024-10-13T04:01:23.138493Z","iopub.status.idle":"2024-10-13T04:01:43.683149Z","shell.execute_reply.started":"2024-10-13T04:01:23.138456Z","shell.execute_reply":"2024-10-13T04:01:43.682181Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"model = M2M100ForConditionalGeneration.from_pretrained(\"facebook/m2m100_418M\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/m2m100_418M\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:43.684472Z","iopub.execute_input":"2024-10-13T04:01:43.685094Z","iopub.status.idle":"2024-10-13T04:01:55.228054Z","shell.execute_reply.started":"2024-10-13T04:01:43.685058Z","shell.execute_reply":"2024-10-13T04:01:55.227221Z"},"trusted":true},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json: 0%| | 0.00/908 [00:00=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2024.6.1)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (6.0.2)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2.32.3)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.66.4)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface_hub) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2024.8.30)\nThe token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\nToken is valid (permission: fineGrained).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n","output_type":"stream"}]},{"cell_type":"code","source":"from datasets import load_dataset\nimport unicodedata\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# Load dataset\ndataset = load_dataset(\"csv\", data_files=\"/kaggle/input/praendataset/prakrit_translation.csv\")\n\n# Preprocessing function\ndef preprocess_function(examples):\n # Helper function to remove punctuation using Unicode categories\n def remove_punctuation(text):\n return ''.join(\n char for char in text\n if not unicodedata.category(char).startswith('P')\n )\n \n # Remove punctuation from English and Prakrit sentences\n examples['english'] = [remove_punctuation(sentence) for sentence in examples['english']]\n examples['prakrit'] = [remove_punctuation(sentence) for sentence in examples['prakrit']]\n \n return examples\n\n# Apply preprocessing\ndataset = dataset.map(preprocess_function, batched=True)\n\n\n# Tokenization function\nmax_length = 128\ndef tokenization(examples):\n inputs = examples[\"prakrit\"]\n targets = examples[\"english\"]\n \n # Tokenize inputs and targets\n model_inputs = tokenizer(inputs, max_length=max_length, truncation=True, padding=\"max_length\")\n labels = tokenizer(targets, max_length=max_length, truncation=True, padding=\"max_length\")\n \n # Set labels in the tokenized inputs\n model_inputs[\"labels\"] = labels[\"input_ids\"]\n \n return model_inputs\n\n# Apply tokenization to the dataset\ntokenized_dataset = dataset.map(tokenization, batched=True)\n\n# Display some tokenized data\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:41.767501Z","iopub.execute_input":"2024-10-13T04:02:41.767865Z","iopub.status.idle":"2024-10-13T04:02:45.646354Z","shell.execute_reply.started":"2024-10-13T04:02:41.767825Z","shell.execute_reply":"2024-10-13T04:02:45.645429Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a9dc86182afd45f7a3c2009cddaea0ca"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Map: 0%| | 0/1474 [00:00=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.0.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.26.4)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.2.2)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.32.3)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.4)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2024.6.1)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.25.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.15.1)\nRequirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (16.1.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.9.5)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (6.0.2)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2024.8.30)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2024.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.2.0)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.5)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\n","output_type":"stream"}]},{"cell_type":"code","source":"from transformers import AutoTokenizer, M2M100ForConditionalGeneration,Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq,pipeline\nfrom huggingface_hub import notebook_login\nfrom datasets import load_dataset\nimport evaluate\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:58.552639Z","iopub.execute_input":"2024-10-13T04:02:58.552992Z","iopub.status.idle":"2024-10-13T04:02:58.559634Z","shell.execute_reply.started":"2024-10-13T04:02:58.552953Z","shell.execute_reply":"2024-10-13T04:02:58.558454Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"model = M2M100ForConditionalGeneration.from_pretrained(\"facebook/m2m100_418M\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/m2m100_418M\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:58.560923Z","iopub.execute_input":"2024-10-13T04:02:58.561376Z","iopub.status.idle":"2024-10-13T04:02:59.695353Z","shell.execute_reply.started":"2024-10-13T04:02:58.561325Z","shell.execute_reply":"2024-10-13T04:02:59.694505Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"from transformers import DataCollatorForSeq2Seq\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.696748Z","iopub.execute_input":"2024-10-13T04:02:59.697199Z","iopub.status.idle":"2024-10-13T04:02:59.703602Z","shell.execute_reply.started":"2024-10-13T04:02:59.697150Z","shell.execute_reply":"2024-10-13T04:02:59.702530Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"# Perform train-test split with 20% of the data as the test set\ntokenized_dataset = tokenized_dataset[\"train\"].train_test_split(test_size=0.2)\n\n# Check the split\ntokenized_dataset\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.704774Z","iopub.execute_input":"2024-10-13T04:02:59.705095Z","iopub.status.idle":"2024-10-13T04:02:59.730080Z","shell.execute_reply.started":"2024-10-13T04:02:59.705062Z","shell.execute_reply":"2024-10-13T04:02:59.729251Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"DatasetDict({\n train: Dataset({\n features: ['english', 'prakrit', 'input_ids', 'attention_mask', 'labels'],\n num_rows: 1179\n })\n test: Dataset({\n features: ['english', 'prakrit', 'input_ids', 'attention_mask', 'labels'],\n num_rows: 295\n })\n})"},"metadata":{}}]},{"cell_type":"code","source":"!pip install --upgrade nltk\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.731136Z","iopub.execute_input":"2024-10-13T04:02:59.731423Z","iopub.status.idle":"2024-10-13T04:03:14.021271Z","shell.execute_reply.started":"2024-10-13T04:02:59.731392Z","shell.execute_reply":"2024-10-13T04:03:14.020343Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (3.2.4)\nCollecting nltk\n Downloading nltk-3.9.1-py3-none-any.whl.metadata (2.9 kB)\nRequirement already satisfied: click in /opt/conda/lib/python3.10/site-packages (from nltk) (8.1.7)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk) (1.4.2)\nRequirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.10/site-packages (from nltk) (2024.5.15)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from nltk) (4.66.4)\nDownloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n\u001b[?25hInstalling collected packages: nltk\n Attempting uninstall: nltk\n Found existing installation: nltk 3.2.4\n Uninstalling nltk-3.2.4:\n Successfully uninstalled nltk-3.2.4\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\npreprocessing 0.1.13 requires nltk==3.2.4, but you have nltk 3.9.1 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed nltk-3.9.1\n","output_type":"stream"}]},{"cell_type":"code","source":"import nltk\nimport evaluate\n\n# Download required NLTK datasets\nnltk.download('wordnet')\nnltk.download('omw-1.4')\n\n# Load metrics\nbleu_metric = evaluate.load(\"sacrebleu\")\nmeteor_metric = evaluate.load(\"meteor\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:03:14.022706Z","iopub.execute_input":"2024-10-13T04:03:14.023036Z","iopub.status.idle":"2024-10-13T04:03:16.255060Z","shell.execute_reply.started":"2024-10-13T04:03:14.022994Z","shell.execute_reply":"2024-10-13T04:03:16.254163Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stderr","text":"[nltk_data] Downloading package wordnet to /usr/share/nltk_data...\n[nltk_data] Package wordnet is already up-to-date!\n[nltk_data] Downloading package omw-1.4 to /usr/share/nltk_data...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading builder script: 0%| | 0.00/8.15k [00:00","text/html":"\n
\n \n \n [1480/1480 1:33:13, Epoch 20/20]\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
EpochTraining LossValidation LossBleuMeteorGen Len
1No log5.8260861.1786000.19970036.383100
2No log4.6169932.6480000.26570037.606800
3No log3.5127935.7069000.32170032.216900
4No log2.5281026.3134000.35470031.857600
5No log1.7177248.5036000.38000029.972900
6No log1.16663010.1169000.39250028.067800
73.5641000.87019410.4207000.42460031.105100
83.5641000.73762612.6153000.43100028.633900
93.5641000.69009213.2966000.45030029.237300
103.5641000.67126511.9772000.43960030.566100
113.5641000.66506014.0436000.45060030.200000
123.5641000.66784313.2632000.45140031.054200
133.5641000.66775014.0924000.45630029.278000
140.5121000.66928914.7460000.46510028.406800
150.5121000.66984514.9278000.46770028.515300
160.5121000.67003814.7431000.46740028.928800
170.5121000.67438115.2934000.47010028.867800
180.5121000.67410215.6776000.47120028.349200
190.5121000.67723214.9420000.47070028.969500
200.5121000.67657215.3416000.47230028.027100

"},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:2618: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 200, 'early_stopping': True, 'num_beams': 5}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n warnings.warn(\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"TrainOutput(global_step=1480, training_loss=1.4915999850711308, metrics={'train_runtime': 5596.282, 'train_samples_per_second': 4.214, 'train_steps_per_second': 0.264, 'total_flos': 6387540814725120.0, 'train_loss': 1.4915999850711308, 'epoch': 20.0})"},"metadata":{}}]},{"cell_type":"code","source":"trainer.push_to_hub()","metadata":{"execution":{"iopub.status.busy":"2024-10-13T05:36:34.582611Z","iopub.execute_input":"2024-10-13T05:36:34.583362Z","iopub.status.idle":"2024-10-13T05:36:53.872348Z","shell.execute_reply.started":"2024-10-13T05:36:34.583312Z","shell.execute_reply":"2024-10-13T05:36:53.871321Z"},"trusted":true},"execution_count":21,"outputs":[{"output_type":"display_data","data":{"text/plain":"events.out.tfevents.1728792198.7afee8f5ccfd.30.0: 0%| | 0.00/14.5k [00:00