Removed Compromized keys and tokens

Model Training/final-method-using-m2m100.ipynb

@@ -1 +0,0 @@
-{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":9585160,"sourceType":"datasetVersion","datasetId":5845201}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from datasets import load_dataset","metadata":{"execution":{"iopub.status.busy":"2024-10-13T03:59:54.293786Z","iopub.execute_input":"2024-10-13T03:59:54.294192Z","iopub.status.idle":"2024-10-13T03:59:56.257624Z","shell.execute_reply.started":"2024-10-13T03:59:54.294138Z","shell.execute_reply":"2024-10-13T03:59:56.256580Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"!pip install sentencepiece\n!pip install transformers\n!pip install datasets evaluate\n!pip install sacrebleu","metadata":{"execution":{"iopub.status.busy":"2024-10-13T03:59:56.261528Z","iopub.execute_input":"2024-10-13T03:59:56.261989Z","iopub.status.idle":"2024-10-13T04:00:47.093610Z","shell.execute_reply.started":"2024-10-13T03:59:56.261955Z","shell.execute_reply":"2024-10-13T04:00:47.092408Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (0.2.0)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.45.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.15.1)\nRequirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.25.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.2)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2024.5.15)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.32.3)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.5)\nRequirement already satisfied: tokenizers<0.21,>=0.20 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.20.0)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.66.4)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.6.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2024.8.30)\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (3.0.1)\nCollecting evaluate\n  Downloading evaluate-0.4.3-py3-none-any.whl.metadata (9.2 kB)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from datasets) (3.15.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.26.4)\nRequirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (16.1.0)\nRequirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.2.2)\nRequirement already satisfied: requests>=2.32.2 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.32.3)\nRequirement already satisfied: tqdm>=4.66.3 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.66.4)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets) (2024.6.1)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.9.5)\nRequirement already satisfied: huggingface-hub>=0.22.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.25.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0.2)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.22.0->datasets) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.32.2->datasets) (2024.8.30)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nDownloading evaluate-0.4.3-py3-none-any.whl (84 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.0/84.0 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hInstalling collected packages: evaluate\nSuccessfully installed evaluate-0.4.3\nCollecting sacrebleu\n  Downloading sacrebleu-2.4.3-py3-none-any.whl.metadata (51 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.8/51.8 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting portalocker (from sacrebleu)\n  Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\nRequirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (2024.5.15)\nRequirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (1.26.4)\nRequirement already satisfied: colorama in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (0.4.6)\nRequirement already satisfied: lxml in /opt/conda/lib/python3.10/site-packages (from sacrebleu) (5.3.0)\nDownloading sacrebleu-2.4.3-py3-none-any.whl (103 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.0/104.0 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading portalocker-2.10.1-py3-none-any.whl (18 kB)\nInstalling collected packages: portalocker, sacrebleu\nSuccessfully installed portalocker-2.10.1 sacrebleu-2.4.3\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install wandb -qqq\nimport wandb\nwandb.login()","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:00:47.095135Z","iopub.execute_input":"2024-10-13T04:00:47.095477Z","iopub.status.idle":"2024-10-13T04:01:19.743885Z","shell.execute_reply.started":"2024-10-13T04:00:47.095441Z","shell.execute_reply":"2024-10-13T04:01:19.742777Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:","output_type":"stream"},{"output_type":"stream","name":"stdin","text":"  ········································\n"},{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n","output_type":"stream"},{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"# Initialize wandb.run first\nwandb.init()\n\n# If cell outputs wandb.run, you'll see live graphs\nwandb.run","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:19.746153Z","iopub.execute_input":"2024-10-13T04:01:19.746742Z","iopub.status.idle":"2024-10-13T04:01:23.136029Z","shell.execute_reply.started":"2024-10-13T04:01:19.746702Z","shell.execute_reply":"2024-10-13T04:01:23.135072Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msarch7040\u001b[0m (\u001b[33msarch7040-\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011112782366666983, max=1.0…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3c66afed2b82401bbe5179218d4ef737"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.18.3"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20241013_040119-uj3tsimu</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/sarch7040-/uncategorized/runs/uj3tsimu' target=\"_blank\">grateful-snow-4</a></strong> to <a href='https://wandb.ai/sarch7040-/uncategorized' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/sarch7040-/uncategorized' target=\"_blank\">https://wandb.ai/sarch7040-/uncategorized</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/sarch7040-/uncategorized/runs/uj3tsimu' target=\"_blank\">https://wandb.ai/sarch7040-/uncategorized/runs/uj3tsimu</a>"},"metadata":{}},{"execution_count":4,"output_type":"execute_result","data":{"text/html":"<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/sarch7040-/uncategorized/runs/uj3tsimu?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>","text/plain":"<wandb.sdk.wandb_run.Run at 0x7d48c92cd0c0>"},"metadata":{}}]},{"cell_type":"code","source":"from transformers import AutoTokenizer, M2M100ForConditionalGeneration,Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq,pipeline\nfrom huggingface_hub import notebook_login\nfrom datasets import load_dataset\nimport evaluate\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:23.137721Z","iopub.execute_input":"2024-10-13T04:01:23.138493Z","iopub.status.idle":"2024-10-13T04:01:43.683149Z","shell.execute_reply.started":"2024-10-13T04:01:23.138456Z","shell.execute_reply":"2024-10-13T04:01:43.682181Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"model = M2M100ForConditionalGeneration.from_pretrained(\"facebook/m2m100_418M\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/m2m100_418M\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:01:43.684472Z","iopub.execute_input":"2024-10-13T04:01:43.685094Z","iopub.status.idle":"2024-10-13T04:01:55.228054Z","shell.execute_reply.started":"2024-10-13T04:01:43.685058Z","shell.execute_reply":"2024-10-13T04:01:55.227221Z"},"trusted":true},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/908 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"36f8129324bf4c3daf16a886e2e0118e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"pytorch_model.bin:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6096c80a15404ae5bc4ad9b5a993a73e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/233 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1933610ef18145efb94d7f8a43d7a367"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/298 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dea7f7584c794371ad5ddb8e746286e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.json:   0%|          | 0.00/3.71M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"61d8b86b22664b2e95628c17914c15d4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/2.42M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dc57afa9ee304cb89b17bddc7c22ae6a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/1.14k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f712c792c5fb486dbb3dc38759837e2f"}},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n  warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install huggingface_hub\n\nfrom huggingface_hub import login\n\n# Prompt for your Hugging Face token\ntoken = \"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\"\n\n# Log in to Hugging Face\nlogin(token)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:29.639557Z","iopub.execute_input":"2024-10-13T04:02:29.639955Z","iopub.status.idle":"2024-10-13T04:02:41.765380Z","shell.execute_reply.started":"2024-10-13T04:02:29.639917Z","shell.execute_reply":"2024-10-13T04:02:41.764308Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"Requirement already satisfied: huggingface_hub in /opt/conda/lib/python3.10/site-packages (0.25.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (3.15.1)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2024.6.1)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (6.0.2)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2.32.3)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.66.4)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.9->huggingface_hub) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2024.8.30)\nThe token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\nToken is valid (permission: fineGrained).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n","output_type":"stream"}]},{"cell_type":"code","source":"from datasets import load_dataset\nimport unicodedata\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# Load dataset\ndataset = load_dataset(\"csv\", data_files=\"/kaggle/input/praendataset/prakrit_translation.csv\")\n\n# Preprocessing function\ndef preprocess_function(examples):\n    # Helper function to remove punctuation using Unicode categories\n    def remove_punctuation(text):\n        return ''.join(\n            char for char in text\n            if not unicodedata.category(char).startswith('P')\n        )\n    \n    # Remove punctuation from English and Prakrit sentences\n    examples['english'] = [remove_punctuation(sentence) for sentence in examples['english']]\n    examples['prakrit'] = [remove_punctuation(sentence) for sentence in examples['prakrit']]\n    \n    return examples\n\n# Apply preprocessing\ndataset = dataset.map(preprocess_function, batched=True)\n\n\n# Tokenization function\nmax_length = 128\ndef tokenization(examples):\n    inputs = examples[\"prakrit\"]\n    targets = examples[\"english\"]\n    \n    # Tokenize inputs and targets\n    model_inputs = tokenizer(inputs, max_length=max_length, truncation=True, padding=\"max_length\")\n    labels = tokenizer(targets, max_length=max_length, truncation=True, padding=\"max_length\")\n    \n    # Set labels in the tokenized inputs\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    \n    return model_inputs\n\n# Apply tokenization to the dataset\ntokenized_dataset = dataset.map(tokenization, batched=True)\n\n# Display some tokenized data\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:41.767501Z","iopub.execute_input":"2024-10-13T04:02:41.767865Z","iopub.status.idle":"2024-10-13T04:02:45.646354Z","shell.execute_reply.started":"2024-10-13T04:02:41.767825Z","shell.execute_reply":"2024-10-13T04:02:45.645429Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a9dc86182afd45f7a3c2009cddaea0ca"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Map:   0%|          | 0/1474 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ad976916753f4d8a88b85b70ecdee2f6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Map:   0%|          | 0/1474 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3df6b5cb800d4af09bdce7ce7673c939"}},"metadata":{}}]},{"cell_type":"code","source":"tokenized_dataset","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:45.648411Z","iopub.execute_input":"2024-10-13T04:02:45.649091Z","iopub.status.idle":"2024-10-13T04:02:45.656029Z","shell.execute_reply.started":"2024-10-13T04:02:45.649033Z","shell.execute_reply":"2024-10-13T04:02:45.655083Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"DatasetDict({\n    train: Dataset({\n        features: ['english', 'prakrit', 'input_ids', 'attention_mask', 'labels'],\n        num_rows: 1474\n    })\n})"},"metadata":{}}]},{"cell_type":"code","source":"!pip install --upgrade evaluate\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:45.657315Z","iopub.execute_input":"2024-10-13T04:02:45.657702Z","iopub.status.idle":"2024-10-13T04:02:58.549969Z","shell.execute_reply.started":"2024-10-13T04:02:45.657657Z","shell.execute_reply":"2024-10-13T04:02:58.548823Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/pty.py:89: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n  pid, fd = os.forkpty()\n","output_type":"stream"},{"name":"stdout","text":"Requirement already satisfied: evaluate in /opt/conda/lib/python3.10/site-packages (0.4.3)\nRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.0.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.26.4)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.2.2)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.32.3)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.4)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2024.6.1)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.25.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.15.1)\nRequirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (16.1.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.9.5)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (6.0.2)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.12.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2024.8.30)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2024.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.2.0)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.5)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\n","output_type":"stream"}]},{"cell_type":"code","source":"from transformers import AutoTokenizer, M2M100ForConditionalGeneration,Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq,pipeline\nfrom huggingface_hub import notebook_login\nfrom datasets import load_dataset\nimport evaluate\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:58.552639Z","iopub.execute_input":"2024-10-13T04:02:58.552992Z","iopub.status.idle":"2024-10-13T04:02:58.559634Z","shell.execute_reply.started":"2024-10-13T04:02:58.552953Z","shell.execute_reply":"2024-10-13T04:02:58.558454Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"model = M2M100ForConditionalGeneration.from_pretrained(\"facebook/m2m100_418M\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/m2m100_418M\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:58.560923Z","iopub.execute_input":"2024-10-13T04:02:58.561376Z","iopub.status.idle":"2024-10-13T04:02:59.695353Z","shell.execute_reply.started":"2024-10-13T04:02:58.561325Z","shell.execute_reply":"2024-10-13T04:02:59.694505Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n  warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"from transformers import DataCollatorForSeq2Seq\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.696748Z","iopub.execute_input":"2024-10-13T04:02:59.697199Z","iopub.status.idle":"2024-10-13T04:02:59.703602Z","shell.execute_reply.started":"2024-10-13T04:02:59.697150Z","shell.execute_reply":"2024-10-13T04:02:59.702530Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"# Perform train-test split with 20% of the data as the test set\ntokenized_dataset = tokenized_dataset[\"train\"].train_test_split(test_size=0.2)\n\n# Check the split\ntokenized_dataset\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.704774Z","iopub.execute_input":"2024-10-13T04:02:59.705095Z","iopub.status.idle":"2024-10-13T04:02:59.730080Z","shell.execute_reply.started":"2024-10-13T04:02:59.705062Z","shell.execute_reply":"2024-10-13T04:02:59.729251Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"DatasetDict({\n    train: Dataset({\n        features: ['english', 'prakrit', 'input_ids', 'attention_mask', 'labels'],\n        num_rows: 1179\n    })\n    test: Dataset({\n        features: ['english', 'prakrit', 'input_ids', 'attention_mask', 'labels'],\n        num_rows: 295\n    })\n})"},"metadata":{}}]},{"cell_type":"code","source":"!pip install --upgrade nltk\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:02:59.731136Z","iopub.execute_input":"2024-10-13T04:02:59.731423Z","iopub.status.idle":"2024-10-13T04:03:14.021271Z","shell.execute_reply.started":"2024-10-13T04:02:59.731392Z","shell.execute_reply":"2024-10-13T04:03:14.020343Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (3.2.4)\nCollecting nltk\n  Downloading nltk-3.9.1-py3-none-any.whl.metadata (2.9 kB)\nRequirement already satisfied: click in /opt/conda/lib/python3.10/site-packages (from nltk) (8.1.7)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk) (1.4.2)\nRequirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.10/site-packages (from nltk) (2024.5.15)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from nltk) (4.66.4)\nDownloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n\u001b[?25hInstalling collected packages: nltk\n  Attempting uninstall: nltk\n    Found existing installation: nltk 3.2.4\n    Uninstalling nltk-3.2.4:\n      Successfully uninstalled nltk-3.2.4\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\npreprocessing 0.1.13 requires nltk==3.2.4, but you have nltk 3.9.1 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed nltk-3.9.1\n","output_type":"stream"}]},{"cell_type":"code","source":"import nltk\nimport evaluate\n\n# Download required NLTK datasets\nnltk.download('wordnet')\nnltk.download('omw-1.4')\n\n# Load metrics\nbleu_metric = evaluate.load(\"sacrebleu\")\nmeteor_metric = evaluate.load(\"meteor\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:03:14.022706Z","iopub.execute_input":"2024-10-13T04:03:14.023036Z","iopub.status.idle":"2024-10-13T04:03:16.255060Z","shell.execute_reply.started":"2024-10-13T04:03:14.022994Z","shell.execute_reply":"2024-10-13T04:03:16.254163Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stderr","text":"[nltk_data] Downloading package wordnet to /usr/share/nltk_data...\n[nltk_data]   Package wordnet is already up-to-date!\n[nltk_data] Downloading package omw-1.4 to /usr/share/nltk_data...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/8.15k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"96c43d90a14d4f8b9c1566be3b7c67bd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/7.02k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cb64ff1907ba40c19d26f187e36135b8"}},"metadata":{}},{"name":"stderr","text":"[nltk_data] Downloading package wordnet to /usr/share/nltk_data...\n[nltk_data]   Package wordnet is already up-to-date!\n[nltk_data] Downloading package punkt_tab to /usr/share/nltk_data...\n[nltk_data]   Unzipping tokenizers/punkt_tab.zip.\n[nltk_data] Downloading package omw-1.4 to /usr/share/nltk_data...\n[nltk_data]   Package omw-1.4 is already up-to-date!\n","output_type":"stream"}]},{"cell_type":"code","source":"def postprocess_text(preds, labels):\n    preds = [pred.strip() for pred in preds]\n    labels = [[label.strip()] for label in labels]\n\n    return preds, labels\n\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    # Postprocess the text (strip extra spaces)\n    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n\n    # Compute BLEU score\n    bleu_result = bleu_metric.compute(predictions=decoded_preds, references=decoded_labels)\n\n    # Compute METEOR score\n    meteor_result = meteor_metric.compute(predictions=decoded_preds, references=decoded_labels)\n\n    # Compute generation lengths\n    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n\n    # Combine the results\n    result = {'bleu': bleu_result['score'], 'meteor': meteor_result['meteor']}\n    result[\"gen_len\"] = np.mean(prediction_lens)\n\n    # Round results to 4 decimal places\n    result = {k: round(v, 4) for k, v in result.items()}\n    \n    return result\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:03:16.258223Z","iopub.execute_input":"2024-10-13T04:03:16.258893Z","iopub.status.idle":"2024-10-13T04:03:16.271170Z","shell.execute_reply.started":"2024-10-13T04:03:16.258856Z","shell.execute_reply":"2024-10-13T04:03:16.269904Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:03:16.272826Z","iopub.execute_input":"2024-10-13T04:03:16.273442Z","iopub.status.idle":"2024-10-13T04:03:16.285551Z","shell.execute_reply.started":"2024-10-13T04:03:16.273387Z","shell.execute_reply":"2024-10-13T04:03:16.284474Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir=\"M2M101\",\n    evaluation_strategy=\"epoch\",\n    learning_rate=1e-5,\n    per_device_train_batch_size=8,\n    per_device_eval_batch_size=8,\n    weight_decay=0.01,\n    save_total_limit=3,\n    num_train_epochs=20,\n    predict_with_generate=True,\n    fp16=True,\n    push_to_hub=True,\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_dataset[\"train\"],\n    eval_dataset=tokenized_dataset[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics,\n)\n\n# for starting the training of model\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2024-10-13T04:03:16.286865Z","iopub.execute_input":"2024-10-13T04:03:16.287267Z","iopub.status.idle":"2024-10-13T05:36:34.581097Z","shell.execute_reply.started":"2024-10-13T04:03:16.287223Z","shell.execute_reply":"2024-10-13T05:36:34.580024Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n  warnings.warn(\n/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:494: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"\n    <div>\n      \n      <progress value='1480' max='1480' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [1480/1480 1:33:13, Epoch 20/20]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Bleu</th>\n      <th>Meteor</th>\n      <th>Gen Len</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>No log</td>\n      <td>5.826086</td>\n      <td>1.178600</td>\n      <td>0.199700</td>\n      <td>36.383100</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>No log</td>\n      <td>4.616993</td>\n      <td>2.648000</td>\n      <td>0.265700</td>\n      <td>37.606800</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>No log</td>\n      <td>3.512793</td>\n      <td>5.706900</td>\n      <td>0.321700</td>\n      <td>32.216900</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>No log</td>\n      <td>2.528102</td>\n      <td>6.313400</td>\n      <td>0.354700</td>\n      <td>31.857600</td>\n    </tr>\n    <tr>\n      <td>5</td>\n      <td>No log</td>\n      <td>1.717724</td>\n      <td>8.503600</td>\n      <td>0.380000</td>\n      <td>29.972900</td>\n    </tr>\n    <tr>\n      <td>6</td>\n      <td>No log</td>\n      <td>1.166630</td>\n      <td>10.116900</td>\n      <td>0.392500</td>\n      <td>28.067800</td>\n    </tr>\n    <tr>\n      <td>7</td>\n      <td>3.564100</td>\n      <td>0.870194</td>\n      <td>10.420700</td>\n      <td>0.424600</td>\n      <td>31.105100</td>\n    </tr>\n    <tr>\n      <td>8</td>\n      <td>3.564100</td>\n      <td>0.737626</td>\n      <td>12.615300</td>\n      <td>0.431000</td>\n      <td>28.633900</td>\n    </tr>\n    <tr>\n      <td>9</td>\n      <td>3.564100</td>\n      <td>0.690092</td>\n      <td>13.296600</td>\n      <td>0.450300</td>\n      <td>29.237300</td>\n    </tr>\n    <tr>\n      <td>10</td>\n      <td>3.564100</td>\n      <td>0.671265</td>\n      <td>11.977200</td>\n      <td>0.439600</td>\n      <td>30.566100</td>\n    </tr>\n    <tr>\n      <td>11</td>\n      <td>3.564100</td>\n      <td>0.665060</td>\n      <td>14.043600</td>\n      <td>0.450600</td>\n      <td>30.200000</td>\n    </tr>\n    <tr>\n      <td>12</td>\n      <td>3.564100</td>\n      <td>0.667843</td>\n      <td>13.263200</td>\n      <td>0.451400</td>\n      <td>31.054200</td>\n    </tr>\n    <tr>\n      <td>13</td>\n      <td>3.564100</td>\n      <td>0.667750</td>\n      <td>14.092400</td>\n      <td>0.456300</td>\n      <td>29.278000</td>\n    </tr>\n    <tr>\n      <td>14</td>\n      <td>0.512100</td>\n      <td>0.669289</td>\n      <td>14.746000</td>\n      <td>0.465100</td>\n      <td>28.406800</td>\n    </tr>\n    <tr>\n      <td>15</td>\n      <td>0.512100</td>\n      <td>0.669845</td>\n      <td>14.927800</td>\n      <td>0.467700</td>\n      <td>28.515300</td>\n    </tr>\n    <tr>\n      <td>16</td>\n      <td>0.512100</td>\n      <td>0.670038</td>\n      <td>14.743100</td>\n      <td>0.467400</td>\n      <td>28.928800</td>\n    </tr>\n    <tr>\n      <td>17</td>\n      <td>0.512100</td>\n      <td>0.674381</td>\n      <td>15.293400</td>\n      <td>0.470100</td>\n      <td>28.867800</td>\n    </tr>\n    <tr>\n      <td>18</td>\n      <td>0.512100</td>\n      <td>0.674102</td>\n      <td>15.677600</td>\n      <td>0.471200</td>\n      <td>28.349200</td>\n    </tr>\n    <tr>\n      <td>19</td>\n      <td>0.512100</td>\n      <td>0.677232</td>\n      <td>14.942000</td>\n      <td>0.470700</td>\n      <td>28.969500</td>\n    </tr>\n    <tr>\n      <td>20</td>\n      <td>0.512100</td>\n      <td>0.676572</td>\n      <td>15.341600</td>\n      <td>0.472300</td>\n      <td>28.027100</td>\n    </tr>\n  </tbody>\n</table><p>"},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:2618: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 200, 'early_stopping': True, 'num_beams': 5}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n  warnings.warn(\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py:79: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.\n  with torch.cuda.device(device), torch.cuda.stream(stream), autocast(enabled=autocast_enabled):\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"TrainOutput(global_step=1480, training_loss=1.4915999850711308, metrics={'train_runtime': 5596.282, 'train_samples_per_second': 4.214, 'train_steps_per_second': 0.264, 'total_flos': 6387540814725120.0, 'train_loss': 1.4915999850711308, 'epoch': 20.0})"},"metadata":{}}]},{"cell_type":"code","source":"trainer.push_to_hub()","metadata":{"execution":{"iopub.status.busy":"2024-10-13T05:36:34.582611Z","iopub.execute_input":"2024-10-13T05:36:34.583362Z","iopub.status.idle":"2024-10-13T05:36:53.872348Z","shell.execute_reply.started":"2024-10-13T05:36:34.583312Z","shell.execute_reply":"2024-10-13T05:36:53.871321Z"},"trusted":true},"execution_count":21,"outputs":[{"output_type":"display_data","data":{"text/plain":"events.out.tfevents.1728792198.7afee8f5ccfd.30.0:   0%|          | 0.00/14.5k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"036f7878ca094bb283ce097c4e0c5626"}},"metadata":{}},{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"CommitInfo(commit_url='https://huggingface.co/sarch7040/M2M101/commit/9b4bf86769b33fe25a0f041812424d698f3d1680', commit_message='End of training', commit_description='', oid='9b4bf86769b33fe25a0f041812424d698f3d1680', pr_url=None, repo_url=RepoUrl('https://huggingface.co/sarch7040/M2M101', endpoint='https://huggingface.co', repo_type='model', repo_id='sarch7040/M2M101'), pr_revision=None, pr_num=None)"},"metadata":{}}]},{"cell_type":"code","source":"# Use a pipeline as a high-level helper\nfrom transformers import pipeline\n\npipe = pipeline(\"text2text-generation\", model=\"sarch7040/M2M101\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T05:36:53.873596Z","iopub.execute_input":"2024-10-13T05:36:53.873929Z","iopub.status.idle":"2024-10-13T05:37:12.785489Z","shell.execute_reply.started":"2024-10-13T05:36:53.873893Z","shell.execute_reply":"2024-10-13T05:37:12.784462Z"},"trusted":true},"execution_count":22,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/935 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0db09f63a8aa4379a5168133798583d0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"97f853df430548df9bb18598ba53b8c9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/198 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c1dde5aa3ba0493a9608e71f6b2679b2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/19.8k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"798e252d89044bc8b2ec925b1fbaf665"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.json:   0%|          | 0.00/3.71M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"020279d8ba894c4eb2b85909a85730b0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/2.42M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5939581fa17f4c8b8879292a5e295ffe"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"added_tokens.json:   0%|          | 0.00/2.01k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1952c6c2a553475c8ec3e36a0fd827f3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/1.56k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0f2c6a33f8b64a62a9d46cd9d6d2ebea"}},"metadata":{}},{"name":"stderr","text":"Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n","output_type":"stream"}]},{"cell_type":"code","source":"pipe(\"चाणक्य\")","metadata":{"execution":{"iopub.status.busy":"2024-10-13T05:37:12.787004Z","iopub.execute_input":"2024-10-13T05:37:12.787889Z","iopub.status.idle":"2024-10-13T05:37:13.833157Z","shell.execute_reply.started":"2024-10-13T05:37:12.787841Z","shell.execute_reply":"2024-10-13T05:37:13.832206Z"},"trusted":true},"execution_count":23,"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"[{'generated_text': 'Charity is'}]"},"metadata":{}}]}]}