{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#!pip install \"modin[all]\" # Install Ray and Dask\n", "# !pip install pytorch \n", "# !pip install intel-extension-for-pytorch\n", "# !pip install transformers\n", "# !pip install datasets" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QuestionsCategory
0About what proportion of the population of the...BT1
1Correctly label the brain lobes indicated on t...BT1
2Define compound interest.BT1
3Define four types of traceabilityBT1
4Define mercantilism.BT1
.........
8762Distinguish between different types of soil st...BT4
8763Invent a blockchain-based solution for transpa...BT6
8764Compare the advantages and disadvantages of us...BT4
8765Describe the purpose of the \"volatile\" keyword...BT1
8766Explain the concept of noise in communication ...BT2
\n", "

8767 rows × 2 columns

\n", "
" ], "text/plain": [ " Questions Category\n", "0 About what proportion of the population of the... BT1\n", "1 Correctly label the brain lobes indicated on t... BT1\n", "2 Define compound interest. BT1\n", "3 Define four types of traceability BT1\n", "4 Define mercantilism. BT1\n", "... ... ...\n", "8762 Distinguish between different types of soil st... BT4\n", "8763 Invent a blockchain-based solution for transpa... BT6\n", "8764 Compare the advantages and disadvantages of us... BT4\n", "8765 Describe the purpose of the \"volatile\" keyword... BT1\n", "8766 Explain the concept of noise in communication ... BT2\n", "\n", "[8767 rows x 2 columns]" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import modin.pandas as pd\n", "df = pd.read_csv('blooms_taxonomy_dataset.csv')\n", "df" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "mapping = {\"BT1\": 0, \"BT2\": 1, \"BT3\": 2, \"BT4\": 3, \"BT5\": 4, \"BT6\": 5}\n", "df[\"Category\"] = df[\"Category\"].map(mapping)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QuestionsCategory
0About what proportion of the population of the...0
1Correctly label the brain lobes indicated on t...0
2Define compound interest.0
3Define four types of traceability0
4Define mercantilism.0
.........
8762Distinguish between different types of soil st...3
8763Invent a blockchain-based solution for transpa...5
8764Compare the advantages and disadvantages of us...3
8765Describe the purpose of the \"volatile\" keyword...0
8766Explain the concept of noise in communication ...1
\n", "

8767 rows × 2 columns

\n", "
" ], "text/plain": [ " Questions Category\n", "0 About what proportion of the population of the... 0\n", "1 Correctly label the brain lobes indicated on t... 0\n", "2 Define compound interest. 0\n", "3 Define four types of traceability 0\n", "4 Define mercantilism. 0\n", "... ... ...\n", "8762 Distinguish between different types of soil st... 3\n", "8763 Invent a blockchain-based solution for transpa... 5\n", "8764 Compare the advantages and disadvantages of us... 3\n", "8765 Describe the purpose of the \"volatile\" keyword... 0\n", "8766 Explain the concept of noise in communication ... 1\n", "\n", "[8767 rows x 2 columns]" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/pytorch_env/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", " warnings.warn(\n" ] }, { "data": { "text/plain": [ "{'input_ids': tensor([[ 101, 2055, 2054, ..., 0, 0, 0],\n", " [ 101, 11178, 3830, ..., 0, 0, 0],\n", " [ 101, 9375, 7328, ..., 0, 0, 0],\n", " ...,\n", " [ 101, 12826, 1996, ..., 0, 0, 0],\n", " [ 101, 6235, 1996, ..., 0, 0, 0],\n", " [ 101, 4863, 1996, ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0],\n", " [1, 1, 1, ..., 0, 0, 0],\n", " [1, 1, 1, ..., 0, 0, 0],\n", " ...,\n", " [1, 1, 1, ..., 0, 0, 0],\n", " [1, 1, 1, ..., 0, 0, 0],\n", " [1, 1, 1, ..., 0, 0, 0]])}" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from transformers import DistilBertTokenizer\n", "import torch\n", "from torch.utils.data import DataLoader\n", "import intel_extension_for_pytorch as ipex\n", "# Load the DistilBERT tokenizer\n", "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n", "\n", "# Tokenize the 'Questions' column\n", "inputs = tokenizer(list(df['Questions']), padding=True, truncation=True, return_tensors='pt', max_length=2048)\n", "inputs" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([8767, 123])" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inputs['input_ids'].size()" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([0, 0, 0, ..., 3, 0, 1])" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "labels = torch.tensor(df['Category'].values)\n", "labels" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "from transformers import DistilBertForSequenceClassification\n", "\n", "# Load the model with a classification head\n", "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=6) # 6 classes: 0 to 5\n", "optimized_model = ipex.optimize(model, dtype=torch.float32)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "\n", "# Split the data into training and validation sets\n", "train_inputs, val_inputs, train_labels, val_labels = train_test_split(inputs['input_ids'], labels, test_size=0.2, random_state=42)\n" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import DataLoader, TensorDataset\n", "\n", "# Create datasets for training and validation\n", "train_dataset = TensorDataset(train_inputs, train_labels)\n", "val_dataset = TensorDataset(val_inputs, val_labels)\n", "\n", "# Create DataLoader for both training and validation\n", "train_dataloader = DataLoader(train_dataset, batch_size=20, shuffle=True)\n", "val_dataloader = DataLoader(val_dataset, batch_size=20)\n" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "cpu\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/pytorch_env/lib/python3.11/site-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] } ], "source": [ "from transformers import AdamW\n", "from torch.optim.lr_scheduler import StepLR\n", "\n", "# Set up the optimizer\n", "optimizer = AdamW(optimized_model.parameters(), lr=0.0001)\n", "\n", "# Define the training loop\n", "epochs = 1\n", "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n", "optimized_model.to(device)\n", "\n", "print(device)" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor(0.1266, grad_fn=)\n", "tensor(0.2361, grad_fn=)\n", "tensor(0.0948, grad_fn=)\n", "tensor(0.0170, grad_fn=)\n", "tensor(0.5257, grad_fn=)\n", "tensor(0.0933, grad_fn=)\n", "tensor(0.1646, grad_fn=)\n", "tensor(0.2118, grad_fn=)\n", "tensor(0.0173, grad_fn=)\n", "tensor(0.1543, grad_fn=)\n", "tensor(0.3518, grad_fn=)\n", "tensor(0.5005, grad_fn=)\n", "tensor(0.3083, grad_fn=)\n", "tensor(0.1673, grad_fn=)\n", "tensor(0.0377, grad_fn=)\n", "tensor(0.1693, grad_fn=)\n", "tensor(0.3132, grad_fn=)\n", "tensor(0.3724, grad_fn=)\n", "tensor(0.0699, grad_fn=)\n", "tensor(0.1015, grad_fn=)\n", "tensor(0.0627, grad_fn=)\n", "tensor(0.0439, grad_fn=)\n", "tensor(0.3108, grad_fn=)\n", "tensor(0.1622, grad_fn=)\n", "tensor(0.2091, grad_fn=)\n", "tensor(0.1177, grad_fn=)\n", "tensor(0.5044, grad_fn=)\n", "tensor(0.0834, grad_fn=)\n", "tensor(0.1307, grad_fn=)\n", "tensor(0.0162, grad_fn=)\n", "tensor(0.1507, grad_fn=)\n", "tensor(0.4310, grad_fn=)\n", "tensor(0.1047, grad_fn=)\n", "tensor(0.3400, grad_fn=)\n", "tensor(0.5385, grad_fn=)\n", "tensor(0.0468, grad_fn=)\n", "tensor(0.0655, grad_fn=)\n", "tensor(0.0421, grad_fn=)\n", "tensor(0.2367, grad_fn=)\n", "tensor(0.1999, grad_fn=)\n", "tensor(0.3367, grad_fn=)\n", "tensor(0.5989, grad_fn=)\n", "tensor(0.0349, grad_fn=)\n", "tensor(0.4536, grad_fn=)\n", "tensor(0.2197, grad_fn=)\n", "tensor(0.2861, grad_fn=)\n", "tensor(0.1133, grad_fn=)\n", "tensor(0.2491, grad_fn=)\n", "tensor(0.2210, grad_fn=)\n", "tensor(0.1425, grad_fn=)\n", "tensor(0.1268, grad_fn=)\n", "tensor(0.2085, grad_fn=)\n", "tensor(0.2444, grad_fn=)\n", "tensor(0.3229, grad_fn=)\n", "tensor(0.1340, grad_fn=)\n", "tensor(0.2742, grad_fn=)\n", "tensor(0.2652, grad_fn=)\n", "tensor(0.1091, grad_fn=)\n", "tensor(0.3718, grad_fn=)\n", "tensor(0.1806, grad_fn=)\n", "tensor(0.1180, grad_fn=)\n", "tensor(0.1474, grad_fn=)\n", "tensor(0.2807, grad_fn=)\n", "tensor(0.2696, grad_fn=)\n", "tensor(0.4681, grad_fn=)\n", "tensor(0.0877, grad_fn=)\n", "tensor(0.3703, grad_fn=)\n", "tensor(0.4087, grad_fn=)\n", "tensor(0.5539, grad_fn=)\n", "tensor(0.1504, grad_fn=)\n", "tensor(0.0107, grad_fn=)\n", "tensor(0.5127, grad_fn=)\n", "tensor(0.5999, grad_fn=)\n", "tensor(0.1659, grad_fn=)\n", "tensor(0.0303, grad_fn=)\n", "tensor(0.2197, grad_fn=)\n", "tensor(0.2298, grad_fn=)\n", "tensor(0.3073, grad_fn=)\n", "tensor(0.3306, grad_fn=)\n", "tensor(0.2281, grad_fn=)\n", "tensor(0.0406, grad_fn=)\n", "tensor(0.1882, grad_fn=)\n", "tensor(0.2777, grad_fn=)\n", "tensor(0.3764, grad_fn=)\n", "tensor(0.2865, grad_fn=)\n", "tensor(0.1368, grad_fn=)\n", "tensor(0.3605, grad_fn=)\n", "tensor(0.1100, grad_fn=)\n", "tensor(0.2140, grad_fn=)\n", "tensor(0.4161, grad_fn=)\n", "tensor(0.2829, grad_fn=)\n", "tensor(0.2951, grad_fn=)\n", "tensor(0.2776, grad_fn=)\n", "tensor(0.0665, grad_fn=)\n", "tensor(0.4622, grad_fn=)\n", "tensor(0.1903, grad_fn=)\n", "tensor(0.1492, grad_fn=)\n", "tensor(0.3531, grad_fn=)\n", "tensor(0.1535, grad_fn=)\n", "tensor(0.4230, grad_fn=)\n", "tensor(0.2674, grad_fn=)\n", "tensor(0.1988, grad_fn=)\n", "tensor(0.1032, grad_fn=)\n", "tensor(0.6737, grad_fn=)\n", "tensor(0.0771, grad_fn=)\n", "tensor(0.0759, grad_fn=)\n", "tensor(0.2127, grad_fn=)\n", "tensor(0.2328, grad_fn=)\n", "tensor(0.4041, grad_fn=)\n", "tensor(0.3188, grad_fn=)\n", "tensor(0.2907, grad_fn=)\n", "tensor(0.1548, grad_fn=)\n", "tensor(0.2523, grad_fn=)\n", "tensor(0.3066, grad_fn=)\n", "tensor(0.2681, grad_fn=)\n", "tensor(0.1790, grad_fn=)\n", "tensor(0.1407, grad_fn=)\n", "tensor(0.4857, grad_fn=)\n", "tensor(0.3541, grad_fn=)\n", "tensor(0.2105, grad_fn=)\n", "tensor(0.2170, grad_fn=)\n", "tensor(0.3173, grad_fn=)\n", "tensor(0.1405, grad_fn=)\n", "tensor(0.2956, grad_fn=)\n", "tensor(0.5343, grad_fn=)\n", "tensor(0.3510, grad_fn=)\n", "tensor(0.1565, grad_fn=)\n", "tensor(0.7312, grad_fn=)\n", "tensor(0.4818, grad_fn=)\n", "tensor(0.3232, grad_fn=)\n", "tensor(0.2504, grad_fn=)\n", "tensor(0.0905, grad_fn=)\n", "tensor(0.2030, grad_fn=)\n", "tensor(0.3142, grad_fn=)\n", "tensor(0.4711, grad_fn=)\n", "tensor(0.0577, grad_fn=)\n", "tensor(0.1709, grad_fn=)\n", "tensor(0.1811, grad_fn=)\n", "tensor(0.4690, grad_fn=)\n", "tensor(0.1305, grad_fn=)\n", "tensor(0.1392, grad_fn=)\n", "tensor(0.1633, grad_fn=)\n", "tensor(0.1361, grad_fn=)\n", "tensor(0.2246, grad_fn=)\n", "tensor(0.1142, grad_fn=)\n", "tensor(0.4056, grad_fn=)\n", "tensor(0.0341, grad_fn=)\n", "tensor(0.7735, grad_fn=)\n", "tensor(0.5424, grad_fn=)\n", "tensor(0.0938, grad_fn=)\n", "tensor(0.2202, grad_fn=)\n", "tensor(0.0883, grad_fn=)\n", "tensor(0.5231, grad_fn=)\n", "tensor(0.3891, grad_fn=)\n", "tensor(0.0318, grad_fn=)\n", "tensor(0.2012, grad_fn=)\n", "tensor(0.2682, grad_fn=)\n", "tensor(0.4051, grad_fn=)\n", "tensor(0.0735, grad_fn=)\n", "tensor(0.0473, grad_fn=)\n", "tensor(0.0671, grad_fn=)\n", "tensor(0.3305, grad_fn=)\n", "tensor(0.2791, grad_fn=)\n", "tensor(0.3031, grad_fn=)\n", "tensor(0.1154, grad_fn=)\n", "tensor(0.1411, grad_fn=)\n", "tensor(0.2358, grad_fn=)\n", "tensor(0.4483, grad_fn=)\n", "tensor(0.1316, grad_fn=)\n", "tensor(0.4731, grad_fn=)\n", "tensor(0.1665, grad_fn=)\n", "tensor(0.0311, grad_fn=)\n", "tensor(0.2365, grad_fn=)\n", "tensor(0.5279, grad_fn=)\n", "tensor(0.4144, grad_fn=)\n", "tensor(0.1594, grad_fn=)\n", "tensor(0.2623, grad_fn=)\n", "tensor(0.2407, grad_fn=)\n", "tensor(0.4914, grad_fn=)\n", "tensor(0.2589, grad_fn=)\n", "tensor(0.3578, grad_fn=)\n", "tensor(0.1238, grad_fn=)\n", "tensor(0.3464, grad_fn=)\n", "tensor(0.1637, grad_fn=)\n", "tensor(0.1750, grad_fn=)\n", "tensor(0.4039, grad_fn=)\n", "tensor(0.3257, grad_fn=)\n", "tensor(0.3095, grad_fn=)\n", "tensor(0.1030, grad_fn=)\n", "tensor(0.2661, grad_fn=)\n", "tensor(0.3043, grad_fn=)\n", "tensor(0.4696, grad_fn=)\n", "tensor(0.2800, grad_fn=)\n", "tensor(0.1741, grad_fn=)\n", "tensor(0.1582, grad_fn=)\n", "tensor(0.0720, grad_fn=)\n", "tensor(0.5691, grad_fn=)\n", "tensor(0.2497, grad_fn=)\n", "tensor(0.3357, grad_fn=)\n", "tensor(0.2267, grad_fn=)\n", "tensor(0.1167, grad_fn=)\n", "tensor(0.0201, grad_fn=)\n", "tensor(0.1358, grad_fn=)\n", "tensor(0.1345, grad_fn=)\n", "tensor(0.8850, grad_fn=)\n", "tensor(0.0556, grad_fn=)\n", "tensor(0.0690, grad_fn=)\n", "tensor(0.3296, grad_fn=)\n", "tensor(0.1559, grad_fn=)\n", "tensor(0.3681, grad_fn=)\n", "tensor(0.1394, grad_fn=)\n", "tensor(0.2133, grad_fn=)\n", "tensor(0.2564, grad_fn=)\n", "tensor(0.3522, grad_fn=)\n", "tensor(0.3458, grad_fn=)\n", "tensor(0.2390, grad_fn=)\n", "tensor(0.2744, grad_fn=)\n", "tensor(0.0902, grad_fn=)\n", "tensor(0.3074, grad_fn=)\n", "tensor(0.2031, grad_fn=)\n", "tensor(0.1170, grad_fn=)\n", "tensor(0.5067, grad_fn=)\n", "tensor(0.2392, grad_fn=)\n", "tensor(0.1138, grad_fn=)\n", "tensor(0.4484, grad_fn=)\n", "tensor(0.1577, grad_fn=)\n", "tensor(0.2137, grad_fn=)\n", "tensor(0.1273, grad_fn=)\n", "tensor(0.1333, grad_fn=)\n", "tensor(0.1629, grad_fn=)\n", "tensor(0.1824, grad_fn=)\n", "tensor(0.8445, grad_fn=)\n", "tensor(0.2046, grad_fn=)\n", "tensor(0.1296, grad_fn=)\n", "tensor(0.1347, grad_fn=)\n", "tensor(0.6210, grad_fn=)\n", "tensor(0.2479, grad_fn=)\n", "tensor(0.3683, grad_fn=)\n", "tensor(0.2815, grad_fn=)\n", "tensor(0.4198, grad_fn=)\n", "tensor(0.5143, grad_fn=)\n", "tensor(0.1253, grad_fn=)\n", "tensor(0.3922, grad_fn=)\n", "tensor(0.2052, grad_fn=)\n", "tensor(0.3182, grad_fn=)\n", "tensor(0.3578, grad_fn=)\n", "tensor(0.2138, grad_fn=)\n", "tensor(0.2801, grad_fn=)\n", "tensor(0.4023, grad_fn=)\n", "tensor(0.2817, grad_fn=)\n", "tensor(0.1442, grad_fn=)\n", "tensor(0.5465, grad_fn=)\n", "tensor(0.0325, grad_fn=)\n", "tensor(0.4592, grad_fn=)\n", "tensor(0.2917, grad_fn=)\n", "tensor(0.4769, grad_fn=)\n", "tensor(0.5182, grad_fn=)\n", "tensor(0.2828, grad_fn=)\n", "tensor(0.2595, grad_fn=)\n", "tensor(0.5020, grad_fn=)\n", "tensor(0.1517, grad_fn=)\n", "tensor(0.3279, grad_fn=)\n", "tensor(0.1594, grad_fn=)\n", "tensor(0.0840, grad_fn=)\n", "tensor(0.3132, grad_fn=)\n", "tensor(0.1184, grad_fn=)\n", "tensor(0.0184, grad_fn=)\n", "tensor(0.2888, grad_fn=)\n", "tensor(0.0821, grad_fn=)\n", "tensor(0.2481, grad_fn=)\n", "tensor(0.0216, grad_fn=)\n", "tensor(0.2419, grad_fn=)\n", "tensor(0.3978, grad_fn=)\n", "tensor(0.1400, grad_fn=)\n", "tensor(0.0140, grad_fn=)\n", "tensor(0.4252, grad_fn=)\n", "tensor(0.0495, grad_fn=)\n", "tensor(0.4713, grad_fn=)\n", "tensor(0.0973, grad_fn=)\n", "tensor(0.1307, grad_fn=)\n", "tensor(0.0592, grad_fn=)\n", "tensor(0.4353, grad_fn=)\n", "tensor(0.3089, grad_fn=)\n", "tensor(0.1569, grad_fn=)\n", "tensor(0.2282, grad_fn=)\n", "tensor(0.4177, grad_fn=)\n", "tensor(0.0643, grad_fn=)\n", "tensor(0.4958, grad_fn=)\n", "tensor(0.3452, grad_fn=)\n", "tensor(0.1051, grad_fn=)\n", "tensor(0.4404, grad_fn=)\n", "tensor(0.3820, grad_fn=)\n", "tensor(0.1086, grad_fn=)\n", "tensor(0.2805, grad_fn=)\n", "tensor(0.4529, grad_fn=)\n", "tensor(0.1772, grad_fn=)\n", "tensor(0.1061, grad_fn=)\n", "tensor(0.1318, grad_fn=)\n", "tensor(0.3808, grad_fn=)\n", "tensor(0.3329, grad_fn=)\n", "tensor(0.1924, grad_fn=)\n", "tensor(0.3695, grad_fn=)\n", "tensor(0.2400, grad_fn=)\n", "tensor(0.2193, grad_fn=)\n", "tensor(0.1588, grad_fn=)\n", "tensor(0.1683, grad_fn=)\n", "tensor(0.3439, grad_fn=)\n", "tensor(0.2541, grad_fn=)\n", "tensor(0.2351, grad_fn=)\n", "tensor(0.2033, grad_fn=)\n", "tensor(0.0757, grad_fn=)\n", "tensor(0.1629, grad_fn=)\n", "tensor(0.3000, grad_fn=)\n", "tensor(0.6601, grad_fn=)\n", "tensor(0.1748, grad_fn=)\n", "tensor(0.4209, grad_fn=)\n", "tensor(0.0594, grad_fn=)\n", "tensor(0.2206, grad_fn=)\n", "tensor(0.2674, grad_fn=)\n", "tensor(0.0595, grad_fn=)\n", "tensor(0.2141, grad_fn=)\n", "tensor(0.1375, grad_fn=)\n", "tensor(0.4534, grad_fn=)\n", "tensor(0.2570, grad_fn=)\n", "tensor(0.2481, grad_fn=)\n", "tensor(0.4599, grad_fn=)\n", "tensor(0.2221, grad_fn=)\n", "tensor(0.2963, grad_fn=)\n", "tensor(0.1427, grad_fn=)\n", "tensor(0.4567, grad_fn=)\n", "tensor(0.1509, grad_fn=)\n", "tensor(0.3520, grad_fn=)\n", "tensor(0.3681, grad_fn=)\n", "tensor(0.5287, grad_fn=)\n", "tensor(0.3123, grad_fn=)\n", "tensor(0.3609, grad_fn=)\n", "tensor(0.1110, grad_fn=)\n", "tensor(0.2717, grad_fn=)\n", "tensor(0.1092, grad_fn=)\n", "tensor(0.2693, grad_fn=)\n", "tensor(0.2787, grad_fn=)\n", "tensor(0.1664, grad_fn=)\n", "tensor(0.0727, grad_fn=)\n", "tensor(0.0400, grad_fn=)\n", "tensor(0.1332, grad_fn=)\n", "tensor(0.4125, grad_fn=)\n", "tensor(0.3152, grad_fn=)\n", "tensor(0.4981, grad_fn=)\n", "tensor(0.1758, grad_fn=)\n", "tensor(0.1878, grad_fn=)\n", "tensor(1.1352, grad_fn=)\n", "Epoch 1 | Loss: 0.25651482065232134\n" ] } ], "source": [ "for epoch in range(epochs):\n", " optimized_model.train()\n", " total_loss = 0\n", " for batch in train_dataloader:\n", " input_ids, labels = batch\n", " input_ids, labels = input_ids.to(device), labels.to(device)\n", "\n", " # Zero the gradients\n", " optimizer.zero_grad()\n", "\n", " # Forward pass\n", " outputs = optimized_model(input_ids, labels=labels)\n", " loss = outputs.loss\n", " total_loss += loss.item()\n", "\n", " # Backward pass\n", " loss.backward()\n", " optimizer.step()\n", " print(loss)\n", " print(f\"Epoch {epoch + 1} | Loss: {total_loss / len(train_dataloader)}\")" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Validation Accuracy: 78.96%\n" ] } ], "source": [ "optimized_model.eval()\n", "correct_predictions = 0\n", "total_predictions = 0\n", "\n", "with torch.no_grad():\n", " for batch in val_dataloader:\n", " input_ids, labels = batch\n", " input_ids, labels = input_ids.to(device), labels.to(device)\n", " # Forward pass\n", " outputs = optimized_model(input_ids)\n", " predictions = torch.argmax(outputs.logits, dim=-1)\n", "\n", " correct_predictions += (predictions == labels).sum().item()\n", " total_predictions += labels.size(0)\n", "\n", "accuracy = correct_predictions / total_predictions\n", "print(f\"Validation Accuracy: {accuracy * 100:.2f}%\")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3\n" ] } ], "source": [ "def predict(text):\n", " inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=512)\n", " input_ids = inputs['input_ids'].to(device)\n", " \n", " optimized_model.eval()\n", " with torch.no_grad():\n", " outputs = optimized_model(input_ids)\n", " prediction = torch.argmax(outputs.logits, dim=-1)\n", " return prediction.item()\n", "\n", "# Example prediction\n", "question = \"Compare two dog food commercials. What is the difference between them and how do they both sell their products?\"\n", "print(predict(question))\n" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Remembering: 0.6210\n", "Understanding: 0.2401\n", "Applying: 0.0801\n", "Analyzing: 0.0533\n", "Evaluating: 0.0028\n", "Creating: 0.0026\n" ] } ], "source": [ "from torch.nn.functional import softmax\n", "\n", "# The mapping of class labels to numeric labels\n", "mapping = {\"Remembering\": 0, \"Understanding\": 1, \"Applying\": 2, \"Analyzing\": 3, \"Evaluating\": 4, \"Creating\": 5}\n", "\n", "# Reverse the mapping to get the class name from the index\n", "reverse_mapping = {v: k for k, v in mapping.items()}\n", "\n", "def predict(text):\n", " # Tokenize the input text\n", " inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=512)\n", " input_ids = inputs['input_ids'].to(device)\n", " \n", " optimized_model.eval()\n", " with torch.no_grad():\n", " # Get the raw logits from the model\n", " outputs = optimized_model(input_ids)\n", " logits = outputs.logits\n", " \n", " # Apply softmax to get probabilities\n", " probabilities = softmax(logits, dim=-1)\n", " \n", " # Convert probabilities to a list or dictionary of class probabilities\n", " probabilities = probabilities.squeeze().cpu().numpy()\n", " \n", " # Map the probabilities to the class labels using the reverse mapping\n", " class_probabilities = {reverse_mapping[i]: prob for i, prob in enumerate(probabilities)}\n", " \n", " return class_probabilities\n", "\n", "# Example prediction\n", "question = \"State and explain rules of inference.\"\n", "class_probabilities = predict(question)\n", "\n", "# Display the probabilities for each class label\n", "for class_label, prob in class_probabilities.items():\n", " print(f\"{class_label}: {prob:.4f}\")\n" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('./fine_tuned_distilbert/tokenizer_config.json',\n", " './fine_tuned_distilbert/special_tokens_map.json',\n", " './fine_tuned_distilbert/vocab.txt',\n", " './fine_tuned_distilbert/added_tokens.json')" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "optimized_model.save_pretrained('./fine_tuned_distilbert')\n", "\n", "# Save the tokenizer\n", "tokenizer.save_pretrained('./fine_tuned_distilbert')" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [], "source": [ "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer\n", "\n", "# Load the saved model\n", "optimized_model = DistilBertForSequenceClassification.from_pretrained('./fine_tuned_distilbert')\n", "\n", "# Load the saved tokenizer\n", "tokenizer = DistilBertTokenizer.from_pretrained('./fine_tuned_distilbert')\n" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Remembering: 0.0049\n", "Understanding: 0.0040\n", "Applying: 0.3104\n", "Analyzing: 0.2497\n", "Evaluating: 0.3769\n", "Creating: 0.0542\n" ] } ], "source": [ "# Example of using the loaded model for prediction\n", "def predict_with_loaded_model(text):\n", " # Tokenize the input text\n", " inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=512)\n", " input_ids = inputs['input_ids'].to(device)\n", "\n", " optimized_model.eval()\n", " with torch.no_grad():\n", " outputs = optimized_model(input_ids)\n", " logits = outputs.logits\n", " probabilities = softmax(logits, dim=-1)\n", " \n", " # Map probabilities to class labels\n", " probabilities = probabilities.squeeze().cpu().numpy()\n", " class_probabilities = {reverse_mapping[i]: prob for i, prob in enumerate(probabilities)}\n", " \n", " return class_probabilities\n", "\n", "# Example usage with the saved model\n", "question = \"The accuracy of each position in a sequence of GGTACTGAT is 98%, 95%, 97%, 97%, 98%, 99%, 94%, 93%, and 97% respectively.(a) What is the average PHRED quality score of this sequence?\"\n", "class_probabilities = predict_with_loaded_model(question)\n", "\n", "# Display class probabilities\n", "for class_label, prob in class_probabilities.items():\n", " print(f\"{class_label}: {prob:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 4 }