Upload eval 2.ipynb

eval 2.ipynb

@@ -0,0 +1,212 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-12-16T01:56:57.350322Z",
+     "start_time": "2024-12-16T01:56:56.339076Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "from datasets import Dataset\n",
+    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
+    "from torch.utils.data import DataLoader\n",
+    "import torch\n",
+    "import evaluate\n",
+    "from tqdm import tqdm\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# 1. Load the model and tokenizer\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"CIS5190ml/bert4\")\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(\"CIS5190ml/bert4\")\n",
+    "\n",
+    "# 2. Load the dataset\n",
+    "import pandas as pd \n",
+    "\n",
+    "ds = load_dataset(\"CIS5190ml/NewData\")\n"
+   ],
+   "outputs": [],
+   "execution_count": 44
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-12-16T01:56:22.105429Z",
+     "start_time": "2024-12-16T01:56:22.089923Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "#choose test dataset\n",
+    "ds = ds[\"test\"]"
+   ],
+   "id": "fd95d0347ad1665a",
+   "outputs": [],
+   "execution_count": 41
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-12-16T01:56:24.245992Z",
+     "start_time": "2024-12-16T01:56:23.609377Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# Preprocessing function\n",
+    "def preprocess_function(examples):\n",
+    "    return tokenizer(examples[\"title\"], truncation=True, padding=\"max_length\")\n",
+    "\n",
+    "encoded_ds = ds.map(preprocess_function, batched=True)\n",
+    "\n",
+    "# Keep only the necessary columns (input_ids, attention_mask, labels)\n",
+    "desired_cols = [\"input_ids\", \"attention_mask\", \"labels\"]\n",
+    "encoded_ds = encoded_ds.remove_columns([col for col in encoded_ds.column_names if col not in desired_cols])\n",
+    "encoded_ds.set_format(\"torch\")\n",
+    "\n",
+    "# Create DataLoader\n",
+    "test_loader = DataLoader(encoded_ds, batch_size=8)\n",
+    "\n",
+    "# Load accuracy metric\n",
+    "accuracy = evaluate.load(\"accuracy\")\n",
+    "\n",
+    "# Set device\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "model.to(device)\n"
+   ],
+   "id": "dfefbe70a4ff8696",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "BertForSequenceClassification(\n",
+       "  (bert): BertModel(\n",
+       "    (embeddings): BertEmbeddings(\n",
+       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
+       "      (position_embeddings): Embedding(512, 768)\n",
+       "      (token_type_embeddings): Embedding(2, 768)\n",
+       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "      (dropout): Dropout(p=0.1, inplace=False)\n",
+       "    )\n",
+       "    (encoder): BertEncoder(\n",
+       "      (layer): ModuleList(\n",
+       "        (0-11): 12 x BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSdpaSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (intermediate_act_fn): GELUActivation()\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (pooler): BertPooler(\n",
+       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "      (activation): Tanh()\n",
+       "    )\n",
+       "  )\n",
+       "  (dropout): Dropout(p=0.1, inplace=False)\n",
+       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 42
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-12-16T01:56:35.444373Z",
+     "start_time": "2024-12-16T01:56:26.083442Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# Evaluate\n",
+    "model.eval()\n",
+    "for batch in tqdm(test_loader, desc=\"Evaluating\"):\n",
+    "    input_ids = batch[\"input_ids\"].to(device)\n",
+    "    attention_mask = batch[\"attention_mask\"].to(device)\n",
+    "    labels = batch[\"labels\"].to(device)\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
+    "        preds = torch.argmax(outputs.logits, dim=-1)\n",
+    "        accuracy.add_batch(predictions=preds, references=labels)\n",
+    "\n",
+    "final_accuracy = accuracy.compute()\n",
+    "print(\"Accuracy:\", final_accuracy[\"accuracy\"])"
+   ],
+   "id": "c6e4fd03bd73664f",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 95/95 [00:09<00:00, 10.21it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.9182058047493403\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "execution_count": 43
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}