Delete evaluation.ipynb
Browse files- evaluation.ipynb +0 -206
evaluation.ipynb
DELETED
@@ -1,206 +0,0 @@
|
|
1 |
-
{
|
2 |
-
"cells": [
|
3 |
-
{
|
4 |
-
"cell_type": "code",
|
5 |
-
"id": "initial_id",
|
6 |
-
"metadata": {
|
7 |
-
"collapsed": true,
|
8 |
-
"ExecuteTime": {
|
9 |
-
"end_time": "2024-12-16T00:57:30.435870Z",
|
10 |
-
"start_time": "2024-12-16T00:57:30.239832Z"
|
11 |
-
}
|
12 |
-
},
|
13 |
-
"source": [
|
14 |
-
"import pandas as pd\n",
|
15 |
-
"from datasets import Dataset\n",
|
16 |
-
"from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
|
17 |
-
"from torch.utils.data import DataLoader\n",
|
18 |
-
"import torch\n",
|
19 |
-
"import evaluate\n",
|
20 |
-
"from tqdm import tqdm\n",
|
21 |
-
"\n",
|
22 |
-
"\n",
|
23 |
-
"# 1. Load the model and tokenizer\n",
|
24 |
-
"tokenizer = AutoTokenizer.from_pretrained(\"CIS5190ml/bert4\")\n",
|
25 |
-
"model = AutoModelForSequenceClassification.from_pretrained(\"CIS5190ml/bert4\")\n",
|
26 |
-
"\n",
|
27 |
-
"# 2. Load the dataset\n",
|
28 |
-
"import pandas as pd\n",
|
29 |
-
"\n",
|
30 |
-
"splits = {'train': 'train_dataset.csv', 'test': 'test_dataset.csv'}\n",
|
31 |
-
"test_df = pd.read_csv("hf://datasets/CIS5190ml/NewData/" + splits["train"])\n",
|
32 |
-
"ds = Dataset.from_pandas(test_df)\n",
|
33 |
-
"\n"
|
34 |
-
],
|
35 |
-
"outputs": [],
|
36 |
-
"execution_count": 32
|
37 |
-
},
|
38 |
-
{
|
39 |
-
"metadata": {
|
40 |
-
"ExecuteTime": {
|
41 |
-
"end_time": "2024-12-16T00:57:33.327397Z",
|
42 |
-
"start_time": "2024-12-16T00:57:32.596149Z"
|
43 |
-
}
|
44 |
-
},
|
45 |
-
"cell_type": "code",
|
46 |
-
"source": [
|
47 |
-
"# Preprocessing function\n",
|
48 |
-
"def preprocess_function(examples):\n",
|
49 |
-
" return tokenizer(examples[\"title\"], truncation=True, padding=\"max_length\")\n",
|
50 |
-
"\n",
|
51 |
-
"encoded_ds = ds.map(preprocess_function, batched=True)\n",
|
52 |
-
"\n",
|
53 |
-
"# Keep only the necessary columns (input_ids, attention_mask, labels)\n",
|
54 |
-
"desired_cols = [\"input_ids\", \"attention_mask\", \"labels\"]\n",
|
55 |
-
"encoded_ds = encoded_ds.remove_columns([col for col in encoded_ds.column_names if col not in desired_cols])\n",
|
56 |
-
"encoded_ds.set_format(\"torch\")\n",
|
57 |
-
"\n",
|
58 |
-
"# Create DataLoader\n",
|
59 |
-
"test_loader = DataLoader(encoded_ds, batch_size=8)\n",
|
60 |
-
"\n",
|
61 |
-
"# Load accuracy metric\n",
|
62 |
-
"accuracy = evaluate.load(\"accuracy\")\n",
|
63 |
-
"\n",
|
64 |
-
"# Set device\n",
|
65 |
-
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
66 |
-
"model.to(device)\n"
|
67 |
-
],
|
68 |
-
"id": "dfefbe70a4ff8696",
|
69 |
-
"outputs": [
|
70 |
-
{
|
71 |
-
"name": "stderr",
|
72 |
-
"output_type": "stream",
|
73 |
-
"text": [
|
74 |
-
"Map: 100%|██████████| 758/758 [00:00<00:00, 7183.73 examples/s]\n"
|
75 |
-
]
|
76 |
-
},
|
77 |
-
{
|
78 |
-
"data": {
|
79 |
-
"text/plain": [
|
80 |
-
"BertForSequenceClassification(\n",
|
81 |
-
" (bert): BertModel(\n",
|
82 |
-
" (embeddings): BertEmbeddings(\n",
|
83 |
-
" (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
|
84 |
-
" (position_embeddings): Embedding(512, 768)\n",
|
85 |
-
" (token_type_embeddings): Embedding(2, 768)\n",
|
86 |
-
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
87 |
-
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
88 |
-
" )\n",
|
89 |
-
" (encoder): BertEncoder(\n",
|
90 |
-
" (layer): ModuleList(\n",
|
91 |
-
" (0-11): 12 x BertLayer(\n",
|
92 |
-
" (attention): BertAttention(\n",
|
93 |
-
" (self): BertSdpaSelfAttention(\n",
|
94 |
-
" (query): Linear(in_features=768, out_features=768, bias=True)\n",
|
95 |
-
" (key): Linear(in_features=768, out_features=768, bias=True)\n",
|
96 |
-
" (value): Linear(in_features=768, out_features=768, bias=True)\n",
|
97 |
-
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
98 |
-
" )\n",
|
99 |
-
" (output): BertSelfOutput(\n",
|
100 |
-
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
101 |
-
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
102 |
-
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
103 |
-
" )\n",
|
104 |
-
" )\n",
|
105 |
-
" (intermediate): BertIntermediate(\n",
|
106 |
-
" (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
|
107 |
-
" (intermediate_act_fn): GELUActivation()\n",
|
108 |
-
" )\n",
|
109 |
-
" (output): BertOutput(\n",
|
110 |
-
" (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
|
111 |
-
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
|
112 |
-
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
113 |
-
" )\n",
|
114 |
-
" )\n",
|
115 |
-
" )\n",
|
116 |
-
" )\n",
|
117 |
-
" (pooler): BertPooler(\n",
|
118 |
-
" (dense): Linear(in_features=768, out_features=768, bias=True)\n",
|
119 |
-
" (activation): Tanh()\n",
|
120 |
-
" )\n",
|
121 |
-
" )\n",
|
122 |
-
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
123 |
-
" (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
|
124 |
-
")"
|
125 |
-
]
|
126 |
-
},
|
127 |
-
"execution_count": 33,
|
128 |
-
"metadata": {},
|
129 |
-
"output_type": "execute_result"
|
130 |
-
}
|
131 |
-
],
|
132 |
-
"execution_count": 33
|
133 |
-
},
|
134 |
-
{
|
135 |
-
"metadata": {
|
136 |
-
"ExecuteTime": {
|
137 |
-
"end_time": "2024-12-16T00:58:18.207058Z",
|
138 |
-
"start_time": "2024-12-16T00:58:08.007420Z"
|
139 |
-
}
|
140 |
-
},
|
141 |
-
"cell_type": "code",
|
142 |
-
"source": [
|
143 |
-
"# Evaluate\n",
|
144 |
-
"model.eval()\n",
|
145 |
-
"for batch in tqdm(test_loader, desc=\"Evaluating\"):\n",
|
146 |
-
" input_ids = batch[\"input_ids\"].to(device)\n",
|
147 |
-
" attention_mask = batch[\"attention_mask\"].to(device)\n",
|
148 |
-
" labels = batch[\"labels\"].to(device)\n",
|
149 |
-
"\n",
|
150 |
-
" with torch.no_grad():\n",
|
151 |
-
" outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
|
152 |
-
" preds = torch.argmax(outputs.logits, dim=-1)\n",
|
153 |
-
" accuracy.add_batch(predictions=preds, references=labels)\n",
|
154 |
-
"\n",
|
155 |
-
"final_accuracy = accuracy.compute()\n",
|
156 |
-
"print(\"Accuracy:\", final_accuracy[\"accuracy\"])"
|
157 |
-
],
|
158 |
-
"id": "c6e4fd03bd73664f",
|
159 |
-
"outputs": [
|
160 |
-
{
|
161 |
-
"name": "stderr",
|
162 |
-
"output_type": "stream",
|
163 |
-
"text": [
|
164 |
-
"Evaluating: 100%|██████████| 95/95 [00:10<00:00, 9.33it/s]"
|
165 |
-
]
|
166 |
-
},
|
167 |
-
{
|
168 |
-
"name": "stdout",
|
169 |
-
"output_type": "stream",
|
170 |
-
"text": [
|
171 |
-
"Accuracy: 0.7823218997361477\n"
|
172 |
-
]
|
173 |
-
},
|
174 |
-
{
|
175 |
-
"name": "stderr",
|
176 |
-
"output_type": "stream",
|
177 |
-
"text": [
|
178 |
-
"\n"
|
179 |
-
]
|
180 |
-
}
|
181 |
-
],
|
182 |
-
"execution_count": 35
|
183 |
-
}
|
184 |
-
],
|
185 |
-
"metadata": {
|
186 |
-
"kernelspec": {
|
187 |
-
"display_name": "Python 3",
|
188 |
-
"language": "python",
|
189 |
-
"name": "python3"
|
190 |
-
},
|
191 |
-
"language_info": {
|
192 |
-
"codemirror_mode": {
|
193 |
-
"name": "ipython",
|
194 |
-
"version": 2
|
195 |
-
},
|
196 |
-
"file_extension": ".py",
|
197 |
-
"mimetype": "text/x-python",
|
198 |
-
"name": "python",
|
199 |
-
"nbconvert_exporter": "python",
|
200 |
-
"pygments_lexer": "ipython2",
|
201 |
-
"version": "2.7.6"
|
202 |
-
}
|
203 |
-
},
|
204 |
-
"nbformat": 4,
|
205 |
-
"nbformat_minor": 5
|
206 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|