retrain on last-linear-layer

classifier.py

@@ -13,7 +13,7 @@ from bert import BertModel
 from optimizer import AdamW
 
 
-TQDM_DISABLE=False
+TQDM_DISABLE=True
 
 
 # Fix the random seed.
@@ -340,14 +340,14 @@ def test(args):
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--seed", type=int, default=11711)
-    parser.add_argument("--num-cpu-cores", type=int, default=4)
+    parser.add_argument("--num-cpu-cores", type=int, default=8)
     parser.add_argument("--epochs", type=int, default=10)
     parser.add_argument("--fine-tune-mode", type=str,
                         help='last-linear-layer: the BERT parameters are frozen and the task specific head parameters are updated; full-model: BERT parameters are updated as well',
                         choices=('last-linear-layer', 'full-model'), default="last-linear-layer")
     parser.add_argument("--use_gpu", action='store_true')
 
-    parser.add_argument("--batch_size_sst", help='64 can fit a 12GB GPU', type=int, default=8)
+    parser.add_argument("--batch_size_sst", help='64 can fit a 12GB GPU', type=int, default=64)
     parser.add_argument("--batch_size_cfimdb", help='8 can fit a 12GB GPU', type=int, default=8)
     parser.add_argument("--hidden_dropout_prob", type=float, default=0.3)
     parser.add_argument("--lr", type=float, help="learning rate, default lr for 'pretrain': 1e-3, 'finetune': 1e-5",

justfile

@@ -1 +1,14 @@
-python classifier.py --num-cpu-cores 8 --batch_size_sst 64 --batch_size_cfimdb 8
+# Testing on Google Cloud VM with no GPU and 8 cpu cores.
+
+# If this doesn't meet your need, add the --use_gpu
+# or --num-cpu-cores arguments to the existing commands.
+
+
+default:
+    @just --list
+
+last-linear:
+    python classifier.py
+
+full-model:
+    python classifier.py --fine-tune-mode full-model --lr 1e-5

trainings/last-layer-no-dropout.txt

@@ -1,38 +0,0 @@
-Training Sentiment Classifier on SST...
-load 8544 data from data/ids-sst-train.csv
-load 1101 data from data/ids-sst-dev.csv
-Epoch 0: train loss :: 1.429, train acc :: 0.454, dev acc :: 0.441
-Epoch 1: train loss :: 1.297, train acc :: 0.467, dev acc :: 0.431
-Epoch 2: train loss :: 1.253, train acc :: 0.489, dev acc :: 0.450
-Epoch 3: train loss :: 1.233, train acc :: 0.491, dev acc :: 0.455
-Epoch 4: train loss :: 1.214, train acc :: 0.501, dev acc :: 0.450
-Epoch 5: train loss :: 1.211, train acc :: 0.511, dev acc :: 0.465
-Epoch 6: train loss :: 1.199, train acc :: 0.515, dev acc :: 0.478
-Epoch 7: train loss :: 1.192, train acc :: 0.518, dev acc :: 0.481
-Epoch 8: train loss :: 1.191, train acc :: 0.513, dev acc :: 0.467
-Epoch 9: train loss :: 1.191, train acc :: 0.505, dev acc :: 0.448
-Evaluating on SST...
-load model from sst-classifier.pt
-load 1101 data from data/ids-sst-dev.csv
-DONE DEV
-DONE Test
-dev acc :: 0.481
-Training Sentiment Classifier on cfimdb...
-load 1707 data from data/ids-cfimdb-train.csv
-load 245 data from data/ids-cfimdb-dev.csv
-Epoch 0: train loss :: 0.574, train acc :: 0.821, dev acc :: 0.829
-Epoch 1: train loss :: 0.466, train acc :: 0.866, dev acc :: 0.857
-Epoch 2: train loss :: 0.419, train acc :: 0.872, dev acc :: 0.873
-Epoch 3: train loss :: 0.386, train acc :: 0.878, dev acc :: 0.833
-Epoch 4: train loss :: 0.373, train acc :: 0.899, dev acc :: 0.849
-Epoch 5: train loss :: 0.357, train acc :: 0.893, dev acc :: 0.865
-Epoch 6: train loss :: 0.342, train acc :: 0.905, dev acc :: 0.873
-Epoch 7: train loss :: 0.334, train acc :: 0.906, dev acc :: 0.873
-Epoch 8: train loss :: 0.345, train acc :: 0.892, dev acc :: 0.824
-Epoch 9: train loss :: 0.321, train acc :: 0.888, dev acc :: 0.820
-Evaluating on cfimdb...
-load model from cfimdb-classifier.pt
-load 245 data from data/ids-cfimdb-dev.csv
-DONE DEV
-DONE Test
-dev acc :: 0.873

trainings/last-layer-w-dropout.txt

@@ -1,38 +0,0 @@
-Training Sentiment Classifier on SST...
-load 8544 data from data/ids-sst-train.csv
-load 1101 data from data/ids-sst-dev.csv
-Epoch 0: train loss :: 1.458, train acc :: 0.460, dev acc :: 0.442
-Epoch 1: train loss :: 1.331, train acc :: 0.472, dev acc :: 0.440
-Epoch 2: train loss :: 1.288, train acc :: 0.476, dev acc :: 0.447
-Epoch 3: train loss :: 1.269, train acc :: 0.490, dev acc :: 0.457
-Epoch 4: train loss :: 1.252, train acc :: 0.485, dev acc :: 0.446
-Epoch 5: train loss :: 1.242, train acc :: 0.487, dev acc :: 0.447
-Epoch 6: train loss :: 1.235, train acc :: 0.511, dev acc :: 0.472
-Epoch 7: train loss :: 1.235, train acc :: 0.512, dev acc :: 0.465
-Epoch 8: train loss :: 1.235, train acc :: 0.512, dev acc :: 0.472
-Epoch 9: train loss :: 1.227, train acc :: 0.509, dev acc :: 0.475
-Evaluating on SST...
-load model from sst-classifier.pt
-load 1101 data from data/ids-sst-dev.csv
-DONE DEV 
-DONE Test
-dev acc :: 0.475
-Training Sentiment Classifier on cfimdb...
-load 1707 data from data/ids-cfimdb-train.csv
-load 245 data from data/ids-cfimdb-dev.csv
-Epoch 0: train loss :: 0.590, train acc :: 0.819, dev acc :: 0.849
-Epoch 1: train loss :: 0.510, train acc :: 0.826, dev acc :: 0.845
-Epoch 2: train loss :: 0.459, train acc :: 0.848, dev acc :: 0.853
-Epoch 3: train loss :: 0.438, train acc :: 0.880, dev acc :: 0.857
-Epoch 4: train loss :: 0.413, train acc :: 0.876, dev acc :: 0.869
-Epoch 5: train loss :: 0.406, train acc :: 0.890, dev acc :: 0.833
-Epoch 6: train loss :: 0.401, train acc :: 0.893, dev acc :: 0.845
-Epoch 7: train loss :: 0.403, train acc :: 0.870, dev acc :: 0.861
-Epoch 8: train loss :: 0.393, train acc :: 0.879, dev acc :: 0.865
-Epoch 9: train loss :: 0.407, train acc :: 0.895, dev acc :: 0.873
-Evaluating on cfimdb...
-load model from cfimdb-classifier.pt
-load 245 data from data/ids-cfimdb-dev.csv
-DONE DEV 
-DONE Test                                            
-dev acc :: 0.873

unsup_simcse.py

@@ -203,14 +203,14 @@ def train(args):
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--seed", type=int, default=11711)
-    parser.add_argument("--num-cpu-cores", type=int, default=4)
+    parser.add_argument("--num-cpu-cores", type=int, default=8)
     parser.add_argument("--epochs", type=int, default=10)
     parser.add_argument("--use_gpu", action='store_true')
     parser.add_argument("--batch_size_cse", type=int, default=8)
     parser.add_argument("--batch_size_sst", type=int, default=64)
     parser.add_argument("--batch_size_cfimdb", type=int, default=8)
     parser.add_argument("--hidden_dropout_prob", type=float, default=0.3)
-    parser.add_argument("--lr_cse", type=float, default=2e-5)
+    parser.add_argument("--lr_cse", type=float, default=1e-5)
     parser.add_argument("--lr_classifier", type=float, default=1e-5)
 
     args = parser.parse_args()