Chrome extension

app.py

@@ -311,9 +311,9 @@ with gr.Blocks() as demo:
 
             We'd love your feedback! Please indicate whether you agree or disagree with Vera's prediction (and don't mind the percentage numbers). If you're unsure or the statement doesn't have a certain correctness label, please select "Uncertain". If your input is actually not a statement about commonsense, please select "I don't think this is a statement about commonsense".
 
-            :warning: **Intended Use**: Vera is a research prototype and may make mistakes. Do not use for making critical decisions. It is intended to predict the correctness of commonsense statements, and may be unreliable when taking input out of this scope. **DO NOT input encyclopedic facts.** Vera is trained on **English** data only, please do not input statements in other languages.
+            **Intended Use**: Vera is a research prototype and may make mistakes. Do not use for making critical decisions. It is intended to predict the correctness of commonsense statements, and may be unreliable when taking input out of this scope. **DO NOT input encyclopedic facts.** Vera is trained on **English** data only, please do not input statements in other languages.
 
-            :warning: **Data Collection**: By default, we are collecting the inputs entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
+            **Data Collection**: By default, we are collecting the inputs entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
             '''
         )
         with gr.Row():

backend/README.md

@@ -0,0 +1,49 @@
+# Vera backend
+
+[http://qa.cs.washington.edu:8372/](http://qa.cs.washington.edu:8372/)
+
+This demo runs on port 8372, so make sure this port is whitelisted in UFW:
+```
+sudo ufw allow 8372
+sudo ufw status
+```
+Also, make sure port 80 and 443 are whitelisted.
+
+This demo requires HTTPS, so make sure HTTPS is enabled on this server.
+This is what I did:
+1. Follwing <https://blog.miguelgrinberg.com/post/running-your-flask-application-over-https>
+1. Make sure nginx is running: `sudo service nginx status`
+1. Add the following lines to `/etc/nginx/nginx.conf`
+```
+    server {
+        listen 80; 
+        server_name qa.cs.washington.edu;
+        location ~ /.well-known {
+            root /home/gary/cd-pi-demo/backend;
+        }   
+        location / { 
+            return 301 https://$host$request_uri;
+        }   
+    }   
+```
+1. Use certbot to create a certificate: `sudo certbot certonly --webroot -w /home/gary/cd-pi-demo/backend/ -d qa.cs.washington.edu`
+1. Add the following lines to `/etc/nginx/nginx.conf`
+```
+    server {
+        listen 443 ssl http2;
+        server_name qa.cs.washington.edu;
+        ssl_certificate /etc/letsencrypt/live/qa.cs.washington.edu/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/qa.cs.washington.edu/privkey.pem;
+        # ... 
+    }   
+```
+
+Then, run the following command to spin up the demo:
+```
+CUDA_VISIBLE_DEVICES=1 sudo ~/anaconda3/envs/default/bin/python run.py
+```
+
+To test the demo, run this command in bash:
+```
+curl https://qa.cs.washington.edu:8372 -X POST -d '{"statement": "Hello."}' -H "content-type: application/json"
+```

backend/run.py

@@ -0,0 +1,66 @@
+from flask import Flask, render_template, redirect, request, jsonify, make_response
+import datetime
+
+import torch
+import transformers
+
+device = torch.device('cuda')
+
+MODEL_NAME = 'liujch1998/vera'
+
+class Interactive:
+    def __init__(self):
+        self.tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)
+        self.model = transformers.T5EncoderModel.from_pretrained(MODEL_NAME, low_cpu_mem_usage=True, device_map='auto', torch_dtype='auto', offload_folder='offload')
+        self.model.D = self.model.shared.embedding_dim
+        self.linear = torch.nn.Linear(self.model.D, 1, dtype=self.model.dtype).to(device)
+        self.linear.weight = torch.nn.Parameter(self.model.shared.weight[32099, :].unsqueeze(0)) # (1, D)
+        self.linear.bias = torch.nn.Parameter(self.model.shared.weight[32098, 0].unsqueeze(0)) # (1)
+        self.model.eval()
+        self.t = self.model.shared.weight[32097, 0].item()
+
+    def run(self, statement):
+        input_ids = self.tokenizer.batch_encode_plus([statement], return_tensors='pt', padding='longest', truncation='longest_first', max_length=128).input_ids.to(device)
+        with torch.no_grad():
+            output = self.model(input_ids)
+            last_hidden_state = output.last_hidden_state.to(device) # (B=1, L, D)
+            hidden = last_hidden_state[0, -1, :] # (D)
+            logit = self.linear(hidden).squeeze(-1) # ()
+            logit_calibrated = logit / self.t
+            score = logit.sigmoid()
+            score_calibrated = logit_calibrated.sigmoid()
+        return {
+            'timestamp': datetime.datetime.now().strftime('%Y%m%d-%H%M%S'),
+            'statement': statement,
+            'logit': logit.item(),
+            'logit_calibrated': logit_calibrated.item(),
+            'score': score.item(),
+            'score_calibrated': score_calibrated.item(),
+        }
+
+interactive = Interactive()
+app = Flask(__name__)
+
+@app.route('/', methods=['GET', 'POST'])
+def main():
+    try:
+        print(request)
+        data = request.get_json()
+        statement = data.get('statement')
+    except Exception as e:
+        return jsonify({
+            'success': False,
+            'message': 'Please provide a statement.',
+        }), 400
+    try:
+        result = interactive.run(statement)
+    except Exception as e:
+        return jsonify({
+            'success': False,
+            'message': 'Internal error.',
+        }), 500
+    return jsonify(result)
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8372, threaded=True, ssl_context=('/etc/letsencrypt/live/qa.cs.washington.edu/fullchain.pem', '/etc/letsencrypt/live/qa.cs.washington.edu/privkey.pem'))
+    # 8372 is when you type Vera on a phone keypad

chrome-ext/README.md

@@ -0,0 +1,7 @@
+# Vera Chrome Extension
+
+This is a Chrome extension that verifies commonsense statements in-the-wild, using Vera.
+
+To install, go to `chrome://extensions`, enable developer mode, and click "Load unpacked extension". Select the `chrome-extension` directory in this repository.
+
+Make sure that the backend is running on `https://qa.cs.washington.edu:8372` before using the extension.

chrome-ext/background.js

@@ -0,0 +1,54 @@
+chrome.action.onClicked.addListener((tab) => {
+  chrome.scripting.executeScript({
+    target: { tabId: tab.id },
+    func: getHighlightedText,
+  }, (result) => {
+    if (chrome.runtime.lastError) {
+      console.error(chrome.runtime.lastError.message);
+      return;
+    }
+    if (result.length == 0) {
+      console.error('No result returned by getHighlightedText!');
+      return;
+    }
+    let text = result[0].result;
+    if (text.length > 0) {
+      getPlausibility(text, tab);
+    }
+  });
+})
+
+// this function is executed in the context of the current tab
+function getHighlightedText() {
+  let text = window.getSelection().toString();
+  console.log(`Highlighted text: ${text}`);
+  if (text.length == 0) {
+    alert('Please highlight some text first!');
+  }
+  return text;
+}
+
+// this function is executed in the context of the extension background
+function getPlausibility(text, tab) {
+  const URL = 'https://qa.cs.washington.edu:8372';
+  const data = { statement: text };
+  fetch(URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(data),
+  })
+  .then(response => response.json())
+  .then(output => {
+    chrome.scripting.executeScript({
+      target: { tabId: tab.id },
+      func: (statement, score) => {
+        disp_score = Math.round(score * 100);
+        alert(`Statement: ${statement}\nPlausibility: ${disp_score}%`);
+      },
+      args: [output.statement, output.score_calibrated],
+    });
+  })
+  .catch((error) => console.error('Error:', error));
+}

chrome-ext/manifest.json

@@ -0,0 +1,35 @@
+{
+    "manifest_version": 3,
+    "name": "Vera",
+    "description": "Verify commonsense statements in-the-wild.",
+    "version": "1.0",
+    "icons": {
+      "16": "images/aloe-16.png",
+      "32": "images/aloe-32.png",
+      "48": "images/aloe-48.png",
+      "128": "images/aloe-128.png"
+    },
+    "background": {
+      "service_worker": "background.js"
+    },
+    "action": {
+      "default_icon": {
+        "16": "images/aloe-16.png",
+        "32": "images/aloe-32.png",
+        "48": "images/aloe-48.png",
+        "128": "images/aloe-128.png"
+      }
+    },
+    "permissions": ["scripting", "activeTab"],
+    "host_permissions": [
+        "https://qa.cs.washington.edu/"
+    ],
+    "commands": {
+      "_execute_action": {
+        "suggested_key": {
+          "default": "Ctrl+B",
+          "mac": "Command+B"
+        }
+      }
+    }
+  }