add requirements, sense vecs, lm head

requirements.txt

@@ -0,0 +1,71 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==22.2.0
+certifi @ file:///Users/cbousseau/work/recipes/ci_py311/certifi_1677903144932/work/certifi
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+entrypoints==0.4
+fastapi==0.95.0
+ffmpy==0.3.0
+filelock==3.10.7
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.3.0
+gradio==3.24.1
+gradio_client==0.0.7
+h11==0.14.0
+httpcore==0.16.3
+httpx==0.23.3
+huggingface-hub==0.13.3
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+networkx==3.1
+numpy==1.24.2
+orjson==3.8.9
+packaging==23.0
+pandas==2.0.0
+Pillow==9.5.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+regex==2023.3.23
+requests==2.28.2
+rfc3986==1.5.0
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+sympy==1.11.1
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.0
+tqdm==4.65.0
+transformers==4.27.4
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0
+yarl==1.8.2

senses/all_vecs_mtx.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f0c9de5688dd793470c40ebc3b49c29be6ddbf9a38804bca64512940671e129
+size 2470232826

senses/lm_head.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f94054e64b4d1a07e18443769df4d3b9e346c00b02ffe4e9579e8313034dac24
+size 154411755

senses/use_senses.py

@@ -0,0 +1,44 @@
+"""Visualize some sense vectors"""
+
+import torch
+import argparse
+
+import transformers
+
+def visualize_word(word, tokenizer, vecs, lm_head, count=20, contents=None):
+  """
+  Prints out the top-scoring words (and lowest-scoring words) for each sense.
+
+  """
+  if contents is None:
+    print(word)
+    token_id = tokenizer(word)['input_ids'][0]
+    contents = vecs[token_id] # torch.Size([16, 768])
+
+  for i in range(contents.shape[0]):
+    print('~~~~~~~~~~~~~~~~~~~~~~~{}~~~~~~~~~~~~~~~~~~~~~~~~'.format(i))
+    logits = contents[i,:] @ lm_head.t() # (vocab,)    [768] @ [768, 50257] -> [50257]
+    sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+    print('~~~Positive~~~')
+    for j in range(count):
+      print(tokenizer.decode(sorted_indices[j]), '\t','{:.2f}'.format(sorted_logits[j].item()))
+    print('~~~Negative~~~')
+    for j in range(count):
+      print(tokenizer.decode(sorted_indices[-j-1]), '\t','{:.2f}'.format(sorted_logits[-j-1].item()))
+  return contents
+  print()
+  print()
+  print()
+
+argp = argparse.ArgumentParser()
+argp.add_argument('vecs_path')
+argp.add_argument('lm_head_path')
+args = argp.parse_args()
+
+# Load tokenizer and parameters
+tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
+vecs = torch.load(args.vecs_path)
+lm_head = torch.load(args.lm_head_path)
+
+visualize_word(input('Enter a word:'), tokenizer, vecs, lm_head, count=5)
+