Remove migration utils

.dockerignore

@@ -1,2 +0,0 @@
-lens
-.git

.gitattributes

@@ -32,4 +32,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-*.pyc filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,25 +0,0 @@
-FROM python:3.9
-
-WORKDIR /code
-
-COPY ./requirements.txt /code/requirements.txt
-
-RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
-
-# Set up a new user named "user" with user ID 1000
-RUN useradd -m -u 1000 user
-
-# Switch to the "user" user
-USER user
-
-# Set home to the user's home directory
-ENV HOME=/home/user \
-    PATH=/home/user/.local/bin:$PATH
-
-# Set the working directory to the user's home directory
-WORKDIR $HOME/app
-
-# Copy the current directory contents into the container at $HOME/app setting the owner to the user
-COPY --chown=user . $HOME/app
-
-CMD ["python", "app.py"]

README.md

@@ -3,7 +3,10 @@ title: Tuned Lens
 emoji: 🔎
 colorFrom: pink
 colorTo: blue
-sdk: docker
+sdk: gradio
+python_version: 3.10.2
+sdk_version: 3.20.0
+app_file: app.py
 pinned: false
 license: mit
 ---

app.py

@@ -1,20 +1,17 @@
 import torch
 from tuned_lens.nn.lenses import TunedLens, LogitLens
 from transformers import AutoModelForCausalLM, AutoTokenizer
-from tuned_lens.plotting import PredictionTrajectory
+from tuned_lens.plotting import plot_lens
 import gradio as gr
 from plotly import graph_objects as go
 
 device = torch.device("cpu")
 print(f"Using device {device} for inference")
-model = AutoModelForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped")
+model = AutoModelForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped-v0")
 model = model.to(device)
-tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped")
-tuned_lens = TunedLens.from_model_and_pretrained(
-    model=model,
-    map_location=device,
-)
-logit_lens = LogitLens.from_model(model)
+tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped-v0")
+tuned_lens = TunedLens.load("pythia-410m-deduped-v0", map_location=device)
+logit_lens = LogitLens(model)
 
 lens_options_dict = {
     "Tuned Lens": tuned_lens,
@@ -23,35 +20,32 @@ lens_options_dict = {
 
 statistic_options_dict = {
     "Entropy": "entropy",
-    "Cross Entropy": "cross_entropy",
+    "Cross Entropy": "ce",
     "Forward KL": "forward_kl",
 }
 
 
 def make_plot(lens, text, statistic, token_cutoff):
-    input_ids = tokenizer.encode(text)
-    input_ids = [tokenizer.bos_token_id] + input_ids
-    targets = input_ids[1:] + [tokenizer.eos_token_id]
+    input_ids = tokenizer.encode(text, return_tensors="pt")
 
-    if len(input_ids) == 1:
+    if len(input_ids[0]) == 0:
         return go.Figure(layout=dict(title="Please enter some text."))
 
     if token_cutoff < 1:
         return go.Figure(layout=dict(title="Please provide valid token cut off."))
 
-    start_pos=max(len(input_ids) - token_cutoff, 0)
-    pred_traj = PredictionTrajectory.from_lens_and_model(
-        lens=lens_options_dict[lens],
-        model=model,
+    fig = plot_lens(
+        model,
+        tokenizer,
+        lens_options_dict[lens],
+        layer_stride=2,
         input_ids=input_ids,
-        tokenizer=tokenizer,
-        targets=targets,
-        start_pos=start_pos,
+        start_pos=max(len(input_ids[0]) - token_cutoff, 0),
+        statistic=statistic_options_dict[statistic],
     )
 
-    return getattr(pred_traj, statistic_options_dict[statistic])().figure(
-        title=f"{lens} ({model.name_or_path}) {statistic}",
-    )
+    return fig
+
 
 preamble = """
 # The Tuned Lens 🔎
@@ -114,4 +108,5 @@ with gr.Blocks() as demo:
     demo.load(make_plot, [lens_options, text, statistic, token_cutoff], plot)
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch()
+

lens/CarperAI/stable-vicuna-13b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "vicuna-13b", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "270c781d8280754a30abaf6fc186b1d754cc5d6bb17173d572d5517d5c7c702c", "lens_type": "linear_tuned_lens"}

lens/CarperAI/stable-vicuna-13b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:47afb57632ac509a309e75da139e666ba5c7e93f2a305b2e286d7e748b7cc476
-size 2097581027

lens/EleutherAI/pythia-1.4b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-1.4b-deduped", "d_model": 2048, "num_hidden_layers": 24, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-1.4b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b56db530d2c0df1bc5916bae58b241cfd4389dd4b1aa29e7210395df97164824
-size 402861347

lens/EleutherAI/pythia-12b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-12b-deduped", "d_model": 5120, "num_hidden_layers": 36, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-12b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8452947780e3caf7009a752d62f1a2d71c66f67b3855c7e409869b7e633db40e
-size 3775628355

lens/EleutherAI/pythia-160m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-160m-deduped", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-160m-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1d16b68baa1eb903bbf74eaf859cee5c34361f3e8dff331b17f55b8ce52a2dbe
-size 28354051

lens/EleutherAI/pythia-2.8b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-2.8b-deduped-v0", "d_model": 2560, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-2.8b-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f7892bd4af71a436aaba3a650ea907c94a077b8f16eb343c9a8762872c73026f
-size 839204003

lens/EleutherAI/pythia-2.8b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-2.8b-deduped", "d_model": 2560, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-2.8b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:314d403f2e1b1bf575ab2e851419d3a687c54138bfae5feace3ff00f1a96fd60
-size 839204003

lens/EleutherAI/pythia-410m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-410m-deduped", "d_model": 1024, "num_hidden_layers": 24, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-410m-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:166ea259b35481e1eb2feba50b5ac4d9a8faed47b0937ede0d7bd6d9830dbc95
-size 100773155

lens/EleutherAI/pythia-6.9b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-6.9b-deduped", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-6.9b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e2d00af6f64631b932b089fac5ca031061e207d51132a0f7433bd7a34fb06da0
-size 2148023459

lens/EleutherAI/pythia-70m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-70m-deduped", "d_model": 512, "num_hidden_layers": 6, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-70m-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c363b641564af68e9f9b73af56015b6b5c5b08caea5880787744eaabfab01343
-size 6306803

lens/facebook/llama-13b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-13b", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "86cefdfd94bb3da225b405dd1328136786f6177b03d82f90e5c734d23c47e8ca", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-13b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2700859b4335f4e4bdd4040232f594170bb717f8af04f65e5560c49cfb6da122
-size 2097581027

lens/facebook/llama-30b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-30b", "d_model": 6656, "num_hidden_layers": 60, "bias": true, "base_model_revision": null, "unembed_hash": "512b41bb8fc0e2b3647a877b5e114ca9503d7c800185b14a8e7ad9e921424367", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-30b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:10e781269b476a6de8f4ac12ea3f8b20fe22d3a8ca8cfdf770366a5f4e36c78f
-size 5317111487

lens/facebook/llama-65b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-65b", "d_model": 8192, "num_hidden_layers": 80, "bias": true, "base_model_revision": null, "unembed_hash": "8ffc2dbf80dd5c131c4be9e347d090856bef2dbc973433dbd42ca9257b00d5e1", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-65b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c9199874796630e1be615b0b34cd48e7f37ed09c6140153cdf6b7c509c45361b
-size 10738779199

lens/facebook/llama-7b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-7b", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "unembed_hash": "7d0c2d720d286bdd706e662ea04f327204090f7d54054b0d5faabbc1b06a72fe", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-7b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02186a08b29e7173ae229218823b65891a7b2f2eadf0e69baf44a9c62b55a7cf
-size 1074019491

lens/{EleutherAI/gpt-neox-20b → gpt-neox-20b}/config.json

@@ -1 +1 @@
-{"base_model_name_or_path": "EleutherAI/gpt-neox-20b", "d_model": 6144, "num_hidden_layers": 44, "bias": true, "base_model_revision": "4e49eadb5d14bd22f314ec3f45b69a87b88c7691", "lens_type": "linear_tuned_lens"}
+{"base_model_name_or_path": "EleutherAI/gpt-neox-20b", "d_model": 6144, "num_hidden_layers": 44, "bias": true, "base_model_revision": "4e49eadb5d14bd22f314ec3f45b69a87b88c7691", "unemebd_hash": "323d4c731c33556e143503e3be913c109ead330080b4065552be97000c19ed67", "lens_type": "linear_tuned_lens"}

lens/gpt2-large/config.json

@@ -1 +1 @@
-{"base_model_name_or_path": "gpt2-large", "d_model": 1280, "num_hidden_layers": 36, "bias": true, "base_model_revision": "212095d5832abbf9926672e1c1e8d14312a3be20", "lens_type": "linear_tuned_lens"}
+{"base_model_name_or_path": "gpt2-large", "d_model": 1280, "num_hidden_layers": 36, "bias": true, "base_model_revision": "212095d5832abbf9926672e1c1e8d14312a3be20", "unemebd_hash": "9b7da774c0a326716dca888539370ddff25804795949e5ace65ef9f761f47397", "lens_type": "linear_tuned_lens"}

lens/gpt2-xl/config.json

@@ -1 +1 @@
-{"base_model_name_or_path": "gpt2-xl", "d_model": 1600, "num_hidden_layers": 48, "bias": true, "base_model_revision": "33cdb5c0db5423c1879b1b9f16c352988e8754a8", "lens_type": "linear_tuned_lens"}
+{"base_model_name_or_path": "gpt2-xl", "d_model": 1600, "num_hidden_layers": 48, "bias": true, "base_model_revision": "33cdb5c0db5423c1879b1b9f16c352988e8754a8", "unemebd_hash": "70bf58a8cf7964b39530e30fdaebb89de39489546244437b1ed56fb81bd4c746", "lens_type": "linear_tuned_lens"}

lens/gpt2/config.json

@@ -1 +1 @@
-{"base_model_name_or_path": "gpt2", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": "e7da7f221d5bf496a48136c0cd264e630fe9fcc8", "lens_type": "linear_tuned_lens"}
+{"base_model_name_or_path": "gpt2", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": "e7da7f221d5bf496a48136c0cd264e630fe9fcc8", "unemebd_hash": "608e50247f57691c90453601e854f2287141e4db9cba436af0b0186003e2daae", "lens_type": "linear_tuned_lens"}

lens/lmsys/vicuna-13b-v1.1/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "vicuna-original-13b", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "86cefdfd94bb3da225b405dd1328136786f6177b03d82f90e5c734d23c47e8ca", "lens_type": "linear_tuned_lens"}

lens/lmsys/vicuna-13b-v1.1/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0281f46b65b728cb390e41bd5848d8ab635b2847b48c080b2de2b78627baa40d
-size 2097581027

lens/meta-llama/Llama-2-13b-chat-hf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-13b-chat-hf", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "05e6ab11c049cc1356b38bf9ff84ffbedaf802a3c7fdda9e763bca0c4e9de2ab", "lens_type": "linear_tuned_lens"}

lens/meta-llama/Llama-2-13b-chat-hf/hh-rlhf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-13b-chat-hf", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "05e6ab11c049cc1356b38bf9ff84ffbedaf802a3c7fdda9e763bca0c4e9de2ab", "lens_type": "linear_tuned_lens"}

lens/meta-llama/Llama-2-13b-chat-hf/hh-rlhf/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6ceabb160e1b3ff887232514159bdf22c7f109a585759284eb7b18e72fb27667
-size 2097581027

lens/meta-llama/Llama-2-13b-chat-hf/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6a8af029394d9be03e6a9a75a03ac477fdb9e42352ad3df035439bbb8ccb1b5d
-size 2097581027

lens/meta-llama/Llama-2-13b-hf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-13b-hf", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "1f421a88f21fcda6d1c8c17a481c46918d355f8b06cfc77b05cf738e11b55f7a", "lens_type": "linear_tuned_lens"}

lens/meta-llama/Llama-2-13b-hf/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f87b2a5f2b3125d9e49343cbac244b901795e3af4117ec0f6ee428f79027778d
-size 2097581027

lens/meta-llama/Llama-2-7b-chat-hf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "unembed_hash": "3313b62209f366e7b19233ae7068501553350611c20f96caf2499c0486f8836b", "lens_type": "linear_tuned_lens"}

lens/meta-llama/Llama-2-7b-chat-hf/hh-rlhf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "unembed_hash": "3313b62209f366e7b19233ae7068501553350611c20f96caf2499c0486f8836b", "lens_type": "linear_tuned_lens"}

lens/meta-llama/Llama-2-7b-chat-hf/hh-rlhf/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:86abc8c2a2044c3a70e88291b636222a6db6cdb25af51e86108e479843a2326e
-size 2148023459

lens/meta-llama/Llama-2-7b-chat-hf/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c66e7d56afd044bc6472e91bf6103ee5ab837c5953b44c791b0f1f8ac8ce8578
-size 2148023459

lens/meta-llama/Llama-2-7b-hf/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "meta-llama/Llama-2-7b-hf", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "unembed_hash": "3313b62209f366e7b19233ae7068501553350611c20f96caf2499c0486f8836b", "lens_type": "linear_tuned_lens"}