test

.dockerignore

@@ -1,2 +0,0 @@
-lens
-.git

.gitattributes

@@ -32,4 +32,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-*.pyc filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,25 +0,0 @@
-FROM python:3.9
-
-WORKDIR /code
-
-COPY ./requirements.txt /code/requirements.txt
-
-RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
-
-# Set up a new user named "user" with user ID 1000
-RUN useradd -m -u 1000 user
-
-# Switch to the "user" user
-USER user
-
-# Set home to the user's home directory
-ENV HOME=/home/user \
-    PATH=/home/user/.local/bin:$PATH
-
-# Set the working directory to the user's home directory
-WORKDIR $HOME/app
-
-# Copy the current directory contents into the container at $HOME/app setting the owner to the user
-COPY --chown=user . $HOME/app
-
-CMD ["python", "app.py"]

README.md

@@ -3,7 +3,10 @@ title: Tuned Lens
 emoji: 🔎
 colorFrom: pink
 colorTo: blue
-sdk: docker
+sdk: gradio
+python_version: 3.10.2
+sdk_version: 3.20.0
+app_file: app.py
 pinned: false
 license: mit
 ---

app.py

@@ -1,20 +1,17 @@
 import torch
 from tuned_lens.nn.lenses import TunedLens, LogitLens
 from transformers import AutoModelForCausalLM, AutoTokenizer
-from tuned_lens.plotting import PredictionTrajectory
+from tuned_lens.plotting import plot_lens
 import gradio as gr
 from plotly import graph_objects as go
 
 device = torch.device("cpu")
 print(f"Using device {device} for inference")
-model = AutoModelForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped")
+model = AutoModelForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped-v0")
 model = model.to(device)
-tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped")
-tuned_lens = TunedLens.from_model_and_pretrained(
-    model=model,
-    map_location=device,
-)
-logit_lens = LogitLens.from_model(model)
+tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped-v0")
+tuned_lens = TunedLens.load("pythia-410m-deduped-v0", map_location=device)
+logit_lens = LogitLens(model)
 
 lens_options_dict = {
     "Tuned Lens": tuned_lens,
@@ -23,35 +20,32 @@ lens_options_dict = {
 
 statistic_options_dict = {
     "Entropy": "entropy",
-    "Cross Entropy": "cross_entropy",
+    "Cross Entropy": "ce",
     "Forward KL": "forward_kl",
 }
 
 
 def make_plot(lens, text, statistic, token_cutoff):
-    input_ids = tokenizer.encode(text)
-    input_ids = [tokenizer.bos_token_id] + input_ids
-    targets = input_ids[1:] + [tokenizer.eos_token_id]
+    input_ids = tokenizer.encode(text, return_tensors="pt")
 
-    if len(input_ids) == 1:
+    if len(input_ids[0]) == 0:
         return go.Figure(layout=dict(title="Please enter some text."))
 
     if token_cutoff < 1:
         return go.Figure(layout=dict(title="Please provide valid token cut off."))
 
-    start_pos=max(len(input_ids) - token_cutoff, 0)
-    pred_traj = PredictionTrajectory.from_lens_and_model(
-        lens=lens_options_dict[lens],
-        model=model,
+    fig = plot_lens(
+        model,
+        tokenizer,
+        lens_options_dict[lens],
+        layer_stride=2,
         input_ids=input_ids,
-        tokenizer=tokenizer,
-        targets=targets,
-        start_pos=start_pos,
+        start_pos=max(len(input_ids[0]) - token_cutoff, 0),
+        statistic=statistic_options_dict[statistic],
     )
 
-    return getattr(pred_traj, statistic_options_dict[statistic])().figure(
-        title=f"{lens} ({model.name_or_path}) {statistic}",
-    )
+    return fig
+
 
 preamble = """
 # The Tuned Lens 🔎
@@ -114,4 +108,5 @@ with gr.Blocks() as demo:
     demo.load(make_plot, [lens_options, text, statistic, token_cutoff], plot)
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch()
+

lens/CarperAI/stable-vicuna-13b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "vicuna-13b", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "270c781d8280754a30abaf6fc186b1d754cc5d6bb17173d572d5517d5c7c702c", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/gpt-neox-20b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/gpt-neox-20b", "d_model": 6144, "num_hidden_layers": 44, "bias": true, "base_model_revision": "4e49eadb5d14bd22f314ec3f45b69a87b88c7691", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-1.4b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-1.4b-deduped-v0", "d_model": 2048, "num_hidden_layers": 24, "bias": true, "base_model_revision": "b541e01fddacd3038799915cf8ff5b52e835a6c4", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-1.4b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-1.4b-deduped", "d_model": 2048, "num_hidden_layers": 24, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-12b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-12b-deduped-v0", "d_model": 5120, "num_hidden_layers": 36, "bias": true, "base_model_revision": "b497662035bf3c80b4f6a1ddfe09bc27763e843a", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-12b-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a9e34c28c900b7230afe7ae1dc99b58b82c22b3265de2cbf939e3946cdcec126
-size 3775627331

lens/EleutherAI/pythia-12b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-12b-deduped", "d_model": 5120, "num_hidden_layers": 36, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-12b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8452947780e3caf7009a752d62f1a2d71c66f67b3855c7e409869b7e633db40e
-size 3775628355

lens/EleutherAI/pythia-160m-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-160m-deduped-v0", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": "7e57cc978f5da949f028f36b5baf8f5d6c3281b1", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-160m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-160m-deduped", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-1b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-1b-deduped-v0", "d_model": 2048, "num_hidden_layers": 16, "bias": true, "base_model_revision": "021f79f50ff000ae1c159e22402ffec62284664d", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-1b-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e10ed1a0b10251aaf1b7c542f14ad2c7a850103dfdffcd9da3d21e107d779eeb
-size 268573731

lens/EleutherAI/pythia-2.8b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-2.8b-deduped-v0", "d_model": 2560, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-2.8b-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f7892bd4af71a436aaba3a650ea907c94a077b8f16eb343c9a8762872c73026f
-size 839204003

lens/EleutherAI/pythia-2.8b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-2.8b-deduped", "d_model": 2560, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-2.8b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:314d403f2e1b1bf575ab2e851419d3a687c54138bfae5feace3ff00f1a96fd60
-size 839204003

lens/EleutherAI/pythia-410m-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-410m-deduped-v0", "d_model": 1024, "num_hidden_layers": 24, "bias": true, "base_model_revision": "3538d3569a7e313e445ad6401c92c6e16777a2da", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-410m-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:257a17d052d6bda84f338b1aa22ded8c17c401e1299b64cdab521178708ee7ac
-size 100772515

lens/EleutherAI/pythia-410m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-410m-deduped", "d_model": 1024, "num_hidden_layers": 24, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-410m-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:166ea259b35481e1eb2feba50b5ac4d9a8faed47b0937ede0d7bd6d9830dbc95
-size 100773155

lens/EleutherAI/pythia-6.9b-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-6.9b-deduped-v0", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": "cbd53efc2e56056e3bd0235277b5d0b668a6dfbb", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-6.9b-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c99da7167d4a45a71c2f04c2db4d716a7a63f27406cb8948c9cb9f6c052b91c
-size 2148022563

lens/EleutherAI/pythia-6.9b-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-6.9b-deduped", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-6.9b-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e2d00af6f64631b932b089fac5ca031061e207d51132a0f7433bd7a34fb06da0
-size 2148023459

lens/EleutherAI/pythia-70m-deduped-v0/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-70m-deduped-v0", "d_model": 512, "num_hidden_layers": 6, "bias": true, "base_model_revision": "ec30f7539a604fcb0b7fbba04fb1eb0110735d29", "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-70m-deduped-v0/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cbfc06e4d8733b1fccd5e1bb9cabd845072abfc8b92902ae1bbcbb2763fbc014
-size 6306739

lens/EleutherAI/pythia-70m-deduped/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "EleutherAI/pythia-70m-deduped", "d_model": 512, "num_hidden_layers": 6, "bias": true, "base_model_revision": null, "lens_type": "linear_tuned_lens"}

lens/EleutherAI/pythia-70m-deduped/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c363b641564af68e9f9b73af56015b6b5c5b08caea5880787744eaabfab01343
-size 6306803

lens/facebook/llama-13b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-13b", "d_model": 5120, "num_hidden_layers": 40, "bias": true, "base_model_revision": null, "unembed_hash": "86cefdfd94bb3da225b405dd1328136786f6177b03d82f90e5c734d23c47e8ca", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-13b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2700859b4335f4e4bdd4040232f594170bb717f8af04f65e5560c49cfb6da122
-size 2097581027

lens/facebook/llama-30b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-30b", "d_model": 6656, "num_hidden_layers": 60, "bias": true, "base_model_revision": null, "unembed_hash": "512b41bb8fc0e2b3647a877b5e114ca9503d7c800185b14a8e7ad9e921424367", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-30b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:10e781269b476a6de8f4ac12ea3f8b20fe22d3a8ca8cfdf770366a5f4e36c78f
-size 5317111487

lens/facebook/llama-65b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-65b", "d_model": 8192, "num_hidden_layers": 80, "bias": true, "base_model_revision": null, "unembed_hash": "8ffc2dbf80dd5c131c4be9e347d090856bef2dbc973433dbd42ca9257b00d5e1", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-65b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c9199874796630e1be615b0b34cd48e7f37ed09c6140153cdf6b7c509c45361b
-size 10738779199

lens/facebook/llama-7b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "huggyllama/llama-7b", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": null, "unembed_hash": "7d0c2d720d286bdd706e662ea04f327204090f7d54054b0d5faabbc1b06a72fe", "lens_type": "linear_tuned_lens"}

lens/facebook/llama-7b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02186a08b29e7173ae229218823b65891a7b2f2eadf0e69baf44a9c62b55a7cf
-size 1074019491

lens/facebook/opt-1.3b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "facebook/opt-1.3b", "d_model": 2048, "num_hidden_layers": 24, "bias": true, "base_model_revision": "8c7b10754972749675d22364c25c428b29face51", "lens_type": "linear_tuned_lens"}

lens/facebook/opt-1.3b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2b5620752d01fcca26f5b7c36ec2b741974b570adfb64bffbd901cdc01dc9df9
-size 402860707

lens/facebook/opt-125m/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "facebook/opt-125m", "d_model": 768, "num_hidden_layers": 12, "bias": true, "base_model_revision": "3d2b5f275bdf882b8775f902e1bfdb790e2cfc32", "lens_type": "linear_tuned_lens"}

lens/facebook/opt-125m/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f61d7716618783244f2e7f5d26022df092b80695e9620ccb02c5ffb70f9b1405
-size 28353795

lens/facebook/opt-6.7b/config.json

@@ -1 +0,0 @@
-{"base_model_name_or_path": "facebook/opt-6.7b", "d_model": 4096, "num_hidden_layers": 32, "bias": true, "base_model_revision": "a45aa65bbeb77c1558bc99bedc6779195462dab0", "lens_type": "linear_tuned_lens"}

lens/facebook/opt-6.7b/params.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9580fbf6e967c1be832ef415f3f74c0f6ee111637fba253dc5a75216dc305ebb
-size 2148022563

lens/gpt-neox-20b/config.json

@@ -0,0 +1 @@
+{"bias": true, "identity_init": true, "include_input": true, "include_final": false, "orthogonal": false, "rank": null, "sublayers": false, "d_model": 6144, "num_layers": 44, "vocab_size": 50432}

lens/{EleutherAI/gpt-neox-20b → gpt-neox-20b}/params.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3c0e5911cbdabd33e5a59112eefb4f234a487fac77f830be7fc238ffb72e776
-size 6644881483
+oid sha256:774144e329ee3741d0a7fff85b342f1a996aa24c788c3f741b03df596948a66d
+size 3942186487