Add app

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.11
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Try and run pip command after setting the user with `USER user` to avoid permission issues with Python
+RUN pip install --no-cache-dir --upgrade pip
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+COPY --chown=user requirements.txt .
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user app.py .
+
+ENTRYPOINT ["solara", "run", "app.py", "--host=0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Alonso Silva Allende
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -6,6 +6,7 @@ colorTo: blue
 sdk: docker
 pinned: false
 license: mit
+app_port: 7860
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,56 @@
+import pandas as pd
+import random
+import solara
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+tokenizer = AutoTokenizer.from_pretrained('gpt2')
+model = AutoModelForCausalLM.from_pretrained('gpt2')
+text = solara.reactive("Example text is here")
+text2 = solara.reactive("")
+text3 = solara.reactive("")
+
+# Create dataframe mapping token IDs and tokens
+df = pd.DataFrame()
+df["token ID"] = range(50257)
+df["token"] = [tokenizer.decode([i]) for i in range(50257)]
+
+@solara.component
+def Page():
+  with solara.Column(margin=10):
+    solara.Markdown("#GPT token encoder and decoder")
+    solara.InputText("Enter text to tokenize it:", value=text, continuous_update=True)
+    tokens = tokenizer.encode(text.value, return_tensors="pt")
+    spans = ""
+    spans1 = ""
+    for i, token in enumerate(tokens[0]):
+      random.seed(i)
+      random_color = ''.join([random.choice('0123456789ABCDEF') for k in range(6)])
+      spans += " " + f"<span style='font-family: cursive;color: #{random_color}'>{token.numpy()}</span>"
+      spans1 += " " + f"""<span style="
+          padding: 5px;
+          border-right: 3px solid white;
+          line-height: 3em;
+          font-family: courier;
+          background-color: #{random_color};
+          color: white;
+          position: relative;
+        "><span style="position: absolute; top: 5.5ch; line-height: 1em; left: -0.5px; font-size: 0.45em">{token.numpy()}</span>{tokenizer.decode(token)}</span>"""
+    solara.Markdown(f"{spans}")
+    if len(tokens[0]) == 1:
+      solara.Markdown(f"{len(tokens[0])} token")
+    else:
+      solara.Markdown(f"{len(tokens[0])} tokens")
+    solara.Markdown(f'{spans1}')
+    solara.InputText("Or convert space separated tokens to text:", value=text2, continuous_update=True)
+    spans2 = text2.value.split(' ')
+    spans2 = [int(span) for span in spans2 if span != ""]
+    spans2 = tokenizer.decode(spans2)
+    solara.Markdown(f'{spans2}')
+    solara.Markdown("##Search tokens")
+    solara.InputText("Search for a token:", value=text3, continuous_update=True)
+    df_subset = df[df["token"].str.startswith(text3.value)]
+    solara.Markdown(f"{df_subset.shape[0]:,} results")
+    solara.DataFrame(df_subset, items_per_page=10)
+
+Page()
+

requirements.txt

@@ -0,0 +1,3 @@
+solara
+pandas
+transformers[torch]