Update app.py

app.py

@@ -27,11 +27,17 @@ def get_first_subword(word):
 
 def search(token_to_lookup, num_neighbors=100):
     i = get_first_subword(token_to_lookup)
-    _ , I = index_IP.search(normalized_input_embeddings[i:i+1], num_neighbors)
-    hits = lookup_table.take(I[0])
-    results = hits.values[1:]
-    results = [r for r in results if not "[unused" in r]
-    return [r for r in results if not "##" in r], [r for r in results if "##" in r]
+    _ , I_IP = index_IP.search(normalized_input_embeddings[i:i+1], num_neighbors)
+    hits_IP = lookup_table.take(I_IP[0])
+    results_IP = hits_IP.values[1:]
+    results_IP = [r for r in results_IP if not "[unused" in r]
+
+    _ , I_L2 = index_L2.search(unnormalized_input_embeddings[i:i+1], num_neighbors)
+    hits_L2 = lookup_table.take(I_L2[0])
+    results_L2 = hits_L2.values[1:]
+    results_L2 = [r for r in results_L2 if not "[unused" in r]
+    
+    return [r for r in results_IP if not "##" in r], [r for r in results_IP if "##" in r], [r for r in results_L2 if not "##" in r], [r for r in results_L2 if "##" in r]
 
 
 iface = gr.Interface(
@@ -39,7 +45,7 @@ iface = gr.Interface(
     
     #inputs=[gr.Textbox(lines=1, label="Vocabulary Token", placeholder="Enter token..."), gr.Number(value=50, label="number of neighbors")],
     inputs=gr.Textbox(lines=1, label="Vocabulary Token", placeholder="Enter token..."),
-    outputs=[gr.Textbox(label="Nearest tokens"), gr.Textbox(label="Nearest subwords")],
+    outputs=[gr.Textbox(label="IP-Nearest tokens"), gr.Textbox(label="IP-Nearest subwords"), gr.Textbox(label="L2-Nearest tokens"), gr.Textbox(label="L2-Nearest subwords")],
     examples=[
         ["##logy"],
         ["##ness"],