Update Gradio interface and fix compatibility issues
Browse files
app.py
CHANGED
@@ -20,31 +20,41 @@ def process_text(text):
|
|
20 |
# Decode back to verify
|
21 |
decoded = tokenizer.decode(encoded)
|
22 |
|
23 |
-
return
|
24 |
-
"Encoded Tokens": str(encoded),
|
25 |
-
"Number of Tokens": len(encoded),
|
26 |
-
"Decoded Text": decoded,
|
27 |
-
"Round-trip Success": text == decoded
|
28 |
-
}
|
29 |
|
30 |
# Create Gradio interface
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
47 |
|
48 |
# Launch the app
|
49 |
if __name__ == "__main__":
|
50 |
-
iface.launch()
|
|
|
20 |
# Decode back to verify
|
21 |
decoded = tokenizer.decode(encoded)
|
22 |
|
23 |
+
return str(encoded), len(encoded), decoded, text == decoded
|
|
|
|
|
|
|
|
|
|
|
24 |
|
25 |
# Create Gradio interface
|
26 |
+
with gr.Blocks(title="Marathi BPE Tokenizer") as iface:
|
27 |
+
gr.Markdown("# Marathi BPE Tokenizer")
|
28 |
+
gr.Markdown("Enter Marathi text to see how it's tokenized using byte-pair encoding.")
|
29 |
+
|
30 |
+
with gr.Row():
|
31 |
+
input_text = gr.Textbox(label="Input Marathi Text", placeholder="नमस्कार, जग!")
|
32 |
+
|
33 |
+
with gr.Row():
|
34 |
+
process_btn = gr.Button("Tokenize")
|
35 |
+
|
36 |
+
with gr.Row():
|
37 |
+
token_ids = gr.Textbox(label="Token IDs")
|
38 |
+
token_count = gr.Number(label="Token Count")
|
39 |
+
decoded_text = gr.Textbox(label="Decoded Text")
|
40 |
+
roundtrip_success = gr.Checkbox(label="Successful Round-trip")
|
41 |
+
|
42 |
+
# Add example inputs
|
43 |
+
gr.Examples(
|
44 |
+
examples=[
|
45 |
+
["नमस्कार, जग!"],
|
46 |
+
["ही एक चाचणी आहे."],
|
47 |
+
],
|
48 |
+
inputs=input_text
|
49 |
+
)
|
50 |
+
|
51 |
+
# Set up click event
|
52 |
+
process_btn.click(
|
53 |
+
fn=process_text,
|
54 |
+
inputs=input_text,
|
55 |
+
outputs=[token_ids, token_count, decoded_text, roundtrip_success]
|
56 |
+
)
|
57 |
|
58 |
# Launch the app
|
59 |
if __name__ == "__main__":
|
60 |
+
iface.launch()
|