nragrawal commited on
Commit
21e0069
·
1 Parent(s): e866ca8

Update Gradio interface and fix compatibility issues

Browse files
Files changed (1) hide show
  1. app.py +33 -23
app.py CHANGED
@@ -20,31 +20,41 @@ def process_text(text):
20
  # Decode back to verify
21
  decoded = tokenizer.decode(encoded)
22
 
23
- return {
24
- "Encoded Tokens": str(encoded),
25
- "Number of Tokens": len(encoded),
26
- "Decoded Text": decoded,
27
- "Round-trip Success": text == decoded
28
- }
29
 
30
  # Create Gradio interface
31
- iface = gr.Interface(
32
- fn=process_text,
33
- inputs=gr.Textbox(label="Input Marathi Text", placeholder="नमस्कार, जग!"),
34
- outputs={
35
- "Encoded Tokens": gr.Textbox(label="Token IDs"),
36
- "Number of Tokens": gr.Number(label="Token Count"),
37
- "Decoded Text": gr.Textbox(label="Decoded Text"),
38
- "Round-trip Success": gr.Checkbox(label="Successful Round-trip")
39
- },
40
- title="Marathi BPE Tokenizer",
41
- description="Enter Marathi text to see how it's tokenized using byte-pair encoding.",
42
- examples=[
43
- ["नमस्कार, जग!"],
44
- ["ही एक चाचणी आहे."],
45
- ]
46
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
  # Launch the app
49
  if __name__ == "__main__":
50
- iface.launch()
 
20
  # Decode back to verify
21
  decoded = tokenizer.decode(encoded)
22
 
23
+ return str(encoded), len(encoded), decoded, text == decoded
 
 
 
 
 
24
 
25
  # Create Gradio interface
26
+ with gr.Blocks(title="Marathi BPE Tokenizer") as iface:
27
+ gr.Markdown("# Marathi BPE Tokenizer")
28
+ gr.Markdown("Enter Marathi text to see how it's tokenized using byte-pair encoding.")
29
+
30
+ with gr.Row():
31
+ input_text = gr.Textbox(label="Input Marathi Text", placeholder="नमस्कार, जग!")
32
+
33
+ with gr.Row():
34
+ process_btn = gr.Button("Tokenize")
35
+
36
+ with gr.Row():
37
+ token_ids = gr.Textbox(label="Token IDs")
38
+ token_count = gr.Number(label="Token Count")
39
+ decoded_text = gr.Textbox(label="Decoded Text")
40
+ roundtrip_success = gr.Checkbox(label="Successful Round-trip")
41
+
42
+ # Add example inputs
43
+ gr.Examples(
44
+ examples=[
45
+ ["नमस्कार, जग!"],
46
+ ["ही एक चाचणी आहे."],
47
+ ],
48
+ inputs=input_text
49
+ )
50
+
51
+ # Set up click event
52
+ process_btn.click(
53
+ fn=process_text,
54
+ inputs=input_text,
55
+ outputs=[token_ids, token_count, decoded_text, roundtrip_success]
56
+ )
57
 
58
  # Launch the app
59
  if __name__ == "__main__":
60
+ iface.launch()