Update app.py

app.py

@@ -1,15 +1,24 @@
 import gradio as gr
 import torch
 from transformers import AutoModelForSeq2SeqLM, NllbTokenizer
-# model_ru_qm_path = 'TSjB/mbart-large-52-ru-qm-v1'
-# model_qm_ru_path = 'TSjB/mbart-large-52-qm-ru-v1'
+import pandas as pd
 MODEL_PATH = 'TSjB/NLLB-201-600M-QM-V1'
 
+# LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский", "English", "Türk dili"], "token": ["krc_Cyrl", "rus_Cyrl", "eng_Latn", "tur_Latn"]})
+LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский"], "token": ["krc_Cyrl", "rus_Cyrl"]})
+DIALECT = pd.DataFrame({"dialect": ["дж\ч", "ж\ч", "з\ц"], "short_name": ["qrc", "hlm", "mqr"]})
+
+SYSTEM_LANG = "rus"
+NAMES = pd.DataFrame({
+   "id": ["title", "from", "to", "your_sent", "transl_sent", "dialect", "translate", "annotation"],
+   "krc": ["# Къарачай-Малкъар кёчюрюўчю", "тилден", "тилге", "Мында джаз...", "Кёчюрюлгени", "Къарачай-Малкъарны диалекти", "Кёчюр","Къарачай-Малкъар тилде биринчи кёчюрюўчюдю. [Богдан Теўуналаны](https://t.me/bogdan_tewunalany), [Али Берберлени](https://t.me/ali_bulat1990) къурагъандыла\n\nМодель Орус бла Къарачай-Малкъар тилледе юйрене тургъаны себебли, Къарачай-Малкъар кёчюрюў башха тиллеге да осал болургъа боллукъду."],
+   "rus": ["# Карачаево-Балкарский переводчик", "из", "на", "Напишите здесь...", "Переведённый текст", "Карачаево-Балкарский диалект", "Перевести","Первый переводчик на карачаево-балкарский язык. Создан [Богданом Теунаевым](https://t.me/bogdan_tewunalany), [Али Берберовым](https://t.me/ali_bulat1990)\n\nТак как модель обучалась на парах Русский и Карачаево-Балкарский, то Карачаево-Балкарский перевод для остальных языков может быть хуже."],
+   "tur": ["# Karaçay-Malkar tercümanı", "dilden", "dile", "Buraya yaz...", "Çevrilmiş metin burada", "Karaçay-Malkar lehçesi", "Tercüme edin", "İlk çevirmen. [Bogdan Tewunalanı](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_bulat1990) tarafından oluşturuldu\n\nModel Rusça ve Karaçay-Malkar çiftleri halinde eğitildiğinden, diğer diller için Karaçay-Malkar çevirisi daha kötü olabilir."],
+   "eng": ["# Qarachay-Malqar translator", "from", "to", "Write here...", "Translated text is here", "Qarachay-Malqar dialect", "Translate", "The first translator. Created by [Bogdan Tewunalany](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_bulat1990)\n\nSince the model was trained in pairs of Russian and Qarachay-Malqar, the Qarachay-Malqar translation for other languages may be worse."]
+})
+
+
 # 2. Models
-#tokenizer_ru_qm = MBart50Tokenizer.from_pretrained(model_ru_qm_path)
-#tokenizer_qm_ru = MBart50Tokenizer.from_pretrained(model_qm_ru_path)
-#model_ru_qm = MBartForConditionalGeneration.from_pretrained(model_ru_qm_path)
-#model_qm_ru = MBartForConditionalGeneration.from_pretrained(model_qm_ru_path)
 tokenizer = NllbTokenizer.from_pretrained(MODEL_PATH)
 model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)
 
@@ -198,7 +207,6 @@ def fromModel(str, dialect = "qrc"):
       str = str.replace("къ|гъ", "х")
   return str
 
-  
 def toModel(str):
     str = str.replace("дж", "j") 
     str = str.replace("Дж", "J") 
@@ -312,41 +320,8 @@ def toModel(str):
     str = str.replace("Нг", "  N")
     str = str.replace("НГ", "  N")
     return str
-    
-
-
 
 # 4. Translate function
-
-
-
-#def translatePy(text, model, tokenizer, src='ru_RU', trg='qm_XX', max_length='auto', num_beams=3, repetition_penalty=5.0, train_mode=False, n_out=None, **kwargs):
-#    tokenizer.src_lang = src
-#    tokenizer.tgt_lang = trg
-#    encoded = tokenizer(text, return_tensors="pt", truncation=True, max_length=1024)
-#    if max_length == 'auto':
-#        max_length = int(32 + 1.5 * encoded.input_ids.shape[1])
-#    if train_mode:
-#        model.train()
-#    else:
-#        model.eval()
-#    generated_tokens = model.generate(
-#        **encoded.to(model.device),
-#        forced_bos_token_id=tokenizer.lang_code_to_id[trg], 
-#        max_length=max_length, 
-#        num_beams=num_beams,
-#        repetition_penalty=repetition_penalty,
-#        # early_stopping=True,
-#        num_return_sequences=n_out or 1,
-#        **kwargs
-#    )
-#    out = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
-#    if isinstance(text, str) and n_out is None:
-#        return out[0]
-#    return out
-
-
-
 def translatePy(text, src_lang='rus_Cyrl', tgt_lang='krc_Cyrl',
     a=32, b=3, max_input_length=1024, num_beams=3, **kwargs
 ):
@@ -366,27 +341,60 @@ def translatePy(text, src_lang='rus_Cyrl', tgt_lang='krc_Cyrl',
     )
     return tokenizer.batch_decode(result, skip_special_tokens=True)[0]
 
-# 5. Translate
-def transl(text, til, change_letters = True):
-    str = ''
-    if til == "Къарачай-Малкъар":
-        if change_letters == True:
-            str = translatePy(toModel(text), src_lang = 'krc_Cyrl', tgt_lang='rus_Cyrl')
-        else:
-            str = translatePy(text, src_lang = 'krc_Cyrl', tgt_lang='rus_Cyrl')
-    elif til == "Русский":
-        if change_letters == True:
-            str = translatePy(text, src_lang = 'rus_Cyrl', tgt_lang='krc_Cyrl')
-            str = fromModel(str)
-        else:
-            str = translatePy(text, src_lang = 'rus_Cyrl', tgt_lang='krc_Cyrl')
 
+# 5. Translate
+def translateProcess(text, from_, to, dialect):
+    from_ = "".join(LANGUAGE[LANGUAGE.language == from_].token.to_list())
+    to = "".join(LANGUAGE[LANGUAGE.language == to].token.to_list())
+    dialect = "".join(DIALECT[DIALECT.dialect == dialect].short_name.to_list())
+    
+    if from_ == 'krc_Cyrl':
+      text = toModel(text)
+    
+    str = translatePy(text, src_lang = from_, tgt_lang = to)
+    
+    if to1 == 'krc_Cyrl':
+      str = fromModel(str, dialect = dialect)
+    
     return str
 
-demo = gr.Interface(
-    fn=transl,
-    inputs=[gr.Textbox(lines=1, placeholder="Your sentence here...", label = "input"), gr.Radio(
-            ["Къарачай-Малкъар", "Русский"], label="Language", value = "Русский"), gr.Checkbox(label="Change letter", info="It's for inner using", value = True)],
-    outputs="text"
-)
+
+_title = "".join(NAMES[NAMES.id == "title"][SYSTEM_LANG].to_list())
+_from = "".join(NAMES[NAMES.id == "from"][SYSTEM_LANG].to_list())
+_to = "".join(NAMES[NAMES.id == "to"][SYSTEM_LANG].to_list())
+_your_sent = "".join(NAMES[NAMES.id == "your_sent"][SYSTEM_LANG].to_list())
+_transl_sent = "".join(NAMES[NAMES.id == "transl_sent"][SYSTEM_LANG].to_list())
+_dialect = "".join(NAMES[NAMES.id == "dialect"][SYSTEM_LANG].to_list())
+_translate = "".join(NAMES[NAMES.id == "translate"][SYSTEM_LANG].to_list())
+_annotation = "".join(NAMES[NAMES.id == "annotation"][SYSTEM_LANG].to_list())
+ 
+with gr.Blocks() as demo:
+    gr.Markdown(_title)
+    with gr.Row():
+      choice_input = gr.Dropdown(
+            choices = LANGUAGE.language.to_list(), label=_from, value = "Русский")
+          
+      with gr.Column():
+        with gr.Row():
+          with gr.Column():
+            choice_output = gr.Dropdown(
+              choices = LANGUAGE.language.to_list(), label=_to, value = "Къарачай-Малкъар тил")
+          with gr.Column():
+            dialect = gr.Dropdown(
+              choices = DIALECT.dialect.to_list(), label=_dialect, value = "дж\ч")
+        
+    with gr.Row():
+      with gr.Column():
+        text_input = gr.Textbox(lines=15, placeholder=_your_sent, label = "")
+        
+      with gr.Column():
+        text_output = gr.Textbox(lines=15, placeholder=_transl_sent, label = "")
+        
+    text_button = gr.Button(_translate, variant = 'primary')
+    
+    text_button.click(translateProcess, inputs=[text_input, choice_input, choice_output, dialect], outputs=[text_output]) # text, from, to, dialect
+    
+    gr.Markdown(_annotation)
+
+# 6. Launch
 demo.launch()