Update app.py

app.py

@@ -1,10 +1,15 @@
+# 1. Libraries
 import gradio as gr
 import torch
 from transformers import AutoModelForSeq2SeqLM, NllbTokenizer
 import pandas as pd
+from datasets import load_dataset
+
+# 2. Constants
 MODEL_PATH = 'TSjB/NLLB-201-600M-QM-V2'
+DATA_PATH = "TSjB/dictionary_krc_rus"
 
-# LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский", "English", "Türk dili"], "token": ["krc_Cyrl", "rus_Cyrl", "eng_Latn", "tur_Latn"]})
+# LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский язык", "English", "Türk dili"], "token": ["krc_Cyrl", "rus_Cyrl", "eng_Latn", "tur_Latn"]})
 LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский язык"], "token": ["krc_Cyrl", "rus_Cyrl"]})
 DIALECT = pd.DataFrame({"dialect": ["дж\ч", "ж\ч", "з\ц"], "short_name": ["qrc", "hlm", "mqr"]})
 
@@ -18,18 +23,34 @@ SYSTEM_LANG = "rus"
 # })
 NAMES = pd.DataFrame({
    "id": ["title", "from", "to", "your_sent", "transl_sent", "dialect", "translate", "annotation"],
-   "krc": ["# Къарачай-Малкъар кёчюрюўчю", "тилден", "тилге", "Мында джаз...", "Кёчюрюлгени", "Къарачай-Малкъарны диалекти", "Кёчюр","Къарачай-малкъар, орус тиллени арасында биринчи кёчюрюўчюдю. [Богдан Теўуналаны](https://t.me/bogdan_tewunalany), [Али Берберлени](https://t.me/ali_berberov) къурагъандыла\n\nСоинвестированиени эмда спонсорлукъ болушлукъну юсюнден [Али Берберовгъа](https://t.me/ali_berberov) соругъуз"],
-   "rus": ["# Карачаево-балкарский переводчик", "из", "на", "Напишите здесь...", "Переведённый текст", "Карачаево-балкарский диалект", "Перевести","Первый переводчик между карачаево-балкарским и русским языками. Разработчики: [Богдан Теунаев](https://t.me/bogdan_tewunalany), [Али Берберов](https://t.me/ali_berberov)\n\nПо вопросам соинвестирования и спонсорской поддержки обращайтесь к [Али Берберову](https://t.me/ali_berberov)"],
-   "tur": ["# Karaçay-Malkar tercümanı", "dilden", "dile", "Buraya yaz...", "Çevrilmiş metin burada", "Karaçay-Malkar lehçesi", "Tercüme edin", "Karaçay-Balkarca ve Rusça dilleri arasındaki ilk çevirmen. Geliştiriciler: [Bogdan Tewunalanı](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_berberov)\n\nOrtak yatırım ve sponsorluk ile ilgili sorularınız için [Ali Berberov](https://t.me/ali_berberov) ile iletişime geçin"],
-   "eng": ["# Qarachay-Malqar translator", "from", "to", "Write here...", "Translated text is here", "Qarachay-Malqar dialect", "Translate", "The first translator between Qarachay-Malqar and Russian languages. Developers: [Bogdan Tewunalany](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_berberov)\n\nFor co-investment and sponsorship, please contact [Ali Berberov] (https://t.me/ali_berberov)"]
+   "krc": ["# Къарачай-Малкъар сёзлюк бла кёчюрюўчю", "тилден", "тилге", "Мында джаз...", "Кёчюрюлгени", "Къарачай-Малкъарны диалекти", "Кёчюр","Къарачай-малкъар, орус тиллени арасында биринчи кёчюрюўчюдю. Сёзлюк да ичине салыннганды.\n\n[Богдан Теўуналаны](https://t.me/bogdan_tewunalany), [Али Берберлени](https://t.me/ali_berberov) къурагъандыла\n\nСоинвестированиени эмда спонсорлукъ болушлукъну юсюнден [Али Берберовгъа](https://t.me/ali_berberov) соругъуз"],
+   "rus": ["# Карачаево-балкарский словарь и переводчик", "из", "на", "Напишите здесь...", "Переведённый текст", "Карачаево-балкарский диалект", "Перевести","Первый переводчик между карачаево-балкарским и русским языками. Также встроен словарь для отдельных слов или коротких фраз.\n\nРазработчики: [Богдан Теунаев](https://t.me/bogdan_tewunalany), [Али Берберов](https://t.me/ali_berberov)\n\nПо вопросам соинвестирования и спонсорской поддержки обращайтесь к [Али Берберову](https://t.me/ali_berberov)"],
+   "tur": ["# Karaçayca-Balkarca sözlük ve çevirmen", "dilden", "dile", "Buraya yaz...", "Çevrilmiş metin burada", "Karaçay-Malkar lehçesi", "Tercüme edin", "Karaçay-Balkarca ve Rusça dilleri arasındaki ilk çevirmen. Tek tek kelimeler veya kısa ifadeler için bir sözlük de yerleşiktir.\n\nGeliştiriciler: [Bogdan Tewunalanı](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_berberov)\n\nOrtak yatırım ve sponsorluk ile ilgili sorularınız için [Ali Berberov](https://t.me/ali_berberov) ile iletişime geçin"],
+   "eng": ["# Qarachay-Malqar dictionary and translator", "from", "to", "Write here...", "Translated text is here", "Qarachay-Malqar dialect", "Translate", "The first translator between Qarachay-Malqar and Russian languages. A dictionary for individual words or short phrases is also built in.\n\nDevelopers: [Bogdan Tewunalany](https://t.me/bogdan_tewunalany), [Ali Berberov](https://t.me/ali_berberov)\n\nFor co-investment and sponsorship, please contact [Ali Berberov] (https://t.me/ali_berberov)"]
 })
 
 
-# 2. Models
+OUTPUT_ROW_BY_EVERY_DICTIONARY = 15
+
+FILEPATH_SOURCE_PREPARED = "1.Data/Dictionary"
+# dictionary = pd.read_csv("%s/dictionary.csv" % FILEPATH_SOURCE_PREPARED, sep = ";")
+
+# 3. Upload
+dictionary = load_dataset(DATA_PATH)
+dictionary = pd.DataFrame(dictionary['train'])
+
+dictionary["soz"] = dictionary.soz.str.upper()
+dictionary["soz_l"] = dictionary.soz.str.lower()
+dictionary["belgi_l"] = dictionary.belgi.str.lower()
+
+dictionary_qm = dictionary[dictionary.til == "krc"]
+dictionary_ru = dictionary[dictionary.til == "rus"]
+
+
 tokenizer = NllbTokenizer.from_pretrained(MODEL_PATH)
 model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)
 
-# 3. Fix tokenizer
+# 4. Fix tokenizer
 def fixTokenizer(tokenizer, new_lang='krc_Cyrl'):
     """
     Add a new language token to the tokenizer vocabulary
@@ -51,7 +72,7 @@ def fixTokenizer(tokenizer, new_lang='krc_Cyrl'):
 
 fixTokenizer(tokenizer)
 
-# 4. Change letters
+# 5. Change letters
 
 def fromModel(str, dialect = "qrc"):
   if dialect == "qrc":
@@ -329,7 +350,7 @@ def toModel(str):
     str = str.replace("НГ", "  N")
     return str
 
-# 4. Translate function
+# 6. Translate function
 def translatePy(text, src_lang='rus_Cyrl', tgt_lang='krc_Cyrl',
     a=32, b=3, max_input_length=1024, num_beams=3, **kwargs
 ):
@@ -350,31 +371,76 @@ def translatePy(text, src_lang='rus_Cyrl', tgt_lang='krc_Cyrl',
     return tokenizer.batch_decode(result, skip_special_tokens=True)[0]
 
 
-# 5. Translate
 def translateProcess(text, from_, to, dialect):
-
-    if dialect == "" or dialect is None:
-      dialect = "дж\ч"
-    if from_ == "" or from_ is None:
-      from_ = "Русский язык"
-    if to == "" or to is None:
-      to = "Къарачай-Малкъар тил"
-    
-    from_ = "".join(LANGUAGE[LANGUAGE.language == from_].token.to_list())
-    to = "".join(LANGUAGE[LANGUAGE.language == to].token.to_list())
-    dialect = "".join(DIALECT[DIALECT.dialect == dialect].short_name.to_list())
-    
+    # print(from_)
+    # print(to)
+    # print(dialect)
     if from_ == 'krc_Cyrl':
       text = toModel(text)
     
-    str = translatePy(text, src_lang = from_, tgt_lang = to)
+    str_ = translatePy(text, src_lang = from_, tgt_lang = to)
     
     if to == 'krc_Cyrl':
-      str = fromModel(str, dialect = dialect)
+      str_ = fromModel(str_, dialect = dialect)
     
-    return str
-
+    return str_
+  
+# 7. Dictionary function
+def dictionaryDisp(from_, text):
+  str_l = text.lower()
+  filter_ = r"\W+" + str_l + r"|^" + str_l
+  
+  df_from_to = pd.DataFrame() 
+  df_to_from = pd.DataFrame()
+  
+  if from_ == 'krc_Cyrl':
+    df_from_to = dictionary_qm.copy()
+    df_to_from = dictionary_ru.copy()
+  elif from_ == 'rus_Cyrl':
+    df_from_to = dictionary_ru.copy()
+    df_to_from = dictionary_qm.copy()
+    
+  sozluk_1 = df_from_to[df_from_to.soz_l.str.startswith(str_l)]
+  # Select rows based on the sequence and output
+  sozluk_1 = sozluk_1.iloc[:OUTPUT_ROW_BY_EVERY_DICTIONARY]
+  
+  sozluk_2 = df_from_to[df_from_to.belgi_l.str.contains(filter_, regex=True)]
+  sozluk_2 = sozluk_2.iloc[:OUTPUT_ROW_BY_EVERY_DICTIONARY]
+  
+  sozluk_3 = df_to_from[df_to_from.belgi_l.str.contains(filter_, regex=True)]
+  sozluk_3 = sozluk_3.iloc[:OUTPUT_ROW_BY_EVERY_DICTIONARY]
+  
+  # Concatenate the DataFrames and drop duplicates
+  sozluk = pd.concat([sozluk_1, sozluk_2, sozluk_3], ignore_index=True).drop_duplicates()[["soz", "belgi"]]
+  sozluk = [x.soz + " ----- " + x.belgi + "\n\n----------\n\n" for x in sozluk.itertuples()]
+  sozluk = "".join(sozluk)
+  
+  return sozluk
+  # len(sozluk)
+  
+  
+# 8. Output function 
+def out(text, from_, to, dialect):
+  if dialect == "" or dialect is None:
+    dialect = "дж\ч"
+  if from_ == "" or from_ is None:
+    from_ = "Русский язык"
+  if to == "" or to is None:
+    to = "Къарачай-Малкъар тил"
+    
+  from_ = "".join(LANGUAGE[LANGUAGE.language == from_].token.to_list())
+  to = "".join(LANGUAGE[LANGUAGE.language == to].token.to_list())
+  dialect = "".join(DIALECT[DIALECT.dialect == dialect].short_name.to_list())
+    
+  str_ = dictionaryDisp(from_, text)
+  
+  if(len(str_) == 0):
+    str_ = translateProcess(text, from_, to, dialect)
+    # str_ = "myaf"
+  
+  return(str_)
 
+# 9. Definition ui
 _title = "".join(NAMES[NAMES.id == "title"][SYSTEM_LANG].to_list())
 _from = "".join(NAMES[NAMES.id == "from"][SYSTEM_LANG].to_list())
 _to = "".join(NAMES[NAMES.id == "to"][SYSTEM_LANG].to_list())
@@ -388,28 +454,32 @@ with gr.Blocks() as demo:
     gr.Markdown(_title)
     with gr.Row():
       choice_input = gr.Dropdown(
-            choices = LANGUAGE.language.to_list(), label=_from, value = "Русский язык", filterable = False)
+            # choices = LANGUAGE.language.to_list(), label=_from, value = "Русский язык", filterable = False)
+            choices = LANGUAGE.language.to_list(), label=_from, value = "Русский язык")
           
       with gr.Column():
         with gr.Row():            
             choice_output = gr.Dropdown(
-              choices = LANGUAGE.language.to_list(), label=_to, value = "Къарачай-Малкъар тил", filterable = False)
+              choices = LANGUAGE.language.to_list(), label=_to, value = "Къарачай-Малкъар тил")
                            
             dialect = gr.Dropdown(
-              choices = DIALECT.dialect.to_list(), label=_dialect, value = "дж\ч", filterable = False)
+              choices = DIALECT.dialect.to_list(), label=_dialect, value = "дж\ч")
         
     with gr.Row():
       with gr.Column():
-        text_input = gr.Textbox(lines=15, placeholder=_your_sent, label = "")
+        text_input = gr.Textbox(lines=15, placeholder=_your_sent, label = "", show_copy_button=True)
         
       with gr.Column():
-        text_output = gr.Textbox(lines=15, placeholder=_transl_sent, label = "")
+        text_output = gr.Textbox(lines=15, placeholder=_transl_sent, label = "", autoscroll=False, show_copy_button=True)
         
     text_button = gr.Button(_translate, variant = 'primary')
     
-    text_button.click(translateProcess, inputs=[text_input, choice_input, choice_output, dialect], outputs=[text_output]) # text, from, to, dialect
+    text_button.click(out, inputs=[text_input, choice_input, choice_output, dialect], outputs=[text_output]) # text, from, to, dialect
     
     gr.Markdown(_annotation)
 
-# 6. Launch
-demo.launch()
+# 10. Launch
+demo.launch()
+
+
+