Added fa-tj model and language detection

app.py

@@ -1,23 +1,35 @@
 import torch
 import streamlit as st
 from model import init_model, predict
-from data import Tokenizer, load_config
+from data import Tokenizer, load_config, language_detect
 
+MODEL_PATH_TJ_FA = 'tj-fa.pt'
+MODEL_PATH_FA_TJ = 'fa-tj.pt'
 
-MODEL_PATH = 'tj-fa.pt'
+config_tj_fa = load_config(MODEL_PATH_TJ_FA)
+tokenizer_tj_fa = Tokenizer(config_tj_fa)
+model_tj_fa = init_model(MODEL_PATH_TJ_FA)
 
-config = load_config(MODEL_PATH)
-print('Config:', config)
-tokenizer = Tokenizer(config)
+config_fa_tj = load_config(MODEL_PATH_FA_TJ)
+tokenizer_fa_tj = Tokenizer(config_fa_tj)
+model_fa_tj = init_model(MODEL_PATH_FA_TJ)
 
-# Load the model
-model = init_model(MODEL_PATH)
-
-# Create a text area box where the user can enter their text
 user_input = st.text_area("Enter some text here", value="Он ҷо, ки висоли дӯстон аст,\nВ-оллоҳ, ки миёни хона саҳрост.")
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
+# Detect language
+detected_language = language_detect(user_input, tokenizer_tj_fa, tokenizer_fa_tj)
+
+if detected_language == 'tj':
+    model = model_tj_fa
+    tokenizer = tokenizer_tj_fa
+    st.text('Detected language: Tajik (TJ) -> Transliterating to Persian (FA)')
+else:
+    model = model_fa_tj
+    tokenizer = tokenizer_fa_tj
+    st.text('Detected language: Persian (FA) -> Transliterating to Tajik (TJ)')
+
 # Run the model on the user's text and store the output
 model_output = predict(model, tokenizer, user_input, device)
 

data.py

@@ -21,7 +21,6 @@ class Tokenizer:
         self.trg_pad_idx = self.trg_char_index['<PAD>']
         self.trg_unk_idx = self.trg_char_index['<UNK>']
         self.src_unk_idx = self.src_char_index['<UNK>']
-            
 
     def encode_src(self, text: str):
         src = [self.src_char_index.get(src_char, self.src_unk_idx) for src_char in text]
@@ -43,3 +42,17 @@ class Tokenizer:
         src_padded = pad_sequence(src, batch_first=True, padding_value=self.src_pad_idx)
         trg_padded = pad_sequence(trg, batch_first=True, padding_value=self.trg_pad_idx)
         return src_padded, trg_padded
+
+
+def language_detect(text, tokenizer_tj_fa: "Tokenizer", tokenizer_fa_tj: "Tokenizer"):
+    # Calculate the percentage of characters in text that are present in the source vocabulary of tokenizer_tj_fa
+    percentage_tj_fa = sum(char in tokenizer_tj_fa.src_vocab for char in text) / len(text)
+
+    # Calculate the percentage of characters in text that are present in the source vocabulary of tokenizer_fa_tj
+    percentage_fa_tj = sum(char in tokenizer_fa_tj.src_vocab for char in text) / len(text)
+
+    # Return the language code of the tokenizer with the higher percentage
+    if percentage_tj_fa > percentage_fa_tj:
+        return 'tj'
+    else:
+        return 'fa'

fa-tj.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcdbc01b0630e0a01e42f5a06e1002a7cf0089ee0f32d3c21d9b359f47846aa7
+size 22892367