Upload 2 files

run.py

@@ -0,0 +1,44 @@
+
+from fire import Fire
+import string
+import tensorflow as tf
+from transformers import  AutoTokenizer
+from hazm import *
+from transformers import pipeline
+from transformers import TextClassificationPipeline
+original_model = "HooshvareLab/bert-fa-base-uncased"
+model_path = 'models'
+def remove_punctuation(input_string):
+    translator = str.maketrans("", "", string.punctuation)
+
+    result = input_string.translate(translator)
+    return result
+def predict(file_path):     
+    normalizer = Normalizer() 
+    tokenizer = AutoTokenizer.from_pretrained(original_model)
+    # classifier = pipeline("text-classification", model="stevhliu/my_awesome_model")
+
+    with open(file_path, 'r') as file:
+        text = file.read()
+    
+        text = remove_punctuation(text)
+        text = normalizer.normalize(text)
+        
+        input_tokens = tokenizer.batch_encode_plus(
+            [text],
+            padding=True,
+            truncation=True,
+            return_tensors="tf",
+            max_length=128
+            )
+        input_ids = input_tokens["input_ids"]
+        attention_mask = input_tokens["attention_mask"]
+        new_model = tf.keras.models.load_model(model_path)
+        # pipe = TextClassificationPipeline(model=new_model, tokenizer=tokenizer, return_all_scores=True)
+
+        print({"input_ids": input_ids, "attention_mask": attention_mask})
+        predictions = new_model.predict([{"input_ids": input_ids, "attention_mask": attention_mask}])
+        print(predictions[0])
+        # print(pipe([text]))
+if __name__ == '__main__':
+    Fire(predict)

trainer.py

@@ -0,0 +1,15 @@
+from fire import Fire
+from src.classifier.classifier import get_model
+from src.dataset.dataset import prepare_dataset
+from src.utils import load_config_file
+
+def trainer(config_path):
+    config = load_config_file(config_path)
+    dataset = prepare_dataset(config)
+    model = get_model(config,dataset)
+    model.train()
+    model.save_model_results()
+
+if __name__ == '__main__':
+    Fire(trainer)
+