add code for huggingface endpoints capability

handler.py

@@ -0,0 +1,63 @@
+from typing import Dict, List, Any
+from transformers import AutoConfig, AutoTokenizer
+from src.models import DNikudModel, ModelConfig
+from src.running_params import BATCH_SIZE, MAX_LENGTH_SEN
+from src.utiles_data import Nikud
+from src.models_utils import predict_single
+import torch
+import os
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        self.DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+        self.tokenizer = AutoTokenizer.from_pretrained("tau/tavbert-he")
+        dir_model_config = os.path.join("models", "config.yml")
+        self.config = ModelConfig.load_from_file(dir_model_config)
+        self.model = DNikudModel(
+            self.config,
+            len(Nikud.label_2_id["nikud"]),
+            len(Nikud.label_2_id["dagesh"]),
+            len(Nikud.label_2_id["sin"]),
+            device=self.DEVICE,
+        ).to(self.DEVICE)
+
+    def back_2_text(self, labels, text):
+        nikud = Nikud()
+        new_line = ""
+        for indx_char, c in enumerate(text):
+            new_line += (
+                c
+                + nikud.id_2_char(labels[0][1][1], "dagesh")
+                + nikud.id_2_char(labels[0][1][2], "sin")
+                + nikud.id_2_char(labels[0][1][0], "nikud")
+            )
+            print(indx_char, c)
+        print(labels)
+        return new_line
+
+    def predict_single_text(
+        self,
+        text,
+    ):
+        data = self.tokenizer(text, return_tensors="pt")
+        all_labels = predict_single(self.model, data, self.DEVICE)
+        return all_labels
+
+    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        data args:
+        """
+
+        # get inputs
+        inputs = data.pop("text", data)
+
+        # run normal prediction
+        prediction = self.predict_single_text(inputs)
+
+        # result = []
+        # for pred in prediction:
+        #     result.append(self.back_2_text(pred, inputs))
+        result = self.back_2_text(prediction, inputs)
+        return result

main.py

@@ -0,0 +1,596 @@
+# general
+import argparse
+import os
+import sys
+from datetime import datetime
+import logging
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+# ML
+import torch
+import torch.nn as nn
+from transformers import AutoConfig, AutoTokenizer
+
+# DL
+from src.models import DNikudModel, ModelConfig
+from src.models_utils import training, evaluate, predict
+from src.plot_helpers import (
+    generate_plot_by_nikud_dagesh_sin_dict,
+    generate_word_and_letter_accuracy_plot,
+)
+from src.running_params import BATCH_SIZE, MAX_LENGTH_SEN
+from src.utiles_data import (
+    NikudDataset,
+    Nikud,
+    create_missing_folders,
+    extract_text_to_compare_nakdimon,
+)
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+assert DEVICE == "cuda"
+
+
+def get_logger(
+    log_level, name_func, date_time=datetime.now().strftime("%d_%m_%y__%H_%M")
+):
+    log_location = os.path.join(
+        os.path.join(Path(__file__).parent, "logging"),
+        f"log_model_{name_func}_{date_time}",
+    )
+    create_missing_folders(log_location)
+
+    log_format = "%(asctime)s %(levelname)-8s Thread_%(thread)-6d ::: %(funcName)s(%(lineno)d) ::: %(message)s"
+    logger = logging.getLogger("algo")
+    logger.setLevel(getattr(logging, log_level))
+    cnsl_log_formatter = logging.Formatter(log_format)
+    cnsl_handler = logging.StreamHandler()
+    cnsl_handler.setFormatter(cnsl_log_formatter)
+    cnsl_handler.setLevel(log_level)
+    logger.addHandler(cnsl_handler)
+
+    create_missing_folders(log_location)
+
+    file_location = os.path.join(log_location, "Diacritization_Model_DEBUG.log")
+    file_log_formatter = logging.Formatter(log_format)
+
+    SINGLE_LOG_SIZE = 2 * 1024 * 1024  # in Bytes
+    MAX_LOG_FILES = 20
+    file_handler = RotatingFileHandler(
+        file_location, mode="a", maxBytes=SINGLE_LOG_SIZE, backupCount=MAX_LOG_FILES
+    )
+    file_handler.setFormatter(file_log_formatter)
+    file_handler.setLevel(log_level)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+def evaluate_text(
+    path,
+    dnikud_model,
+    tokenizer_tavbert,
+    logger,
+    plots_folder=None,
+    batch_size=BATCH_SIZE,
+):
+    path_name = os.path.basename(path)
+
+    msg = f"evaluate text: {path_name} on D-nikud Model"
+    logger.debug(msg)
+
+    if os.path.isfile(path):
+        dataset = NikudDataset(
+            tokenizer_tavbert, file=path, logger=logger, max_length=MAX_LENGTH_SEN
+        )
+    elif os.path.isdir(path):
+        dataset = NikudDataset(
+            tokenizer_tavbert, folder=path, logger=logger, max_length=MAX_LENGTH_SEN
+        )
+    else:
+        raise Exception("input path doesnt exist")
+
+    dataset.prepare_data(name="evaluate")
+    mtb_dl = torch.utils.data.DataLoader(dataset.prepered_data, batch_size=batch_size)
+
+    word_level_correct, letter_level_correct_dev = evaluate(
+        dnikud_model, mtb_dl, plots_folder, device=DEVICE
+    )
+
+    msg = (
+        f"Dnikud Model\n{path_name} evaluate\nLetter level accuracy:{letter_level_correct_dev}\n"
+        f"Word level accuracy: {word_level_correct}"
+    )
+    logger.debug(msg)
+
+
+def predict_text(
+    text_file,
+    tokenizer_tavbert,
+    output_file,
+    logger,
+    dnikud_model,
+    compare_nakdimon=False,
+):
+    dataset = NikudDataset(
+        tokenizer_tavbert, file=text_file, logger=logger, max_length=MAX_LENGTH_SEN
+    )
+
+    dataset.prepare_data(name="prediction")
+    mtb_prediction_dl = torch.utils.data.DataLoader(
+        dataset.prepered_data, batch_size=BATCH_SIZE
+    )
+    all_labels = predict(dnikud_model, mtb_prediction_dl, DEVICE)
+    text_data_with_labels = dataset.back_2_text(labels=all_labels)
+
+    if output_file is None:
+        for line in text_data_with_labels:
+            print(line)
+    else:
+        with open(output_file, "w", encoding="utf-8") as f:
+            if compare_nakdimon:
+                f.write(extract_text_to_compare_nakdimon(text_data_with_labels))
+            else:
+                f.write(text_data_with_labels)
+
+
+def predict_folder(
+    folder,
+    output_folder,
+    logger,
+    tokenizer_tavbert,
+    dnikud_model,
+    compare_nakdimon=False,
+):
+    create_missing_folders(output_folder)
+
+    for filename in os.listdir(folder):
+        file_path = os.path.join(folder, filename)
+
+        if filename.lower().endswith(".txt") and os.path.isfile(file_path):
+            output_file = os.path.join(output_folder, filename)
+            predict_text(
+                file_path,
+                output_file=output_file,
+                logger=logger,
+                tokenizer_tavbert=tokenizer_tavbert,
+                dnikud_model=dnikud_model,
+                compare_nakdimon=compare_nakdimon,
+            )
+        elif (
+            os.path.isdir(file_path) and filename != ".git" and filename != "README.md"
+        ):
+            sub_folder = file_path
+            sub_folder_output = os.path.join(output_folder, filename)
+            predict_folder(
+                sub_folder,
+                sub_folder_output,
+                logger,
+                tokenizer_tavbert,
+                dnikud_model,
+                compare_nakdimon=compare_nakdimon,
+            )
+
+
+def update_compare_folder(folder, output_folder):
+    create_missing_folders(output_folder)
+
+    for filename in os.listdir(folder):
+        file_path = os.path.join(folder, filename)
+
+        if filename.lower().endswith(".txt") and os.path.isfile(file_path):
+            output_file = os.path.join(output_folder, filename)
+            with open(file_path, "r", encoding="utf-8") as f:
+                text_data_with_labels = f.read()
+            with open(output_file, "w", encoding="utf-8") as f:
+                f.write(extract_text_to_compare_nakdimon(text_data_with_labels))
+        elif os.path.isdir(file_path) and filename != ".git":
+            sub_folder = file_path
+            sub_folder_output = os.path.join(output_folder, filename)
+            update_compare_folder(sub_folder, sub_folder_output)
+
+
+def check_files_excepted(folder):
+    for filename in os.listdir(folder):
+        file_path = os.path.join(folder, filename)
+
+        if filename.lower().endswith(".txt") and os.path.isfile(file_path):
+            try:
+                x = NikudDataset(None, file=file_path)
+            except:
+                print(f"failed in file: {filename}")
+        elif os.path.isdir(file_path) and filename != ".git":
+            check_files_excepted(file_path)
+
+
+def do_predict(
+    input_path, output_path, tokenizer_tavbert, logger, dnikud_model, compare_nakdimon
+):
+    if os.path.isdir(input_path):
+        predict_folder(
+            input_path,
+            output_path,
+            logger,
+            tokenizer_tavbert,
+            dnikud_model,
+            compare_nakdimon=compare_nakdimon,
+        )
+    elif os.path.isfile(input_path):
+        predict_text(
+            input_path,
+            output_file=output_path,
+            logger=logger,
+            tokenizer_tavbert=tokenizer_tavbert,
+            dnikud_model=dnikud_model,
+            compare_nakdimon=compare_nakdimon,
+        )
+    else:
+        raise Exception("Input file not exist")
+
+
+def evaluate_folder(folder_path, logger, dnikud_model, tokenizer_tavbert, plots_folder):
+    msg = f"evaluate sub folder: {folder_path}"
+    logger.info(msg)
+
+    evaluate_text(
+        folder_path,
+        dnikud_model=dnikud_model,
+        tokenizer_tavbert=tokenizer_tavbert,
+        logger=logger,
+        plots_folder=plots_folder,
+        batch_size=BATCH_SIZE,
+    )
+
+    msg = f"\n***************************************\n"
+    logger.info(msg)
+
+    for sub_folder_name in os.listdir(folder_path):
+        sub_folder_path = os.path.join(folder_path, sub_folder_name)
+
+        if (
+            not os.path.isdir(sub_folder_path)
+            or sub_folder_path == ".git"
+            or "not_use" in sub_folder_path
+            or "NakdanResults" in sub_folder_path
+        ):
+            continue
+
+        evaluate_folder(
+            sub_folder_path, logger, dnikud_model, tokenizer_tavbert, plots_folder
+        )
+
+
+def do_evaluate(
+    input_path,
+    logger,
+    dnikud_model,
+    tokenizer_tavbert,
+    plots_folder,
+    eval_sub_folders=False,
+):
+    msg = f"evaluate all_data: {input_path}"
+    logger.info(msg)
+
+    evaluate_text(
+        input_path,
+        dnikud_model=dnikud_model,
+        tokenizer_tavbert=tokenizer_tavbert,
+        logger=logger,
+        plots_folder=plots_folder,
+        batch_size=BATCH_SIZE,
+    )
+
+    msg = f"\n\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n"
+    logger.info(msg)
+
+    if eval_sub_folders:
+        for sub_folder_name in os.listdir(input_path):
+            sub_folder_path = os.path.join(input_path, sub_folder_name)
+
+            if (
+                not os.path.isdir(sub_folder_path)
+                or sub_folder_path == ".git"
+                or "not_use" in sub_folder_path
+                or "NakdanResults" in sub_folder_path
+            ):
+                continue
+
+            evaluate_folder(
+                sub_folder_path, logger, dnikud_model, tokenizer_tavbert, plots_folder
+            )
+
+
+def do_train(
+    logger,
+    plots_folder,
+    dir_model_config,
+    tokenizer_tavbert,
+    dnikud_model,
+    output_trained_model_dir,
+    data_folder,
+    n_epochs,
+    checkpoints_frequency,
+    learning_rate,
+    batch_size,
+):
+    msg = "Loading data..."
+    logger.debug(msg)
+
+    dataset_train = NikudDataset(
+        tokenizer_tavbert,
+        folder=os.path.join(data_folder, "train"),
+        logger=logger,
+        max_length=MAX_LENGTH_SEN,
+        is_train=True,
+    )
+    dataset_dev = NikudDataset(
+        tokenizer=tokenizer_tavbert,
+        folder=os.path.join(data_folder, "dev"),
+        logger=logger,
+        max_length=dataset_train.max_length,
+        is_train=True,
+    )
+    dataset_test = NikudDataset(
+        tokenizer=tokenizer_tavbert,
+        folder=os.path.join(data_folder, "test"),
+        logger=logger,
+        max_length=dataset_train.max_length,
+        is_train=True,
+    )
+
+    dataset_train.show_data_labels(plots_folder=plots_folder)
+
+    msg = f"Max length of data: {dataset_train.max_length}"
+    logger.debug(msg)
+
+    msg = (
+        f"Num rows in train data: {len(dataset_train.data)}, "
+        f"Num rows in dev data: {len(dataset_dev.data)}, "
+        f"Num rows in test data: {len(dataset_test.data)}"
+    )
+    logger.debug(msg)
+
+    msg = "Loading tokenizer and prepare data..."
+    logger.debug(msg)
+
+    dataset_train.prepare_data(name="train")
+    dataset_dev.prepare_data(name="dev")
+    dataset_test.prepare_data(name="test")
+
+    mtb_train_dl = torch.utils.data.DataLoader(
+        dataset_train.prepered_data, batch_size=batch_size
+    )
+    mtb_dev_dl = torch.utils.data.DataLoader(
+        dataset_dev.prepered_data, batch_size=batch_size
+    )
+
+    if not os.path.isfile(dir_model_config):
+        our_model_config = ModelConfig(dataset_train.max_length)
+        our_model_config.save_to_file(dir_model_config)
+
+    optimizer = torch.optim.Adam(dnikud_model.parameters(), lr=learning_rate)
+
+    msg = "training..."
+    logger.debug(msg)
+
+    criterion_nikud = nn.CrossEntropyLoss(ignore_index=Nikud.PAD_OR_IRRELEVANT).to(
+        DEVICE
+    )
+    criterion_dagesh = nn.CrossEntropyLoss(ignore_index=Nikud.PAD_OR_IRRELEVANT).to(
+        DEVICE
+    )
+    criterion_sin = nn.CrossEntropyLoss(ignore_index=Nikud.PAD_OR_IRRELEVANT).to(DEVICE)
+
+    training_params = {
+        "n_epochs": n_epochs,
+        "checkpoints_frequency": checkpoints_frequency,
+    }
+    (
+        best_model_details,
+        best_accuracy,
+        epochs_loss_train_values,
+        steps_loss_train_values,
+        loss_dev_values,
+        accuracy_dev_values,
+    ) = training(
+        dnikud_model,
+        mtb_train_dl,
+        mtb_dev_dl,
+        criterion_nikud,
+        criterion_dagesh,
+        criterion_sin,
+        training_params,
+        logger,
+        output_trained_model_dir,
+        optimizer,
+        device=DEVICE,
+    )
+
+    generate_plot_by_nikud_dagesh_sin_dict(
+        epochs_loss_train_values, "Train epochs loss", "Loss", plots_folder
+    )
+    generate_plot_by_nikud_dagesh_sin_dict(
+        steps_loss_train_values, "Train steps loss", "Loss", plots_folder
+    )
+    generate_plot_by_nikud_dagesh_sin_dict(
+        loss_dev_values, "Dev epochs loss", "Loss", plots_folder
+    )
+    generate_plot_by_nikud_dagesh_sin_dict(
+        accuracy_dev_values, "Dev accuracy", "Accuracy", plots_folder
+    )
+    generate_word_and_letter_accuracy_plot(
+        accuracy_dev_values, "Accuracy", plots_folder
+    )
+
+    msg = "Done"
+    logger.info(msg)
+
+
+if __name__ == "__main__":
+    tokenizer_tavbert = AutoTokenizer.from_pretrained("tau/tavbert-he")
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="""Predict D-nikud""",
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        dest="log_level",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        default="DEBUG",
+        help="Set the logging level",
+    )
+    parser.add_argument(
+        "-m",
+        "--output_model_dir",
+        type=str,
+        default="models",
+        help="save directory for model",
+    )
+    subparsers = parser.add_subparsers(
+        help="sub-command help", dest="command", required=True
+    )
+
+    parser_predict = subparsers.add_parser("predict", help="diacritize a text files ")
+    parser_predict.add_argument("input_path", help="input file or folder")
+    parser_predict.add_argument("output_path", help="output file")
+    parser_predict.add_argument(
+        "-ptmp",
+        "--pretrain_model_path",
+        type=str,
+        default=os.path.join(Path(__file__).parent, "models", "Dnikud_best_model.pth"),
+        help="pre-train model path - use only if you want to use trained model weights",
+    )
+    parser_predict.add_argument(
+        "-c",
+        "--compare",
+        dest="compare_nakdimon",
+        default=False,
+        help="predict text for comparing with Nakdimon",
+    )
+    parser_predict.set_defaults(func=do_predict)
+
+    parser_evaluate = subparsers.add_parser("evaluate", help="evaluate D-nikud")
+    parser_evaluate.add_argument("input_path", help="input file or folder")
+    parser_evaluate.add_argument(
+        "-ptmp",
+        "--pretrain_model_path",
+        type=str,
+        default=os.path.join(Path(__file__).parent, "models", "Dnikud_best_model.pth"),
+        help="pre-train model path - use only if you want to use trained model weights",
+    )
+    parser_evaluate.add_argument(
+        "-df",
+        "--plots_folder",
+        dest="plots_folder",
+        default=os.path.join(Path(__file__).parent, "plots"),
+        help="set the debug folder",
+    )
+    parser_evaluate.add_argument(
+        "-es",
+        "--eval_sub_folders",
+        dest="eval_sub_folders",
+        default=False,
+        help="accuracy calculation includes the evaluation of sub-folders "
+        "within the input_path folder, providing independent assessments "
+        "for each subfolder.",
+    )
+    parser_evaluate.set_defaults(func=do_evaluate)
+
+    # train --n_epochs 20
+
+    parser_train = subparsers.add_parser("train", help="train D-nikud")
+    parser_train.add_argument(
+        "-ptmp",
+        "--pretrain_model_path",
+        type=str,
+        default=None,
+        help="pre-train model path - use only if you want to use trained model weights",
+    )
+    parser_train.add_argument(
+        "--learning_rate", type=float, default=0.001, help="Learning rate"
+    )
+    parser_train.add_argument("--batch_size", type=int, default=32, help="batch_size")
+    parser_train.add_argument(
+        "--n_epochs", type=int, default=10, help="number of epochs"
+    )
+    parser_train.add_argument(
+        "--data_folder",
+        dest="data_folder",
+        default=os.path.join(Path(__file__).parent, "data"),
+        help="Set the debug folder",
+    )
+    parser_train.add_argument(
+        "--checkpoints_frequency",
+        type=int,
+        default=1,
+        help="checkpoints frequency for save the model",
+    )
+    parser_train.add_argument(
+        "-df",
+        "--plots_folder",
+        dest="plots_folder",
+        default=os.path.join(Path(__file__).parent, "plots"),
+        help="Set the debug folder",
+    )
+    parser_train.set_defaults(func=do_train)
+
+    args = parser.parse_args()
+    kwargs = vars(args).copy()
+    date_time = datetime.now().strftime("%d_%m_%y__%H_%M")
+    logger = get_logger(kwargs["log_level"], args.command, date_time)
+
+    del kwargs["log_level"]
+
+    kwargs["tokenizer_tavbert"] = tokenizer_tavbert
+    kwargs["logger"] = logger
+
+    msg = "Loading model..."
+    logger.debug(msg)
+
+    if args.command in ["evaluate", "predict"] or (
+        args.command == "train" and args.pretrain_model_path is not None
+    ):
+        dir_model_config = os.path.join("models", "config.yml")
+        config = ModelConfig.load_from_file(dir_model_config)
+
+        dnikud_model = DNikudModel(
+            config,
+            len(Nikud.label_2_id["nikud"]),
+            len(Nikud.label_2_id["dagesh"]),
+            len(Nikud.label_2_id["sin"]),
+            device=DEVICE,
+        ).to(DEVICE)
+        state_dict_model = dnikud_model.state_dict()
+        state_dict_model.update(torch.load(args.pretrain_model_path))
+        dnikud_model.load_state_dict(state_dict_model)
+    else:
+        base_model_name = "tau/tavbert-he"
+        config = AutoConfig.from_pretrained(base_model_name)
+        dnikud_model = DNikudModel(
+            config,
+            len(Nikud.label_2_id["nikud"]),
+            len(Nikud.label_2_id["dagesh"]),
+            len(Nikud.label_2_id["sin"]),
+            pretrain_model=base_model_name,
+            device=DEVICE,
+        ).to(DEVICE)
+
+    if args.command == "train":
+        output_trained_model_dir = os.path.join(
+            kwargs["output_model_dir"], "latest", f"output_models_{date_time}"
+        )
+        create_missing_folders(output_trained_model_dir)
+        dir_model_config = os.path.join(kwargs["output_model_dir"], "config.yml")
+        kwargs["dir_model_config"] = dir_model_config
+        kwargs["output_trained_model_dir"] = output_trained_model_dir
+    del kwargs["pretrain_model_path"]
+    del kwargs["output_model_dir"]
+    kwargs["dnikud_model"] = dnikud_model
+
+    del kwargs["command"]
+    del kwargs["func"]
+    args.func(**kwargs)
+
+    sys.exit(0)

models/config.yml

@@ -0,0 +1,78 @@
+_commit_hash: 41265b09a862144b2517afdfd46da4388f1380df
+_name_or_path: tau/tavbert-he
+add_cross_attention: false
+architectures:
+- RobertaForMaskedLM
+attention_probs_dropout_prob: 0.1
+bad_words_ids: null
+begin_suppress_tokens: null
+bos_token_id: 0
+chunk_size_feed_forward: 0
+classifier_dropout: null
+cross_attention_hidden_size: null
+decoder_start_token_id: null
+diversity_penalty: 0.0
+do_sample: false
+early_stopping: false
+encoder_no_repeat_ngram_size: 0
+eos_token_id: 2
+exponential_decay_length_penalty: null
+finetuning_task: null
+forced_bos_token_id: null
+forced_eos_token_id: null
+gradient_checkpointing: false
+hidden_act: gelu
+hidden_dropout_prob: 0.1
+hidden_size: 768
+id2label:
+  0: LABEL_0
+  1: LABEL_1
+initializer_range: 0.02
+intermediate_size: 3072
+is_decoder: false
+is_encoder_decoder: false
+label2id:
+  LABEL_0: 0
+  LABEL_1: 1
+layer_norm_eps: 1.0e-05
+length_penalty: 1.0
+max_length: 512
+max_position_embeddings: 2050
+min_length: 0
+model_type: roberta
+no_repeat_ngram_size: 0
+num_attention_heads: 12
+num_beam_groups: 1
+num_beams: 1
+num_hidden_layers: 12
+num_return_sequences: 1
+output_attentions: false
+output_hidden_states: false
+output_scores: false
+pad_token_id: 1
+position_embedding_type: absolute
+prefix: null
+problem_type: null
+pruned_heads: {}
+remove_invalid_values: false
+repetition_penalty: 1.0
+return_dict: true
+return_dict_in_generate: false
+sep_token_id: null
+suppress_tokens: null
+task_specific_params: null
+temperature: 1.0
+tf_legacy_loss: false
+tie_encoder_decoder: false
+tie_word_embeddings: true
+tokenizer_class: null
+top_k: 50
+top_p: 1.0
+torch_dtype: null
+torchscript: false
+transformers_version: 4.6.0.dev0
+type_vocab_size: 2
+typical_p: 1.0
+use_bfloat16: false
+use_cache: true
+vocab_size: 345

src/models.py

@@ -0,0 +1,74 @@
+# general
+import subprocess
+import yaml
+
+# ML
+import torch.nn as nn
+from transformers import AutoConfig, RobertaForMaskedLM, PretrainedConfig
+
+
+class DNikudModel(nn.Module):
+    def __init__(self, config, nikud_size, dagesh_size, sin_size, pretrain_model=None, device='cpu'):
+        super(DNikudModel, self).__init__()
+
+        if pretrain_model is not None:
+            model_base = RobertaForMaskedLM.from_pretrained(pretrain_model).to(device)
+        else:
+            model_base = RobertaForMaskedLM(config=config).to(device)
+
+        self.model = model_base.roberta
+        for name, param in self.model.named_parameters():
+            param.requires_grad = False
+
+        self.lstm1 = nn.LSTM(config.hidden_size, config.hidden_size, bidirectional=True, dropout=0.1, batch_first=True)
+        self.lstm2 = nn.LSTM(2 * config.hidden_size, config.hidden_size, bidirectional=True, dropout=0.1, batch_first=True)
+        self.dense = nn.Linear(2 * config.hidden_size, config.hidden_size)
+        self.out_n = nn.Linear(config.hidden_size, nikud_size)
+        self.out_d = nn.Linear(config.hidden_size, dagesh_size)
+        self.out_s = nn.Linear(config.hidden_size, sin_size)
+
+    def forward(self, input_ids, attention_mask):
+        last_hidden_state = self.model(input_ids, attention_mask=attention_mask).last_hidden_state
+        lstm1, _ = self.lstm1(last_hidden_state)
+        lstm2, _ = self.lstm2(lstm1)
+        dense = self.dense(lstm2)
+
+        nikud = self.out_n(dense)
+        dagesh = self.out_d(dense)
+        sin = self.out_s(dense)
+
+        return nikud, dagesh, sin
+
+
+def get_git_commit_hash():
+    try:
+        commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
+        return commit_hash
+    except subprocess.CalledProcessError:
+        # This will be raised if you're not in a Git repository
+        print("Not inside a Git repository!")
+        return None
+
+
+class ModelConfig(PretrainedConfig):
+    def __init__(self, max_length=None, dict=None):
+        super(ModelConfig, self).__init__()
+        if dict is None:
+            self.__dict__.update(AutoConfig.from_pretrained("tau/tavbert-he").__dict__)
+            self.max_length = max_length
+            self._commit_hash = get_git_commit_hash()
+        else:
+            self.__dict__.update(dict)
+
+    def print(self):
+        print(self.__dict__)
+
+    def save_to_file(self, file_path):
+        with open(file_path, "w") as yaml_file:
+            yaml.dump(self.__dict__, yaml_file, default_flow_style=False)
+
+    @classmethod
+    def load_from_file(cls, file_path):
+        with open(file_path, "r") as yaml_file:
+            config_dict = yaml.safe_load(yaml_file)
+        return cls(dict=config_dict)

src/models_utils.py

@@ -0,0 +1,559 @@
+# general
+import json
+import os
+
+# ML
+import numpy as np
+import pandas as pd
+import torch
+
+# visual
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics import confusion_matrix
+from tqdm import tqdm
+
+from src.running_params import DEBUG_MODE
+from src.utiles_data import Nikud, create_missing_folders
+
+CLASSES_LIST = ["nikud", "dagesh", "sin"]
+
+
+def calc_num_correct_words(input, letter_correct_mask):
+    SPACE_TOKEN = 104
+    START_SENTENCE_TOKEN = 1
+    END_SENTENCE_TOKEN = 2
+
+    correct_words_count = 0
+    words_count = 0
+    for index in range(input.shape[0]):
+        input[index][np.where(input[index] == SPACE_TOKEN)[0]] = 0
+        input[index][np.where(input[index] == START_SENTENCE_TOKEN)[0]] = 0
+        input[index][np.where(input[index] == END_SENTENCE_TOKEN)[0]] = 0
+        words_end_index = np.concatenate(
+            (np.array([-1]), np.where(input[index] == 0)[0])
+        )
+        is_correct_words_array = [
+            bool(
+                letter_correct_mask[index][
+                    list(range((words_end_index[s] + 1), words_end_index[s + 1]))
+                ].all()
+            )
+            for s in range(len(words_end_index) - 1)
+            if words_end_index[s + 1] - (words_end_index[s] + 1) > 1
+        ]
+        correct_words_count += np.array(is_correct_words_array).sum()
+        words_count += len(is_correct_words_array)
+
+    return correct_words_count, words_count
+
+
+def predict(model, data_loader, device="cpu"):
+    model.to(device)
+
+    all_labels = None
+    with torch.no_grad():
+        for index_data, data in enumerate(data_loader):
+            (inputs, attention_mask, labels_demo) = data
+            inputs = inputs.to(device)
+            attention_mask = attention_mask.to(device)
+            labels_demo = labels_demo.to(device)
+
+            mask_cant_be_nikud = np.array(labels_demo.cpu())[:, :, 0] == -1
+            mask_cant_be_dagesh = np.array(labels_demo.cpu())[:, :, 1] == -1
+            mask_cant_be_sin = np.array(labels_demo.cpu())[:, :, 2] == -1
+
+            nikud_probs, dagesh_probs, sin_probs = model(inputs, attention_mask)
+
+            pred_nikud = np.array(torch.max(nikud_probs, 2).indices.cpu()).reshape(
+                inputs.shape[0], inputs.shape[1], 1
+            )
+            pred_dagesh = np.array(torch.max(dagesh_probs, 2).indices.cpu()).reshape(
+                inputs.shape[0], inputs.shape[1], 1
+            )
+            pred_sin = np.array(torch.max(sin_probs, 2).indices.cpu()).reshape(
+                inputs.shape[0], inputs.shape[1], 1
+            )
+
+            pred_nikud[mask_cant_be_nikud] = -1
+            pred_dagesh[mask_cant_be_dagesh] = -1
+            pred_sin[mask_cant_be_sin] = -1
+
+            pred_labels = np.concatenate((pred_nikud, pred_dagesh, pred_sin), axis=2)
+
+            if all_labels is None:
+                all_labels = pred_labels
+            else:
+                all_labels = np.concatenate((all_labels, pred_labels), axis=0)
+
+    return all_labels
+
+
+def predict_single(model, data, device="cpu"):
+    # model.to(device)
+
+    all_labels = None
+    with torch.no_grad():
+        inputs = data["input_ids"].to(device)
+        attention_mask = data["attention_mask"].to(device)
+
+        # mask_cant_be_nikud = np.array(labels_demo.cpu())[:, :, 0] == -1
+        # mask_cant_be_dagesh = np.array(labels_demo.cpu())[:, :, 1] == -1
+        # mask_cant_be_sin = np.array(labels_demo.cpu())[:, :, 2] == -1
+
+        nikud_probs, dagesh_probs, sin_probs = model(inputs, attention_mask)
+        print(nikud_probs, dagesh_probs, sin_probs)
+
+        pred_nikud = np.array(torch.max(nikud_probs, 2).indices.cpu()).reshape(
+            inputs.shape[0], inputs.shape[1], 1
+        )
+        pred_dagesh = np.array(torch.max(dagesh_probs, 2).indices.cpu()).reshape(
+            inputs.shape[0], inputs.shape[1], 1
+        )
+        pred_sin = np.array(torch.max(sin_probs, 2).indices.cpu()).reshape(
+            inputs.shape[0], inputs.shape[1], 1
+        )
+
+        # pred_nikud[mask_cant_be_nikud] = -1
+        # pred_dagesh[mask_cant_be_dagesh] = -1
+        # pred_sin[mask_cant_be_sin] = -1
+        # print(pred_nikud, pred_dagesh, pred_sin)
+        pred_labels = np.concatenate((pred_nikud, pred_dagesh, pred_sin), axis=2)
+        print(pred_labels)
+        if all_labels is None:
+            all_labels = pred_labels
+        else:
+            all_labels = np.concatenate((all_labels, pred_labels), axis=0)
+
+    return all_labels
+
+
+def training(
+    model,
+    train_loader,
+    dev_loader,
+    criterion_nikud,
+    criterion_dagesh,
+    criterion_sin,
+    training_params,
+    logger,
+    output_model_path,
+    optimizer,
+    device="cpu",
+):
+    max_length = None
+    best_accuracy = 0.0
+
+    logger.info(f"start training with training_params: {training_params}")
+    model = model.to(device)
+
+    criteria = {
+        "nikud": criterion_nikud.to(device),
+        "dagesh": criterion_dagesh.to(device),
+        "sin": criterion_sin.to(device),
+    }
+
+    output_checkpoints_path = os.path.join(output_model_path, "checkpoints")
+    create_missing_folders(output_checkpoints_path)
+
+    train_steps_loss_values = {"nikud": [], "dagesh": [], "sin": []}
+    train_epochs_loss_values = {"nikud": [], "dagesh": [], "sin": []}
+    dev_loss_values = {"nikud": [], "dagesh": [], "sin": []}
+    dev_accuracy_values = {
+        "nikud": [],
+        "dagesh": [],
+        "sin": [],
+        "all_nikud_letter": [],
+        "all_nikud_word": [],
+    }
+
+    for epoch in tqdm(range(training_params["n_epochs"]), desc="Training"):
+        model.train()
+        train_loss = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        relevant_count = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+
+        for index_data, data in enumerate(train_loader):
+            (inputs, attention_mask, labels) = data
+
+            if max_length is None:
+                max_length = labels.shape[1]
+
+            inputs = inputs.to(device)
+            attention_mask = attention_mask.to(device)
+            labels = labels.to(device)
+
+            optimizer.zero_grad()
+            nikud_probs, dagesh_probs, sin_probs = model(inputs, attention_mask)
+
+            for i, (probs, class_name) in enumerate(
+                zip([nikud_probs, dagesh_probs, sin_probs], CLASSES_LIST)
+            ):
+                reshaped_tensor = (
+                    torch.transpose(probs, 1, 2)
+                    .contiguous()
+                    .view(probs.shape[0], probs.shape[2], probs.shape[1])
+                )
+                loss = criteria[class_name](reshaped_tensor, labels[:, :, i]).to(device)
+
+                num_relevant = (labels[:, :, i] != -1).sum()
+                train_loss[class_name] += loss.item() * num_relevant
+                relevant_count[class_name] += num_relevant
+
+                loss.backward(retain_graph=True)
+
+            for i, class_name in enumerate(CLASSES_LIST):
+                train_steps_loss_values[class_name].append(
+                    float(train_loss[class_name] / relevant_count[class_name])
+                )
+
+            optimizer.step()
+            if (index_data + 1) % 100 == 0:
+                msg = f"epoch: {epoch} , index_data: {index_data + 1}\n"
+                for i, class_name in enumerate(CLASSES_LIST):
+                    msg += f"mean loss train {class_name}: {float(train_loss[class_name] / relevant_count[class_name])}, "
+
+                logger.debug(msg[:-2])
+
+        for i, class_name in enumerate(CLASSES_LIST):
+            train_epochs_loss_values[class_name].append(
+                float(train_loss[class_name] / relevant_count[class_name])
+            )
+
+        for class_name in train_loss.keys():
+            train_loss[class_name] /= relevant_count[class_name]
+
+        msg = f"Epoch {epoch + 1}/{training_params['n_epochs']}\n"
+        for i, class_name in enumerate(CLASSES_LIST):
+            msg += f"mean loss train {class_name}: {train_loss[class_name]}, "
+        logger.debug(msg[:-2])
+
+        model.eval()
+        dev_loss = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        dev_accuracy = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        relevant_count = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        correct_preds = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        un_masks = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        predictions = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+        labels_class = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+
+        all_nikud_types_correct_preds_letter = 0.0
+
+        letter_count = 0.0
+        correct_words_count = 0.0
+        word_count = 0.0
+        with torch.no_grad():
+            for index_data, data in enumerate(dev_loader):
+                (inputs, attention_mask, labels) = data
+                inputs = inputs.to(device)
+                attention_mask = attention_mask.to(device)
+                labels = labels.to(device)
+
+                nikud_probs, dagesh_probs, sin_probs = model(inputs, attention_mask)
+
+                for i, (probs, class_name) in enumerate(
+                    zip([nikud_probs, dagesh_probs, sin_probs], CLASSES_LIST)
+                ):
+                    reshaped_tensor = (
+                        torch.transpose(probs, 1, 2)
+                        .contiguous()
+                        .view(probs.shape[0], probs.shape[2], probs.shape[1])
+                    )
+                    loss = criteria[class_name](reshaped_tensor, labels[:, :, i]).to(
+                        device
+                    )
+                    un_masked = labels[:, :, i] != -1
+                    num_relevant = un_masked.sum()
+                    relevant_count[class_name] += num_relevant
+                    _, preds = torch.max(probs, 2)
+                    dev_loss[class_name] += loss.item() * num_relevant
+                    correct_preds[class_name] += torch.sum(
+                        preds[un_masked] == labels[:, :, i][un_masked]
+                    )
+                    un_masks[class_name] = un_masked
+                    predictions[class_name] = preds
+                    labels_class[class_name] = labels[:, :, i]
+
+                un_mask_all_or = torch.logical_or(
+                    torch.logical_or(un_masks["nikud"], un_masks["dagesh"]),
+                    un_masks["sin"],
+                )
+
+                correct = {
+                    class_name: (torch.ones(un_mask_all_or.shape) == 1).to(device)
+                    for class_name in CLASSES_LIST
+                }
+
+                for i, class_name in enumerate(CLASSES_LIST):
+                    correct[class_name][un_masks[class_name]] = (
+                        predictions[class_name][un_masks[class_name]]
+                        == labels_class[class_name][un_masks[class_name]]
+                    )
+
+                letter_correct_mask = torch.logical_and(
+                    torch.logical_and(correct["sin"], correct["dagesh"]),
+                    correct["nikud"],
+                )
+                all_nikud_types_correct_preds_letter += torch.sum(
+                    letter_correct_mask[un_mask_all_or]
+                )
+
+                letter_correct_mask[~un_mask_all_or] = True
+                correct_num, total_words_num = calc_num_correct_words(
+                    inputs.cpu(), letter_correct_mask
+                )
+
+                word_count += total_words_num
+                correct_words_count += correct_num
+                letter_count += un_mask_all_or.sum()
+
+        for class_name in CLASSES_LIST:
+            dev_loss[class_name] /= relevant_count[class_name]
+            dev_accuracy[class_name] = float(
+                correct_preds[class_name].double() / relevant_count[class_name]
+            )
+
+            dev_loss_values[class_name].append(float(dev_loss[class_name]))
+            dev_accuracy_values[class_name].append(float(dev_accuracy[class_name]))
+
+        dev_all_nikud_types_accuracy_letter = float(
+            all_nikud_types_correct_preds_letter / letter_count
+        )
+
+        dev_accuracy_values["all_nikud_letter"].append(
+            dev_all_nikud_types_accuracy_letter
+        )
+
+        word_all_nikud_accuracy = correct_words_count / word_count
+        dev_accuracy_values["all_nikud_word"].append(word_all_nikud_accuracy)
+
+        msg = (
+            f"Epoch {epoch + 1}/{training_params['n_epochs']}\n"
+            f'mean loss Dev nikud: {train_loss["nikud"]}, '
+            f'mean loss Dev dagesh: {train_loss["dagesh"]}, '
+            f'mean loss Dev sin: {train_loss["sin"]}, '
+            f"Dev all nikud types letter Accuracy: {dev_all_nikud_types_accuracy_letter}, "
+            f'Dev nikud letter Accuracy: {dev_accuracy["nikud"]}, '
+            f'Dev dagesh letter Accuracy: {dev_accuracy["dagesh"]}, '
+            f'Dev sin letter Accuracy: {dev_accuracy["sin"]}, '
+            f"Dev word Accuracy: {word_all_nikud_accuracy}"
+        )
+        logger.debug(msg)
+
+        save_progress_details(
+            dev_accuracy_values,
+            train_epochs_loss_values,
+            dev_loss_values,
+            train_steps_loss_values,
+        )
+
+        if dev_all_nikud_types_accuracy_letter > best_accuracy:
+            best_accuracy = dev_all_nikud_types_accuracy_letter
+            best_model = {
+                "epoch": epoch,
+                "model_state_dict": model.state_dict(),
+                "optimizer_state_dict": optimizer.state_dict(),
+                "loss": loss,
+            }
+
+        if epoch % training_params["checkpoints_frequency"] == 0:
+            save_checkpoint_path = os.path.join(
+                output_checkpoints_path, f"checkpoint_model_epoch_{epoch + 1}.pth"
+            )
+            checkpoint = {
+                "epoch": epoch,
+                "model_state_dict": model.state_dict(),
+                "optimizer_state_dict": optimizer.state_dict(),
+                "loss": loss,
+            }
+            torch.save(checkpoint["model_state_dict"], save_checkpoint_path)
+
+    save_model_path = os.path.join(output_model_path, "best_model.pth")
+    torch.save(best_model["model_state_dict"], save_model_path)
+    return (
+        best_model,
+        best_accuracy,
+        train_epochs_loss_values,
+        train_steps_loss_values,
+        dev_loss_values,
+        dev_accuracy_values,
+    )
+
+
+def save_progress_details(
+    accuracy_dev_values,
+    epochs_loss_train_values,
+    loss_dev_values,
+    steps_loss_train_values,
+):
+    epochs_data_path = "epochs_data"
+    create_missing_folders(epochs_data_path)
+
+    save_dict_as_json(
+        steps_loss_train_values, epochs_data_path, "steps_loss_train_values.json"
+    )
+    save_dict_as_json(
+        epochs_loss_train_values, epochs_data_path, "epochs_loss_train_values.json"
+    )
+    save_dict_as_json(loss_dev_values, epochs_data_path, "loss_dev_values.json")
+    save_dict_as_json(accuracy_dev_values, epochs_data_path, "accuracy_dev_values.json")
+
+
+def save_dict_as_json(dict, file_path, file_name):
+    json_data = json.dumps(dict, indent=4)
+    with open(os.path.join(file_path, file_name), "w") as json_file:
+        json_file.write(json_data)
+
+
+def evaluate(model, test_data, plots_folder=None, device="cpu"):
+    model.to(device)
+    model.eval()
+
+    true_labels = {"nikud": [], "dagesh": [], "sin": []}
+    predictions = {"nikud": 0, "dagesh": 0, "sin": 0}
+    predicted_labels_2_report = {"nikud": [], "dagesh": [], "sin": []}
+    not_masks = {"nikud": 0, "dagesh": 0, "sin": 0}
+    correct_preds = {"nikud": 0, "dagesh": 0, "sin": 0}
+    relevant_count = {"nikud": 0, "dagesh": 0, "sin": 0}
+    labels_class = {"nikud": 0.0, "dagesh": 0.0, "sin": 0.0}
+
+    all_nikud_types_letter_level_correct = 0.0
+    nikud_letter_level_correct = 0.0
+    dagesh_letter_level_correct = 0.0
+    sin_letter_level_correct = 0.0
+
+    letters_count = 0.0
+    words_count = 0.0
+    correct_words_count = 0.0
+    with torch.no_grad():
+        for index_data, data in enumerate(test_data):
+            if DEBUG_MODE and index_data > 100:
+                break
+
+            (inputs, attention_mask, labels) = data
+
+            inputs = inputs.to(device)
+            attention_mask = attention_mask.to(device)
+            labels = labels.to(device)
+
+            nikud_probs, dagesh_probs, sin_probs = model(inputs, attention_mask)
+
+            for i, (probs, class_name) in enumerate(
+                zip([nikud_probs, dagesh_probs, sin_probs], CLASSES_LIST)
+            ):
+                labels_class[class_name] = labels[:, :, i]
+                not_masked = labels_class[class_name] != -1
+                num_relevant = not_masked.sum()
+                relevant_count[class_name] += num_relevant
+                _, preds = torch.max(probs, 2)
+                correct_preds[class_name] += torch.sum(
+                    preds[not_masked] == labels_class[class_name][not_masked]
+                )
+                predictions[class_name] = preds
+                not_masks[class_name] = not_masked
+
+                if len(true_labels[class_name]) == 0:
+                    true_labels[class_name] = (
+                        labels_class[class_name][not_masked].cpu().numpy()
+                    )
+                else:
+                    true_labels[class_name] = np.concatenate(
+                        (
+                            true_labels[class_name],
+                            labels_class[class_name][not_masked].cpu().numpy(),
+                        )
+                    )
+
+                if len(predicted_labels_2_report[class_name]) == 0:
+                    predicted_labels_2_report[class_name] = (
+                        preds[not_masked].cpu().numpy()
+                    )
+                else:
+                    predicted_labels_2_report[class_name] = np.concatenate(
+                        (
+                            predicted_labels_2_report[class_name],
+                            preds[not_masked].cpu().numpy(),
+                        )
+                    )
+
+            not_mask_all_or = torch.logical_or(
+                torch.logical_or(not_masks["nikud"], not_masks["dagesh"]),
+                not_masks["sin"],
+            )
+
+            correct_nikud = (torch.ones(not_mask_all_or.shape) == 1).to(device)
+            correct_dagesh = (torch.ones(not_mask_all_or.shape) == 1).to(device)
+            correct_sin = (torch.ones(not_mask_all_or.shape) == 1).to(device)
+
+            correct_nikud[not_masks["nikud"]] = (
+                predictions["nikud"][not_masks["nikud"]]
+                == labels_class["nikud"][not_masks["nikud"]]
+            )
+            correct_dagesh[not_masks["dagesh"]] = (
+                predictions["dagesh"][not_masks["dagesh"]]
+                == labels_class["dagesh"][not_masks["dagesh"]]
+            )
+            correct_sin[not_masks["sin"]] = (
+                predictions["sin"][not_masks["sin"]]
+                == labels_class["sin"][not_masks["sin"]]
+            )
+
+            letter_correct_mask = torch.logical_and(
+                torch.logical_and(correct_sin, correct_dagesh), correct_nikud
+            )
+            all_nikud_types_letter_level_correct += torch.sum(
+                letter_correct_mask[not_mask_all_or]
+            )
+
+            letter_correct_mask[~not_mask_all_or] = True
+            total_correct_count, total_words_num = calc_num_correct_words(
+                inputs.cpu(), letter_correct_mask
+            )
+
+            words_count += total_words_num
+            correct_words_count += total_correct_count
+
+            letters_count += not_mask_all_or.sum()
+
+            nikud_letter_level_correct += torch.sum(correct_nikud[not_mask_all_or])
+            dagesh_letter_level_correct += torch.sum(correct_dagesh[not_mask_all_or])
+            sin_letter_level_correct += torch.sum(correct_sin[not_mask_all_or])
+
+    for i, name in enumerate(CLASSES_LIST):
+        index_labels = np.unique(true_labels[name])
+        cm = confusion_matrix(
+            true_labels[name], predicted_labels_2_report[name], labels=index_labels
+        )
+
+        vowel_label = [Nikud.id_2_label[name][l] for l in index_labels]
+        unique_vowels_names = [
+            Nikud.sign_2_name[int(vowel)] for vowel in vowel_label if vowel != "WITHOUT"
+        ]
+        if "WITHOUT" in vowel_label:
+            unique_vowels_names += ["WITHOUT"]
+        cm_df = pd.DataFrame(cm, index=unique_vowels_names, columns=unique_vowels_names)
+
+        # Display confusion matrix
+        plt.figure(figsize=(10, 8))
+        sns.heatmap(cm_df, annot=True, cmap="Blues", fmt="d")
+        plt.title("Confusion Matrix")
+        plt.xlabel("True Label")
+        plt.ylabel("Predicted Label")
+        if plots_folder is None:
+            plt.show()
+        else:
+            plt.savefig(os.path.join(plots_folder, f"Confusion_Matrix_{name}.jpg"))
+
+    all_nikud_types_letter_level_correct = (
+        all_nikud_types_letter_level_correct / letters_count
+    )
+    all_nikud_types_word_level_correct = correct_words_count / words_count
+    nikud_letter_level_correct = nikud_letter_level_correct / letters_count
+    dagesh_letter_level_correct = dagesh_letter_level_correct / letters_count
+    sin_letter_level_correct = sin_letter_level_correct / letters_count
+    print("\n")
+    print(f"nikud_letter_level_correct = {nikud_letter_level_correct}")
+    print(f"dagesh_letter_level_correct = {dagesh_letter_level_correct}")
+    print(f"sin_letter_level_correct = {sin_letter_level_correct}")
+    print(f"word_level_correct = {all_nikud_types_word_level_correct}")
+
+    return all_nikud_types_word_level_correct, all_nikud_types_letter_level_correct

src/plot_helpers.py

@@ -0,0 +1,58 @@
+# general
+import os
+
+# visual
+import matplotlib.pyplot as plt
+
+cols = ["precision", "recall", "f1-score", "support"]
+
+
+def generate_plot_by_nikud_dagesh_sin_dict(nikud_dagesh_sin_dict, title, y_axis, plot_folder=None):
+    # Create a figure and axis
+    plt.figure(figsize=(8, 6))
+    plt.title(title)
+
+    ax = plt.gca()
+    indexes = list(range(1, len(nikud_dagesh_sin_dict["nikud"]) + 1))
+
+    # Plot data series with different colors and labels
+    ax.plot(indexes, nikud_dagesh_sin_dict["nikud"], color='blue', label='Nikud')
+    ax.plot(indexes, nikud_dagesh_sin_dict["dagesh"], color='green', label='Dagesh')
+    ax.plot(indexes, nikud_dagesh_sin_dict["sin"], color='red', label='Sin')
+
+    # Add legend
+    ax.legend()
+
+    # Set labels and title
+    ax.set_xlabel('Epoch')
+    ax.set_ylabel(y_axis)
+
+    if plot_folder is None:
+        plt.show()
+    else:
+        plt.savefig(os.path.join(plot_folder, f'{title.replace(" ", "_")}_plot.jpg'))
+
+
+def generate_word_and_letter_accuracy_plot(word_and_letter_accuracy_dict, title, plot_folder=None):
+    # Create a figure and axis
+    plt.figure(figsize=(8, 6))
+    plt.title(title)
+
+    ax = plt.gca()
+    indexes = list(range(1, len(word_and_letter_accuracy_dict["all_nikud_letter"]) + 1))
+
+    # Plot data series with different colors and labels
+    ax.plot(indexes, word_and_letter_accuracy_dict["all_nikud_letter"], color='blue', label='Letter')
+    ax.plot(indexes, word_and_letter_accuracy_dict["all_nikud_word"], color='green', label='Word')
+
+    # Add legend
+    ax.legend()
+
+    # Set labels and title
+    ax.set_xlabel("Epoch")
+    ax.set_ylabel("Accuracy")
+
+    if plot_folder is None:
+        plt.show()
+    else:
+        plt.savefig(os.path.join(plot_folder, 'word_and_letter_accuracy_plot.jpg'))

src/running_params.py

@@ -0,0 +1,3 @@
+DEBUG_MODE = False
+BATCH_SIZE = 32
+MAX_LENGTH_SEN = 1024

src/utiles_data.py

@@ -0,0 +1,676 @@
+# general
+import os.path
+from datetime import datetime
+from pathlib import Path
+from typing import List, Tuple
+from uuid import uuid1
+import re
+import glob2
+
+# visual
+import matplotlib
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+
+# ML
+import numpy as np
+import torch
+from torch.utils.data import Dataset
+
+from src.running_params import DEBUG_MODE, MAX_LENGTH_SEN
+
+matplotlib.use("agg")
+unique_key = str(uuid1())
+
+
+class Nikud:
+    """
+    1456 HEBREW POINT SHEVA
+    1457 HEBREW POINT HATAF SEGOL
+    1458 HEBREW POINT HATAF PATAH
+    1459 HEBREW POINT HATAF QAMATS
+    1460 HEBREW POINT HIRIQ
+    1461 HEBREW POINT TSERE
+    1462 HEBREW POINT SEGOL
+    1463 HEBREW POINT PATAH
+    1464 HEBREW POINT QAMATS
+    1465 HEBREW POINT HOLAM
+    1466 HEBREW POINT HOLAM HASER FOR VAV     ***EXTENDED***
+    1467 HEBREW POINT QUBUTS
+    1468 HEBREW POINT DAGESH OR MAPIQ
+    1469 HEBREW POINT METEG                   ***EXTENDED***
+    1470 HEBREW PUNCTUATION MAQAF             ***EXTENDED***
+    1471 HEBREW POINT RAFE                    ***EXTENDED***
+    1472 HEBREW PUNCTUATION PASEQ             ***EXTENDED***
+    1473 HEBREW POINT SHIN DOT
+    1474 HEBREW POINT SIN DOT
+    """
+
+    nikud_dict = {
+        "SHVA": 1456,
+        "REDUCED_SEGOL": 1457,
+        "REDUCED_PATAKH": 1458,
+        "REDUCED_KAMATZ": 1459,
+        "HIRIK": 1460,
+        "TZEIRE": 1461,
+        "SEGOL": 1462,
+        "PATAKH": 1463,
+        "KAMATZ": 1464,
+        "KAMATZ_KATAN": 1479,
+        "HOLAM": 1465,
+        "HOLAM HASER VAV": 1466,
+        "KUBUTZ": 1467,
+        "DAGESH OR SHURUK": 1468,
+        "METEG": 1469,
+        "PUNCTUATION MAQAF": 1470,
+        "RAFE": 1471,
+        "PUNCTUATION PASEQ": 1472,
+        "SHIN_YEMANIT": 1473,
+        "SHIN_SMALIT": 1474,
+    }
+
+    skip_nikud = (
+        []
+    )  # [nikud_dict["KAMATZ_KATAN"], nikud_dict["HOLAM HASER VAV"], nikud_dict["METEG"], nikud_dict["PUNCTUATION MAQAF"], nikud_dict["PUNCTUATION PASEQ"]]
+    sign_2_name = {sign: name for name, sign in nikud_dict.items()}
+    sin = [nikud_dict["RAFE"], nikud_dict["SHIN_YEMANIT"], nikud_dict["SHIN_SMALIT"]]
+    dagesh = [
+        nikud_dict["RAFE"],
+        nikud_dict["DAGESH OR SHURUK"],
+    ]  # note that DAGESH and SHURUK are one and the same
+    nikud = []
+    for v in nikud_dict.values():
+        if v not in sin and v not in skip_nikud:
+            nikud.append(v)
+    all_nikud_ord = {v for v in nikud_dict.values()}
+    all_nikud_chr = {chr(v) for v in nikud_dict.values()}
+
+    label_2_id = {
+        "nikud": {label: i for i, label in enumerate(nikud + ["WITHOUT"])},
+        "dagesh": {label: i for i, label in enumerate(dagesh + ["WITHOUT"])},
+        "sin": {label: i for i, label in enumerate(sin + ["WITHOUT"])},
+    }
+    id_2_label = {
+        "nikud": {i: label for i, label in enumerate(nikud + ["WITHOUT"])},
+        "dagesh": {i: label for i, label in enumerate(dagesh + ["WITHOUT"])},
+        "sin": {i: label for i, label in enumerate(sin + ["WITHOUT"])},
+    }
+
+    DAGESH_LETTER = nikud_dict["DAGESH OR SHURUK"]
+    RAFE = nikud_dict["RAFE"]
+    PAD_OR_IRRELEVANT = -1
+
+    LEN_NIKUD = len(label_2_id["nikud"])
+    LEN_DAGESH = len(label_2_id["dagesh"])
+    LEN_SIN = len(label_2_id["sin"])
+
+    def id_2_char(self, c, class_type):
+        if c == -1:
+            return ""
+
+        label = self.id_2_label[class_type][c]
+
+        if label != "WITHOUT":
+            print("Label =", chr(self.id_2_label[class_type][c]))
+            return chr(self.id_2_label[class_type][c])
+        return ""
+
+
+class Letters:
+    hebrew = [chr(c) for c in range(0x05D0, 0x05EA + 1)]
+    VALID_LETTERS = [
+        " ",
+        "!",
+        '"',
+        "'",
+        "(",
+        ")",
+        ",",
+        "-",
+        ".",
+        ":",
+        ";",
+        "?",
+    ] + hebrew
+    SPECIAL_TOKENS = ["H", "O", "5", "1"]
+    ENDINGS_TO_REGULAR = dict(zip("ךםןףץ", "כמנפצ"))
+    vocab = VALID_LETTERS + SPECIAL_TOKENS
+    vocab_size = len(vocab)
+
+
+class Letter:
+    def __init__(self, letter):
+        self.letter = letter
+        self.normalized = None
+        self.dagesh = None
+        self.sin = None
+        self.nikud = None
+
+    def normalize(self, letter):
+        if letter in Letters.VALID_LETTERS:
+            return letter
+        if letter in Letters.ENDINGS_TO_REGULAR:
+            return Letters.ENDINGS_TO_REGULAR[letter]
+        if letter in ["\n", "\t"]:
+            return " "
+        if letter in ["‒", "–", "—", "―", "−", "+"]:
+            return "-"
+        if letter == "[":
+            return "("
+        if letter == "]":
+            return ")"
+        if letter in ["´", "‘", "’"]:
+            return "'"
+        if letter in ["“", "”", "״"]:
+            return '"'
+        if letter.isdigit():
+            if int(letter) == 1:
+                return "1"
+            else:
+                return "5"
+        if letter == "…":
+            return ","
+        if letter in ["ײ", "װ", "ױ"]:
+            return "H"
+        return "O"
+
+    def can_dagesh(self, letter):
+        return letter in ("בגדהוזטיכלמנספצקשת" + "ךף")
+
+    def can_sin(self, letter):
+        return letter == "ש"
+
+    def can_nikud(self, letter):
+        return letter in ("אבגדהוזחטיכלמנסעפצקרשת" + "ךן")
+
+    def get_label_letter(self, labels):
+        dagesh_sin_nikud = [
+            True if self.can_dagesh(self.letter) else False,
+            True if self.can_sin(self.letter) else False,
+            True if self.can_nikud(self.letter) else False,
+        ]
+
+        labels_ids = {
+            "nikud": Nikud.PAD_OR_IRRELEVANT,
+            "dagesh": Nikud.PAD_OR_IRRELEVANT,
+            "sin": Nikud.PAD_OR_IRRELEVANT,
+        }
+
+        normalized = self.normalize(self.letter)
+
+        i = 0
+        if Nikud.nikud_dict["PUNCTUATION PASEQ"] in labels:
+            labels.remove(Nikud.nikud_dict["PUNCTUATION PASEQ"])
+        if Nikud.nikud_dict["PUNCTUATION MAQAF"] in labels:
+            labels.remove(Nikud.nikud_dict["PUNCTUATION MAQAF"])
+        if Nikud.nikud_dict["HOLAM HASER VAV"] in labels:
+            labels.remove(Nikud.nikud_dict["HOLAM HASER VAV"])
+        if Nikud.nikud_dict["METEG"] in labels:
+            labels.remove(Nikud.nikud_dict["METEG"])
+        if Nikud.nikud_dict["KAMATZ_KATAN"] in labels:
+            labels[labels.index(Nikud.nikud_dict["KAMATZ_KATAN"])] = Nikud.nikud_dict[
+                "KAMATZ"
+            ]
+        for index, (class_name, group) in enumerate(
+            zip(
+                ["dagesh", "sin", "nikud"],
+                [[Nikud.DAGESH_LETTER], Nikud.sin, Nikud.nikud],
+            )
+        ):
+            # notice - order is important: dagesh then sin and then nikud
+            if dagesh_sin_nikud[index]:
+                if i < len(labels) and labels[i] in group:
+                    labels_ids[class_name] = Nikud.label_2_id[class_name][labels[i]]
+                    i += 1
+                else:
+                    labels_ids[class_name] = Nikud.label_2_id[class_name]["WITHOUT"]
+
+        if (
+            np.array(dagesh_sin_nikud).all()
+            and len(labels) == 3
+            and labels[0] in Nikud.sin
+        ):
+            labels_ids["nikud"] = Nikud.label_2_id["nikud"][labels[2]]
+            labels_ids["dagesh"] = Nikud.label_2_id["dagesh"][labels[1]]
+
+        if (
+            self.can_sin(self.letter)
+            and len(labels) == 2
+            and labels[1] == Nikud.DAGESH_LETTER
+        ):
+            labels_ids["dagesh"] = Nikud.label_2_id["dagesh"][labels[1]]
+            labels_ids["nikud"] = Nikud.label_2_id[class_name]["WITHOUT"]
+
+        if (
+            self.letter == "ו"
+            and labels_ids["dagesh"] == Nikud.DAGESH_LETTER
+            and labels_ids["nikud"] == Nikud.label_2_id["nikud"]["WITHOUT"]
+        ):
+            labels_ids["dagesh"] = Nikud.label_2_id["dagesh"]["WITHOUT"]
+            labels_ids["nikud"] = Nikud.DAGESH_LETTER
+
+        self.normalized = normalized
+        self.dagesh = labels_ids["dagesh"]
+        self.sin = labels_ids["sin"]
+        self.nikud = labels_ids["nikud"]
+
+    def name_of(self, letter):
+        if "א" <= letter <= "ת":
+            return letter
+        if letter == Nikud.DAGESH_LETTER:
+            return "דגש\שורוק"
+        if letter == Nikud.KAMATZ:
+            return "קמץ"
+        if letter == Nikud.PATAKH:
+            return "פתח"
+        if letter == Nikud.TZEIRE:
+            return "צירה"
+        if letter == Nikud.SEGOL:
+            return "סגול"
+        if letter == Nikud.SHVA:
+            return "שוא"
+        if letter == Nikud.HOLAM:
+            return "חולם"
+        if letter == Nikud.KUBUTZ:
+            return "קובוץ"
+        if letter == Nikud.HIRIK:
+            return "חיריק"
+        if letter == Nikud.REDUCED_KAMATZ:
+            return "חטף-קמץ"
+        if letter == Nikud.REDUCED_PATAKH:
+            return "חטף-פתח"
+        if letter == Nikud.REDUCED_SEGOL:
+            return "חטף-סגול"
+        if letter == Nikud.SHIN_SMALIT:
+            return "שין-שמאלית"
+        if letter == Nikud.SHIN_YEMANIT:
+            return "שין-ימנית"
+        if letter.isprintable():
+            return letter
+        return "לא ידוע ({})".format(hex(ord(letter)))
+
+
+def text_contains_nikud(text):
+    return len(set(text) & Nikud.all_nikud_chr) > 0
+
+
+def combine_sentences(list_sentences, max_length=0, is_train=False):
+    all_new_sentences = []
+    new_sen = ""
+    index = 0
+    while index < len(list_sentences):
+        sen = list_sentences[index]
+
+        if not text_contains_nikud(sen) and (
+            "------------------" in sen or sen == "\n"
+        ):
+            if len(new_sen) > 0:
+                all_new_sentences.append(new_sen)
+                if not is_train:
+                    all_new_sentences.append(sen)
+                new_sen = ""
+                index += 1
+                continue
+
+        if not text_contains_nikud(sen) and is_train:
+            index += 1
+            continue
+
+        if len(sen) > max_length:
+            update_sen = sen.replace(". ", f". {unique_key}")
+            update_sen = update_sen.replace("? ", f"? {unique_key}")
+            update_sen = update_sen.replace("! ", f"! {unique_key}")
+            update_sen = update_sen.replace("” ", f"” {unique_key}")
+            update_sen = update_sen.replace("\t", f"\t{unique_key}")
+            part_sentence = update_sen.split(unique_key)
+
+            good_parts = []
+            for p in part_sentence:
+                if len(p) < max_length:
+                    good_parts.append(p)
+                else:
+                    prev = 0
+                    while prev <= len(p):
+                        part = p[prev : (prev + max_length)]
+                        last_space = 0
+                        if " " in part:
+                            last_space = part[::-1].index(" ") + 1
+                        next = prev + max_length - last_space
+                        part = p[prev:next]
+                        good_parts.append(part)
+                        prev = next
+            list_sentences = (
+                list_sentences[:index] + good_parts + list_sentences[index + 1 :]
+            )
+            continue
+        if new_sen == "":
+            new_sen = sen
+        elif len(new_sen) + len(sen) < max_length:
+            new_sen += sen
+        else:
+            all_new_sentences.append(new_sen)
+            new_sen = sen
+
+        index += 1
+    if len(new_sen) > 0:
+        all_new_sentences.append(new_sen)
+    return all_new_sentences
+
+
+class NikudDataset(Dataset):
+    def __init__(
+        self,
+        tokenizer,
+        folder=None,
+        file=None,
+        logger=None,
+        max_length=0,
+        is_train=False,
+    ):
+        self.max_length = max_length
+        self.tokenizer = tokenizer
+        self.is_train = is_train
+        if folder is not None:
+            self.data, self.origin_data = self.read_data_folder(folder, logger)
+        elif file is not None:
+            self.data, self.origin_data = self.read_data(file, logger)
+        self.prepered_data = None
+
+    def read_data_folder(self, folder_path: str, logger=None):
+        all_files = glob2.glob(f"{folder_path}/**/*.txt", recursive=True)
+        msg = f"number of files: " + str(len(all_files))
+        if logger:
+            logger.debug(msg)
+        else:
+            print(msg)
+        all_data = []
+        all_origin_data = []
+        if DEBUG_MODE:
+            all_files = all_files[0:2]
+        for file in all_files:
+            if "not_use" in file or "NakdanResults" in file:
+                continue
+            data, origin_data = self.read_data(file, logger)
+            all_data.extend(data)
+            all_origin_data.extend(origin_data)
+        return all_data, all_origin_data
+
+    def read_data(self, filepath: str, logger=None) -> List[Tuple[str, list]]:
+        msg = f"read file: {filepath}"
+        if logger:
+            logger.debug(msg)
+        else:
+            print(msg)
+        data = []
+        orig_data = []
+        with open(filepath, "r", encoding="utf-8") as file:
+            file_data = file.read()
+        data_list = self.split_text(file_data)
+
+        for sen in tqdm(data_list, desc=f"Source: {os.path.basename(filepath)}"):
+            if sen == "":
+                continue
+
+            labels = []
+            text = ""
+            text_org = ""
+            index = 0
+            sentence_length = len(sen)
+            while index < sentence_length:
+                if (
+                    ord(sen[index]) == Nikud.nikud_dict["PUNCTUATION MAQAF"]
+                    or ord(sen[index]) == Nikud.nikud_dict["PUNCTUATION PASEQ"]
+                    or ord(sen[index]) == Nikud.nikud_dict["METEG"]
+                ):
+                    index += 1
+                    continue
+
+                label = []
+                l = Letter(sen[index])
+                if not (l.letter not in Nikud.all_nikud_chr):
+                    if sen[index - 1] == "\n":
+                        index += 1
+                        continue
+                assert l.letter not in Nikud.all_nikud_chr
+                if sen[index] in Letters.hebrew:
+                    index += 1
+                    while (
+                        index < sentence_length
+                        and ord(sen[index]) in Nikud.all_nikud_ord
+                    ):
+                        label.append(ord(sen[index]))
+                        index += 1
+                else:
+                    index += 1
+
+                l.get_label_letter(label)
+                text += l.normalized
+                text_org += l.letter
+                labels.append(l)
+
+            data.append((text, labels))
+            orig_data.append(text_org)
+
+        return data, orig_data
+
+    def split_text(self, file_data):
+        file_data = file_data.replace("\n", f"\n{unique_key}")
+        data_list = file_data.split(unique_key)
+        data_list = combine_sentences(
+            data_list, is_train=self.is_train, max_length=MAX_LENGTH_SEN
+        )
+        return data_list
+
+    def show_data_labels(self, plots_folder=None):
+        nikud = [
+            Nikud.id_2_label["nikud"][label.nikud]
+            for _, label_list in self.data
+            for label in label_list
+            if label.nikud != -1
+        ]
+        dagesh = [
+            Nikud.id_2_label["dagesh"][label.dagesh]
+            for _, label_list in self.data
+            for label in label_list
+            if label.dagesh != -1
+        ]
+        sin = [
+            Nikud.id_2_label["sin"][label.sin]
+            for _, label_list in self.data
+            for label in label_list
+            if label.sin != -1
+        ]
+
+        vowels = nikud + dagesh + sin
+        unique_vowels, label_counts = np.unique(vowels, return_counts=True)
+        unique_vowels_names = [
+            Nikud.sign_2_name[int(vowel)]
+            for vowel in unique_vowels
+            if vowel != "WITHOUT"
+        ] + ["WITHOUT"]
+        fig, ax = plt.subplots(figsize=(16, 6))
+
+        bar_positions = np.arange(len(unique_vowels))
+        bar_width = 0.15
+        ax.bar(bar_positions, list(label_counts), bar_width)
+
+        ax.set_title("Distribution of Vowels in dataset")
+        ax.set_xlabel("Vowels")
+        ax.set_ylabel("Count")
+        ax.legend(loc="right", bbox_to_anchor=(1, 0.85))
+        ax.set_xticks(bar_positions)
+        ax.set_xticklabels(unique_vowels_names, rotation=30, ha="right", fontsize=8)
+
+        if plots_folder is None:
+            plt.show()
+        else:
+            plt.savefig(os.path.join(plots_folder, "show_data_labels.jpg"))
+
+    def calc_max_length(self, maximum=MAX_LENGTH_SEN):
+        if self.max_length > maximum:
+            self.max_length = maximum
+        return self.max_length
+
+    def prepare_data(self, name="train"):
+        dataset = []
+        for index, (sentence, label) in tqdm(
+            enumerate(self.data), desc=f"prepare data {name}"
+        ):
+            encoded_sequence = self.tokenizer.encode_plus(
+                sentence,
+                add_special_tokens=True,
+                max_length=self.max_length,
+                padding="max_length",
+                truncation=True,
+                return_attention_mask=True,
+                return_tensors="pt",
+            )
+            label_lists = [
+                [letter.nikud, letter.dagesh, letter.sin] for letter in label
+            ]
+            label = torch.tensor(
+                [
+                    [
+                        Nikud.PAD_OR_IRRELEVANT,
+                        Nikud.PAD_OR_IRRELEVANT,
+                        Nikud.PAD_OR_IRRELEVANT,
+                    ]
+                ]
+                + label_lists[: (self.max_length - 1)]
+                + [
+                    [
+                        Nikud.PAD_OR_IRRELEVANT,
+                        Nikud.PAD_OR_IRRELEVANT,
+                        Nikud.PAD_OR_IRRELEVANT,
+                    ]
+                    for i in range(self.max_length - len(label) - 1)
+                ]
+            )
+
+            dataset.append(
+                (
+                    encoded_sequence["input_ids"][0],
+                    encoded_sequence["attention_mask"][0],
+                    label,
+                )
+            )
+
+        self.prepered_data = dataset
+
+    def back_2_text(self, labels):
+        nikud = Nikud()
+        all_text = ""
+        for indx_sentance, (input_ids, _, label) in enumerate(self.prepered_data):
+            new_line = ""
+            for indx_char, c in enumerate(self.origin_data[indx_sentance]):
+                new_line += (
+                    c
+                    + nikud.id_2_char(labels[indx_sentance, indx_char + 1, 1], "dagesh")
+                    + nikud.id_2_char(labels[indx_sentance, indx_char + 1, 2], "sin")
+                    + nikud.id_2_char(labels[indx_sentance, indx_char + 1, 0], "nikud")
+                )
+            all_text += new_line
+        return all_text
+
+    def __len__(self):
+        return self.data.shape[0]
+
+    def __getitem__(self, idx):
+        row = self.data[idx]
+
+
+def get_sub_folders_paths(main_folder):
+    list_paths = []
+    for filename in os.listdir(main_folder):
+        path = os.path.join(main_folder, filename)
+        if os.path.isdir(path) and filename != ".git":
+            list_paths.append(path)
+            list_paths.extend(get_sub_folders_paths(path))
+    return list_paths
+
+
+def create_missing_folders(folder_path):
+    # Check if the folder doesn't exist and create it if needed
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+
+
+def info_folder(folder, num_files, num_hebrew_letters):
+    """
+    Recursively counts the number of files and the number of Hebrew letters in all subfolders of the given folder path.
+
+    Args:
+        folder (str): The path of the folder to be analyzed.
+        num_files (int): The running total of the number of files encountered so far.
+        num_hebrew_letters (int): The running total of the number of Hebrew letters encountered so far.
+
+    Returns:
+        Tuple[int, int]: A tuple containing the total number of files and the total number of Hebrew letters.
+    """
+    for filename in os.listdir(folder):
+        file_path = os.path.join(folder, filename)
+        if filename.lower().endswith(".txt") and os.path.isfile(file_path):
+            num_files += 1
+            dataset = NikudDataset(None, file=file_path)
+            for line in dataset.data:
+                for c in line[0]:
+                    if c in Letters.hebrew:
+                        num_hebrew_letters += 1
+
+        elif os.path.isdir(file_path) and filename != ".git":
+            sub_folder = file_path
+            n1, n2 = info_folder(sub_folder, num_files, num_hebrew_letters)
+            num_files += n1
+            num_hebrew_letters += n2
+    return num_files, num_hebrew_letters
+
+
+def extract_text_to_compare_nakdimon(text):
+    res = text.replace("|", "")
+    res = res.replace(
+        chr(Nikud.nikud_dict["KUBUTZ"]) + "ו" + chr(Nikud.nikud_dict["METEG"]),
+        "ו" + chr(Nikud.nikud_dict["DAGESH OR SHURUK"]),
+    )
+    res = res.replace(
+        chr(Nikud.nikud_dict["HOLAM"]) + "ו" + chr(Nikud.nikud_dict["METEG"]), "ו"
+    )
+    res = res.replace(
+        "ו" + chr(Nikud.nikud_dict["HOLAM"]) + chr(Nikud.nikud_dict["KAMATZ"]),
+        "ו" + chr(Nikud.nikud_dict["KAMATZ"]),
+    )
+    res = res.replace(chr(Nikud.nikud_dict["METEG"]), "")
+    res = res.replace(
+        chr(Nikud.nikud_dict["KAMATZ"]) + chr(Nikud.nikud_dict["HIRIK"]),
+        chr(Nikud.nikud_dict["KAMATZ"]) + "י" + chr(Nikud.nikud_dict["HIRIK"]),
+    )
+    res = res.replace(
+        chr(Nikud.nikud_dict["PATAKH"]) + chr(Nikud.nikud_dict["HIRIK"]),
+        chr(Nikud.nikud_dict["PATAKH"]) + "י" + chr(Nikud.nikud_dict["HIRIK"]),
+    )
+    res = res.replace(chr(Nikud.nikud_dict["PUNCTUATION MAQAF"]), "")
+    res = res.replace(chr(Nikud.nikud_dict["PUNCTUATION PASEQ"]), "")
+    res = res.replace(
+        chr(Nikud.nikud_dict["KAMATZ_KATAN"]), chr(Nikud.nikud_dict["KAMATZ"])
+    )
+
+    res = re.sub(chr(Nikud.nikud_dict["KUBUTZ"]) + "ו" + "(?=[א-ת])", "ו", res)
+    res = res.replace(chr(Nikud.nikud_dict["REDUCED_KAMATZ"]) + "ו", "ו")
+
+    res = res.replace(
+        chr(Nikud.nikud_dict["DAGESH OR SHURUK"]) * 2,
+        chr(Nikud.nikud_dict["DAGESH OR SHURUK"]),
+    )
+    res = res.replace("\u05be", "-")
+    res = res.replace("יְהוָֹה", "יהוה")
+
+    return res
+
+
+def orgenize_data(main_folder, logger):
+    x = NikudDataset(None)
+    x.delete_files(os.path.join(Path(main_folder).parent, "train"))
+    x.delete_files(os.path.join(Path(main_folder).parent, "dev"))
+    x.delete_files(os.path.join(Path(main_folder).parent, "test"))
+    x.split_data(
+        main_folder, main_folder_name=os.path.basename(main_folder), logger=logger
+    )