translation/tools/indic_nlp_library/indicnlp/transliterate/script_unifier.py

# 
#  Copyright (c) 2013-present, Anoop Kunchukuttan
#  All rights reserved.
#  
#  This source code is licensed under the MIT license found in the
#  LICENSE file in the root directory of this source tree.
# 

#Program for normalization of text written in Unicode. This is mainly geared towards Indic scripts 
#
# @author Anoop Kunchukuttan 
#

import sys
from indicnlp.normalize import indic_normalize
from indicnlp.transliterate import unicode_transliterate
from indicnlp import loader

class AggressiveScriptUnifier():

    def __init__(self,common_lang='hi',nasals_mode='to_nasal_consonants'):
        self.common_lang=common_lang
        self.nasals_mode=nasals_mode
        self.do_normalize_chandras=True
        self.do_normalize_vowel_ending=True
        self.remove_nuktas=True
        self.normalizer_map={}
        self._init_normalizers()

    def _init_normalizers(self):
        normalizer_factory=indic_normalize.IndicNormalizerFactory()

        ## for languages with common parameters
        for lang in ['hi','mr','sa','kK','ne','sd','bn','gu','ta','te','kn']:
            self.normalizer_map[lang]=normalizer_factory.get_normalizer(lang, nasals_mode=self.nasals_mode, 
                    do_normalize_chandras=self.do_normalize_chandras, remove_nuktas=self.remove_nuktas,
                    do_normalize_vowel_ending=self.do_normalize_vowel_ending)

        ## for languages with language specific parameters
        self.normalizer_map['pa']=normalizer_factory.get_normalizer('pa', nasals_mode=self.nasals_mode, 
                    do_normalize_chandras=self.do_normalize_chandras, remove_nuktas=self.remove_nuktas,
                    do_normalize_vowel_ending=self.do_normalize_vowel_ending,
                    do_canonicalize_addak=True, do_canonicalize_tippi=True,
                    do_replace_vowel_bases=True)
        self.normalizer_map['or']=normalizer_factory.get_normalizer('or', nasals_mode=self.nasals_mode, 
                    do_normalize_chandras=self.do_normalize_chandras, remove_nuktas=self.remove_nuktas,
                    do_normalize_vowel_ending=self.do_normalize_vowel_ending,
                    do_remap_wa=True)
        self.normalizer_map['as']=normalizer_factory.get_normalizer('as', nasals_mode=self.nasals_mode,
                    do_normalize_chandras=self.do_normalize_chandras, remove_nuktas=self.remove_nuktas,
                    do_normalize_vowel_ending=self.do_normalize_vowel_ending,
                    do_remap_assamese_chars=True)
        self.normalizer_map['ml']=normalizer_factory.get_normalizer('ml', nasals_mode=self.nasals_mode,
                    do_normalize_chandras=self.do_normalize_chandras, remove_nuktas=self.remove_nuktas,
                    do_normalize_vowel_ending=self.do_normalize_vowel_ending,
                    do_canonicalize_chillus=True, do_correct_geminated_T=True)

    def transform(self,text,lang):
        text=self.normalizer_map[lang].normalize(text)
        text=unicode_transliterate.UnicodeIndicTransliterator.transliterate(text, lang, self.common_lang)
        return text

class BasicScriptUnifier():

    def __init__(self,common_lang='hi',nasals_mode='do_nothing'):
        self.common_lang=common_lang
        self.nasals_mode=nasals_mode
        self.normalizer_map={}
        self._init_normalizers()

    def _init_normalizers(self):
        normalizer_factory=indic_normalize.IndicNormalizerFactory()

        for lang in ['hi','mr','sa','kK','ne','sd','bn','gu','ta','te','kn','pa','or','as','ml']:
            self.normalizer_map[lang]=normalizer_factory.get_normalizer(lang, nasals_mode=self.nasals_mode)    

    def transform(self,text,lang):

        if lang in self.normalizer_map:
            text=self.normalizer_map[lang].normalize(text)

        text=unicode_transliterate.UnicodeIndicTransliterator.transliterate(text, lang, self.common_lang)
        return text

class NaiveScriptUnifier():

    def __init__(self,common_lang='hi'):
        self.common_lang=common_lang

    def transform(self,text,lang):

        text=unicode_transliterate.UnicodeIndicTransliterator.transliterate(text, lang, self.common_lang)
        return text

if __name__ == '__main__': 

    loader.load()

    if len(sys.argv)<=4:
        print("Usage: python script_unifier <command> <infile> <outfile> <language>")
        sys.exit(1)

    if sys.argv[1]=='aggressive':

        language=sys.argv[4]

        unifier=AggressiveScriptUnifier(nasals_mode='to_nasal_consonants')

        with open(sys.argv[2],'r',encoding='utf-8') as ifile:
            with open(sys.argv[3],'w',encoding='utf-8') as ofile:
                for i, line in enumerate(ifile.readlines()):

                    line=line.strip()
                    transliterated_line=unifier.transform(line,language)
                    ofile.write(transliterated_line+'\n')

    elif sys.argv[1]=='moderate':

        language=sys.argv[4]

        unifier=AggressiveScriptUnifier(nasals_mode='do_nothing')

        with open(sys.argv[2],'r',encoding='utf-8') as ifile:
            with open(sys.argv[3],'w',encoding='utf-8') as ofile:
                for i, line in enumerate(ifile.readlines()):

                    line=line.strip()
                    transliterated_line=unifier.transform(line,language)
                    ofile.write(transliterated_line+'\n')
                    
    elif sys.argv[1]=='basic':

        language=sys.argv[4]

        unifier=BasicScriptUnifier()

        with open(sys.argv[2],'r',encoding='utf-8') as ifile:
            with open(sys.argv[3],'w',encoding='utf-8') as ofile:
                for i, line in enumerate(ifile.readlines()):

                    line=line.strip()
                    transliterated_line=unifier.transform(line,language)
                    ofile.write(transliterated_line+'\n')

    elif sys.argv[1]=='naive':

        language=sys.argv[4]

        unifier=NaiveScriptUnifier()

        with open(sys.argv[2],'r',encoding='utf-8') as ifile:
            with open(sys.argv[3],'w',encoding='utf-8') as ofile:
                for i, line in enumerate(ifile.readlines()):

                    line=line.strip()
                    transliterated_line=unifier.transform(line,language)
                    ofile.write(transliterated_line+'\n')