import re import itertools import string import utils class InformalWord: def __init__(self, lemma, prefixs=None, postfixs=None, pos=None, append_h=False): if prefixs is None: prefixs = [] if postfixs is None: postfixs = [] self.is_verb = False self.is_mapper = False self.semi_mapper = False self.append_h = append_h self.lemma = lemma self.prefixs = prefixs self.postfixs = postfixs self.pos = pos class Prefix: def __init__(self, word, level, formal=None, ignore_poses=None, poses=None, non_connecting_chars=None, connector='nim'): if non_connecting_chars is None: non_connecting_chars = [] self.word = word self.level = level self.ignore_poses = ignore_poses self.poses = poses self.connector = connector if formal is None: self.formal = word else: self.formal = formal self.non_connecting_chars = non_connecting_chars class Postfix: def __init__(self, word, level, formal=None, ignore_poses=None, non_connecting_chars=None, poses=None, connector='nim'): if non_connecting_chars is None: non_connecting_chars = [] self.word = word self.level = level self.ignore_poses = ignore_poses self.poses = poses self.connector = connector if formal is None: self.formal = word else: self.formal = formal self.non_connecting_chars = non_connecting_chars class OneShotTransformer: NIM_FASELE = chr(8204) # prefixs HAMUN = Prefix('همون', 1, 'همان',connector='fasele',non_connecting_chars=['ه']) HAMIN = Prefix('همین', 1,connector='fasele') HAR = Prefix('هر', 1,connector='fasele') UN = Prefix('اون', 1, 'آن',connector='fasele',non_connecting_chars=['ه']) IN = Prefix('این', 1,connector='fasele',non_connecting_chars=['ه']) HICH = Prefix('هیچ', 1,connector='nim',non_connecting_chars=['ه', 'ا', 'آ']) B = Prefix('ب', 1, 'به', ignore_poses=['VERB', 'CCONJ', 'SCONJ'],connector='fasele',non_connecting_chars=['ا', 'ه', 'آ']) Y = Prefix('ی', 1, 'یک', ignore_poses=['VERB', 'CCONJ', 'SCONJ'],connector='fasele',non_connecting_chars=['ا', 'آ']) BI = Prefix('بی', 1, ignore_poses=['VERB'],connector='nim',non_connecting_chars=['ا']) POR = Prefix('پر', 1, ignore_poses=['VERB'],connector='nim') pres = [[HAMIN, HAMUN, UN, IN, HAMIN, BI, B, Y, POR, HAR]] #postfixs Y1 = Postfix('ی', 0, ignore_poses=['VERB'], connector='none',non_connecting_chars=['ی', 'ا', 'و', 'آ', 'اً']) TAR = Postfix('تر', 1, connector='nim') TARIN = Postfix('ترین', 1, connector='nim') HAY = Postfix('های', 2, connector='nim') HA = Postfix('ها', 2, connector='nim') A = Postfix('ا', 2, 'ها', ignore_poses=['VERB'], connector='nim',non_connecting_chars=['ا', 'و', 'آ', 'اً']) A1 = Postfix('ای', 2, 'های', ignore_poses=['VERB'], connector='nim',non_connecting_chars=['ا', 'و', 'آ', 'اً']) YY = Postfix('یی', 3, 'یی', ignore_poses=['VERB'], connector='none') M = Postfix('م', 3, ignore_poses=['VERB'], connector='none') M_MAN = Postfix('م', 3, 'من', ignore_poses=['VERB'], connector='fasele') T = Postfix('ت', 3, connector='none') T1 = Postfix('ت', 3, 'تو', connector='fasele') # T2 = Postfix('ت', 3, 'خود', ignore_poses=['VERB'], connector='fasele') SH = Postfix('ش', 3, connector='none') # SH1 = Postfix('ش', 3, 'خود', connector='fasele') # SH2 = Postfix('ش', 3, 'آن', connector='fasele') # SH3 = Postfix('ش', 3, 'او', connector='fasele') MAN = Postfix('مان', 3, connector='nim') MAN1 = Postfix('مان', 3, 'ما', connector='fasele') # MAN2 = Postfix('مان', 3, 'خود', connector='fasele') MUN = Postfix('مون', 3, 'مان', connector='nim') # MUN1 = Postfix('مون', 3, 'خود', connector='fasele') MUN2 = Postfix('مون', 3, 'ما', connector='fasele') TAN = Postfix('تان', 3, connector='nim') # TAN1 = Postfix('تان', 3, 'خود', connector='fasele') TAN2 = Postfix('تان', 3, 'شما', connector='fasele') TUN = Postfix('تون', 3, 'تان', connector='nim') # TUN1 = Postfix('تون', 3, 'خود', connector='fasele') TUN2 = Postfix('تون', 3, 'شما', connector='fasele') SHAN = Postfix('شان', 3, connector='nim') # SHAN1 = Postfix('شان', 3, 'خود', connector='fasele') SHAN2 = Postfix('شان', 3, 'آنان', connector='fasele') SHUN = Postfix('شون', 3, 'شان', connector='nim') # SHUN1 = Postfix('شون', 3, 'خود', connector='fasele') SHUN2 = Postfix('شون', 3, 'آنان', connector='fasele') N = Postfix('ن', 4, 'هستند', ignore_poses=['VERB', 'CCONJ', 'SCONJ'], connector='fasele', non_connecting_chars=['ی']) SHAM = Postfix('شم', 4, 'بشوم',ignore_poses=['VERB'], connector='fasele') SHI= Postfix('شی', 4, 'بشوی',ignore_poses=['VERB'], connector='fasele') SHE= Postfix('شه', 4, 'شود',ignore_poses=['VERB'], connector='fasele') SHIN= Postfix('شین', 4, 'شوید',ignore_poses=['VERB'], connector='fasele') SHID= Postfix('شید', 4, 'شوید',ignore_poses=['VERB'], connector='fasele') SHAAN= Postfix('شن', 4, 'شوند',ignore_poses=['VERB'], connector='fasele') SHAND= Postfix('شند', 4, 'شوند',ignore_poses=['VERB'], connector='fasele') M2 = Postfix('م', 4, 'هم',ignore_poses=['VERB'], connector='fasele') V = Postfix('و', 4, 'را', connector='fasele', non_connecting_chars=['ا', 'ای', 'آ', 'اً']) V1 = Postfix('رو', 4, 'را', connector='fasele') H = Postfix('ه', 4, '', ignore_poses=['VERB', 'CCONJ', 'SCONJ'], connector='none') # H2 = Postfix('ه', 4) M1 = Postfix('م', 4, 'هستم',ignore_poses=['VERB'], connector='fasele') Y2 = Postfix('ی', 4, 'ی', ignore_poses=['VERB'], connector='none') H1 = Postfix('ه', 4, 'است', ignore_poses=['VERB'], connector='fasele', non_connecting_chars=['ا', 'آ', 'اً']) S = Postfix('س', 4, 'است', connector='fasele') ST = Postfix('ست', 4, 'است', connector='fasele') ED = Postfix('ید', 4, 'هستید', ignore_poses=['VERB'], connector='fasele') EN = Postfix('ین', 4, 'هستید', ignore_poses=['VERB'], connector='fasele', non_connecting_chars=['تر']) EM = Postfix('یم', 4, 'هستیم', ignore_poses=['VERB'], connector='fasele') ND = Postfix('ند', 4, 'هستند', ignore_poses=['VERB'], connector='fasele') # posts = [[Y1], [TAR, TARIN], [HA, HAY, A, A1], [M, T, SH, MAN, MUN, TAN, TUN, SHAN, SHUN], [N, S, ST, M1, M2, V, V1,Y2, H, H1, ED, EN, EM, ND, SHAM, SHI, SHID, SHE, SHAND, SHIN, SHAAN]] # posts = [[Y1], [TAR, TARIN], [HA, HAY, A, A1], [YY, M, M_MAN, T, T1, T2, SH, MAN, MAN1, MAN2,MUN,MUN1,MUN2, TAN,TAN1,TAN2, TUN,TUN1,TUN2, SHAN,SHAN1,SHAN2, SHUN, SHUN1, SHUN2], [N, S, ST, M1, M2, V, V1,Y2, H1, ED, EN, EM, ND, SHAM, SHI, SHID, SHE, SHAND, SHIN, SHAAN]] posts = [[Y1], [TAR, TARIN], [HA, HAY, A, A1], [YY, M, M_MAN, T, T1, SH, MAN, MAN1,MUN,MUN2, TAN,TAN2, TUN,TUN2, SHAN,SHAN2, SHUN, SHUN2], [N, S, ST, M1, M2, V, V1,Y2, H1, ED, EN, EM, ND, SHAM, SHI, SHID, SHE, SHAND, SHIN, SHAAN]] PossessiveـPronouns = [M,T,SH, MAN, MUN, TAN, TUN, SHAN, SHUN] cant_append_h_posts = [Y1, TAR, TARIN] As = [A, A1] def get_separator(self, w1, w2, append_h): connector_2_str = {'none': '', 'nim': OneShotTransformer.NIM_FASELE, 'fasele': ' '} not_connect_chars = ['ا', 'د', 'ذ', 'ر', 'ز', 'ژ', 'و'] # if w2 == OneShotTransformer.Y2: # return '' # if w2 in [OneShotTransformer.M, OneShotTransformer.T, OneShotTransformer.SH] and ( type(w1) == str and w1[-1] in ['ا', 'و']): # return 'ی' # if type(w1) != str and w1.level == 1: # return ' ' # not_connect_chars = ['ا', 'د', 'ذ', 'ر', 'ز', 'ژ', 'و'] # if w1 in [OneShotTransformer.Y, OneShotTransformer.B, OneShotTransformer.HAMIN, OneShotTransformer.IN, OneShotTransformer.HAMUN] or w2 in [OneShotTransformer.ED, OneShotTransformer.EN, OneShotTransformer.EM, OneShotTransformer.ND, OneShotTransformer.H1, OneShotTransformer.M1, OneShotTransformer.S, OneShotTransformer.ST, OneShotTransformer.V, OneShotTransformer.N, OneShotTransformer.M2]: # return ' ' # # if ((type(w1) == str and len(w1)> 0 and w1[-1] in ['ا', 'و']) or (type(w1) != str and w1.formal[-1] in [ 'ا', 'و']))and w2.level == 3 : # return 'ی' + '‌' # if (type(w1) == str and len(w1)> 0 and w1[-1] in not_connect_chars) or (type(w1) != str and w1.word[-1] in not_connect_chars): # return '' all_pres = [p for pres in OneShotTransformer.pres for p in pres] all_posts = [p for posts in OneShotTransformer.posts for p in posts] if type(w1) == str: last_ch = w1[-1] else: last_ch = w1.word[-1] separator = '' extra_sep = '' if type(w1) == str and append_h and w2 in [OneShotTransformer.M, OneShotTransformer.T, OneShotTransformer.SH]: extra_sep = OneShotTransformer.NIM_FASELE + 'ا' if w2 in [OneShotTransformer.M, OneShotTransformer.T, OneShotTransformer.SH, OneShotTransformer.MAN, OneShotTransformer.MUN, OneShotTransformer.TAN, OneShotTransformer.TUN, OneShotTransformer.SHAN, OneShotTransformer.SHUN] and ( last_ch in ['ا', 'و']) : extra_sep = 'ی' if w1 in all_pres: separator = connector_2_str[w1.connector] if w2 in all_posts: separator = connector_2_str[w2.connector] # replace nim_fasele with '' for non connected words if last_ch in not_connect_chars and separator == OneShotTransformer.NIM_FASELE: separator = '' return extra_sep + separator def lemma_to_formals(self, iword): out_iwords = [iword] if iword.lemma in self.mapper and self.iword2str(iword) != self.mapper[iword.lemma]: for map_words in self.mapper[iword.lemma]: new_iw = InformalWord(lemma=map_words,prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos, append_h=iword.append_h) if not iword.prefixs and not iword.postfixs: new_iw.is_mapper = True new_iw.semi_mapper = True else: new_iw.semi_mapper = True out_iwords.append(new_iw) formal_verbs = self.verb_to_formal_func(iword.lemma) if formal_verbs is not None: for f_v in formal_verbs: new_iw = InformalWord(lemma=f_v,prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos, append_h=iword.append_h) new_iw.is_verb = True out_iwords.append(new_iw) return out_iwords def should_ignore_by_postagg(self, iword): post_pres = [pre for pre in iword.prefixs] + [post for post in iword.postfixs] for p in post_pres: if (p.ignore_poses and iword.pos in p.ignore_poses) or (p.poses and iword.pos not in p.poses): return True return False def filtered_based_on_rules(self, iword): #YY ha_p = [OneShotTransformer.A, OneShotTransformer.HA] if iword.postfixs and OneShotTransformer.YY in iword.postfixs and not all(p in ha_p + [OneShotTransformer.YY] for p in iword.postfixs): return True #hasti! if (iword.postfixs and len(iword.postfixs) == 1 and OneShotTransformer.Y2 in iword.postfixs and iword.lemma and iword.lemma[-1] in ['و', 'ا']) or (iword.postfixs and len(iword.postfixs) == 2 and OneShotTransformer.Y2 in iword.postfixs and iword.postfixs[0] in [OneShotTransformer.A, OneShotTransformer.HA]): return True #non connecting chars if iword.prefixs: last_pre = iword.prefixs[-1] if last_pre.non_connecting_chars and iword.lemma and any(iword.lemma.startswith(ch) for ch in last_pre.non_connecting_chars): return True if iword.postfixs: first_post = iword.postfixs[0] if first_post.non_connecting_chars and iword.lemma and any(iword.lemma.endswith(ch) for ch in first_post.non_connecting_chars): return True #hidden H # goshnashe if not iword.semi_mapper and not iword.append_h and iword.lemma and iword.lemma[-1] == 'ه' and iword.postfixs and iword.lemma not in self.non_hidden_h_words: return True # h + h if iword.prefixs and iword.postfixs and len(iword.lemma) < 2: return True # خونهه - خونششونه if iword.append_h and (OneShotTransformer.H in iword.postfixs or (len(iword.postfixs) == 1 and OneShotTransformer.H1 in iword.postfixs) ): return True if iword.prefixs and (OneShotTransformer.B in iword.prefixs or OneShotTransformer.Y in iword.prefixs) and (iword.lemma and iword.lemma[0] in ['ا', 'ی', 'و']): return True if iword.lemma in self.isolated_words and (iword.prefixs or iword.postfixs): return True # verb + postfixs ex: برنامه if (iword.is_verb and iword.prefixs) or(iword.is_verb and iword.postfixs and (len(iword.postfixs) > 1 or not any(p in iword.postfixs for p in OneShotTransformer.PossessiveـPronouns +[OneShotTransformer.V]))): return True return False def iword2str(self, iword): sorted_prefixs = list(sorted(iword.prefixs, key=lambda prefix: prefix.level)) sorted_postfixs = list(sorted(iword.postfixs, key=lambda postfix: postfix.level)) concated_str = '' zipped_prefixs = [(sorted_prefixs[i], sorted_prefixs[i + 1]) if i < len(sorted_prefixs) - 1 else ( sorted_prefixs[i], iword.lemma) for i in range(len(sorted_prefixs))] for prev_prefix, prefix in zipped_prefixs: separator = self.get_separator(prev_prefix, prefix, append_h=False) prefix_formal = prev_prefix.formal concated_str += prefix_formal concated_str += separator concated_str += iword.lemma zipped_postfix = [(sorted_postfixs[i - 1], sorted_postfixs[i]) if i > 0 else (iword.lemma, sorted_postfixs[i]) for i in range(len(sorted_postfixs))] for postfix, next_postfix in zipped_postfix: separator = self.get_separator(postfix, next_postfix, append_h=iword.append_h) concated_str += separator postfix_formal = next_postfix.formal concated_str += postfix_formal return concated_str def to_formals(self, iword): str_iwords = [] all_iwords = self.lemma_to_formals(iword) for iword in all_iwords: # if iword.lemma == 'اون': # print('') if len(iword.lemma) == 1 and iword.lemma != 'و': str_iwords.append(('', None)) continue if self.filtered_based_on_rules(iword): str_iwords.append(('', None)) continue if self.should_ignore_by_postagg(iword): str_iwords.append(('', None)) continue if not iword.is_verb and not iword.semi_mapper and iword.lemma not in self.vocab: str_iwords.append(('', None)) continue concated_str = self.iword2str(iword) str_iwords.append((concated_str, iword)) return str_iwords def un_in(self, iword): new_lemma = iword.lemma.replace('ون', 'ان') if new_lemma != iword.lemma: return InformalWord(lemma=new_lemma, prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos) else: return False def prefix_obj(self, word): op_separete = {'م': 'من', 'ت': 'تو', 'ش': 'آن', 'تان': 'شما', 'تون': 'شما', 'شون': 'آنان', 'شان': 'آنان', 'مان': 'ما', 'مون': 'ما'} candidates = [] formal = '' m = self.pre_obj_pattern.match(word) if m: tokens = m.groups() if tokens[0] == 'باها': formal += 'با' else: formal += tokens[0] formal_obj = op_separete[tokens[1]] formal += ' ' formal += formal_obj if tokens[2] is not None: formal += ' ' formal += 'هم' alts = {'هم': 'هستم', 'آن': 'او'} tokens = [[w] for w in formal.split()] for t in tokens: if t[0] in alts: t.append(alts[t[0]]) candidates = itertools.product(*tokens) candidates = [' '.join(cnd) for cnd in candidates] return [(c, c) for c in candidates] def append_tanvin_hat(self, iword): if len(iword.lemma) > 1 and iword.lemma[0] == 'ا' and iword.lemma[-1] != 'ا': new_lemma = 'آ' + iword.lemma[1:] return InformalWord(lemma=new_lemma, prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos) if len(iword.lemma) > 1 and iword.lemma[-1] == 'ا': new_lemma = iword.lemma[:-1] + 'اً' return InformalWord(lemma=new_lemma, prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos) return False def append_h(self, iword): not_apply = self.verb_to_formal_func(iword.lemma) or (iword.lemma and iword.lemma[-1] in ['ا', 'و', 'ی']) or len(iword.lemma) <= 1 or iword.lemma =='' or iword.lemma[-1] == 'ه' or (OneShotTransformer.H in iword.postfixs and len(iword.postfixs) == 1) or any(p in iword.postfixs for p in OneShotTransformer.As) or(OneShotTransformer.V in iword.postfixs) or (iword.postfixs and iword.postfixs[0].word[0] in ['ی', 'و','ا']) ######## when add h? new_lemma = iword.lemma + 'ه' ############# new_lemma in self.vocab if len(iword.postfixs) > 0 and not any([p in OneShotTransformer.cant_append_h_posts for p in iword.postfixs]) and not not_apply and new_lemma not in self.non_hidden_h_words: # if len(iword.postfixs) > 0 and not not_apply and new_lemma in self.vocab and new_lemma not in self.non_hidden_h_words: return InformalWord(lemma=new_lemma, prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos, append_h= True) return False def __init__(self, vocab, mapper, verb_to_formal_func, ignore_words, postfix_mapper, isolated_words, non_hidden_h_words): self.vocab = vocab self.mapper = mapper self.verb_to_formal_func = verb_to_formal_func self.ignore_words = ignore_words self.postfix_mapper = postfix_mapper self.isolated_words = isolated_words self.non_hidden_h_words = non_hidden_h_words self.operators = [self.un_in, self.append_h, self.append_tanvin_hat] patt = r'(از|به|باها)(مان|شون|شان|مون|م|تون|تان|ت|ش)(م)?$' self.pre_obj_pattern = re.compile(patt) def all_sequence_of_postfixs(self, word, index): all_seqs =[] for p in OneShotTransformer.posts[index]: p_w = p.word if word.startswith(p_w): w = word[len(p_w):] if len(w) == 0: all_seqs.append(p) else: if index < len(OneShotTransformer.posts) -1 : resp = self.all_sequence_of_postfixs(w, index+1) if len(resp) > 0: for item in resp: if type(item) == list: item.append(p) sequence_with_p = item else: sequence_with_p = [p, item] all_seqs.append(sequence_with_p) if index < len(OneShotTransformer.posts) - 1: resp = self.all_sequence_of_postfixs(word, index + 1) all_seqs.extend(resp) else: return all_seqs return all_seqs def combine(self, l1, l2): if len(l1) == 0: return l2 elif len(l2) == 0: return l1 return list(itertools.product(l1, l2)) def get_expand(self, iword): all_possible_words = [] for subset_operators in utils.powerset(self.operators): new_iword = InformalWord(lemma=iword.lemma, prefixs=iword.prefixs, postfixs=iword.postfixs, pos=iword.pos) for so in subset_operators: so_resp = so(new_iword) if so_resp: new_iword = so_resp all_possible_words.append(new_iword) return all_possible_words def match_postfixs(self, word, pos): possible_combinatios = [] for i in range(len(OneShotTransformer.posts)): for p in OneShotTransformer.posts[i]: p_word = p.word p_indxs = [indx for indx, ch in enumerate(word) if word[indx:indx+len(p_word)] == p_word] for p_indx in p_indxs: if p_indx != -1: lemma = word[:p_indx] pp = word[p_indx + len(p_word):] if len(pp) ==0: iw = InformalWord(lemma=lemma, postfixs=[p], pos=pos) possible_combinatios.append(iw) continue if i < len(OneShotTransformer.posts) -1: all_postfix = self.all_sequence_of_postfixs(pp, index=i+1) if len(all_postfix) > 0: for pfixs in all_postfix: if type(pfixs) == list: pfixs.append(p) else: pfixs = [p, pfixs] iw = InformalWord(lemma=lemma, postfixs=pfixs, pos=pos) possible_combinatios.append(iw) elif len(pp) == 0: iw = InformalWord(lemma=lemma, postfixs=[p], pos=pos) possible_combinatios.append(iw) return possible_combinatios def match_prefixs(self, word, pos): possible_combinatios = [] for i in range(len(OneShotTransformer.pres)): for p in OneShotTransformer.pres[i]: if word.startswith(p.word): lemma = word[len(p.word):] prefixs = [p] iw = InformalWord(lemma=lemma, prefixs=prefixs, postfixs=[], pos=pos) possible_combinatios.append(iw) return possible_combinatios return [] def parse_word(self, iword): parsed_resp = [] prefixed_word = self.match_prefixs(iword.lemma,pos=iword.pos) prefixed_word.append(iword) parsed_resp.extend(prefixed_word) for pw in prefixed_word: postfixed_iwords = self.match_postfixs(pw.lemma,pos=iword.pos) for piw in postfixed_iwords: piw.prefixs = pw.prefixs parsed_resp.append(piw) return parsed_resp def is_seqs_of_verbs(self, txt): words = txt.split() if len(words) < 2: return False for w in words: formal_verb = self.verb_to_formal_func(w) if formal_verb is None: return False if words[-1] in ['است', 'هست']: return False return True def filter_results(self, word_lemmas): return list(filter(lambda wl: len(wl[0])>0 and wl[0][-1] != '‌' and not self.is_seqs_of_verbs(wl[0]), word_lemmas)) def concatenate_formal_words(self, pre, next): """ خانه +‌ ت -> خانه‌ات دیگر + ای -> دیگری """ nim_fasele = '‌' not_connect_chars = ['ا', 'د', 'ذ', 'ر', 'ز', 'ژ', 'و'] if len(pre) < 1 : return next if pre[-1] in ['ه'] and next in ['م', 'ت', 'ش']: return pre + nim_fasele + 'ا' + next if pre[-1] == 'ا'and next.split() and next.split()[0] in ['م', 'ت', 'ش', 'مان', 'تان', 'شان']: return pre + nim_fasele + 'ی' + next if pre[-1] not in ['ه'] and next in ['ای']: return pre + 'ی' out = pre + next if pre[-1] not in not_connect_chars or next.startswith('ها') or pre[-1] in ['ه'] or pre + nim_fasele + next in self.vocab: out = pre + nim_fasele + next if self.verb_to_formal_func(next): out = pre + ' ' + next return out def handle_nim_fasele_words(self, word, pos): def extract_lemma_nim_fasele_words(word, pos): formal_prefixs = [] formal_postfixs = [] prefixs = {'اون': 'آن', 'همون': 'همین'} postfixs = self.postfix_mapper tokens = word.split('‌') index = 0 for i in range(len(tokens)): index = i if tokens[i] not in prefixs: break else: formal_prefixs.append(prefixs[tokens[i]]) for i in range(len(tokens), index, -1): current_tok = '‌'.join(tokens[index:i]) if current_tok in self.vocab or tokens[i - 1] not in postfixs: return formal_prefixs, current_tok, formal_postfixs else: formal_postfixs.append(postfixs[tokens[i - 1]]) return formal_prefixs, current_tok, formal_postfixs nim_fasele = '‌' candidates = [] formal_word = '' verbs = self.verb_to_formal_func(word) if verbs: return [(v, v) for v in verbs] all_candidates = set() # lemma formal_prefixs, lemma, formal_postfixs = extract_lemma_nim_fasele_words(word, pos) word_lemmas = self.transform(lemma, pos, ignore_nim_fasele=True) # lemma with postfix should len=1 one_token_words = [wl for wl in word_lemmas if len(wl[0].split()) == 1] if formal_postfixs and one_token_words: all_formal_lemma_candidates = one_token_words else: all_formal_lemma_candidates = word_lemmas if not all_formal_lemma_candidates: if formal_postfixs or formal_prefixs: all_formal_lemma_candidates = [(lemma, lemma)] else: tokens = lemma.split(nim_fasele) if all(self.transform(t, None, ignore_nim_fasele=True) for t in tokens): w = ' '.join(tokens) return [(w, w)] else: return [] for cnd_lemma, formal_word_lemma in all_formal_lemma_candidates: formal_word = '' toks = formal_prefixs + [cnd_lemma] + formal_postfixs for index, t in enumerate(toks): formal_word = self.concatenate_formal_words(formal_word, t) all_candidates.add((formal_word, formal_word_lemma)) # if t in self.postfix_mapper: # formal_t = self.postfix_mapper[t] # else: # transform_outputs = self.transform(t, pos) # if not transform_outputs: # formal_t = t # else: # one_word_outputs = [ft for ft in transform_outputs if len(ft.split()) == 1] # if one_word_outputs: # if t in one_word_outputs: # formal_t = t # else: # formal_t = one_word_outputs[0] # else: # formal_t = transform_outputs.pop() return all_candidates def transform(self, word, pos, ignore_nim_fasele=False): """ignore emoji , punctuation, numbers""" ignore_chars = '.1234567890!@#$%^&*()_+۱۲۳۴۵۶۷۸۹÷؟×−+?><}،,{":' + string.ascii_lowercase + string.ascii_uppercase if any(ic in word for ic in ignore_chars) or utils.if_emoji(word): return [(word, word)] """handle nim fasele""" nim_fasele = '‌' if not ignore_nim_fasele and nim_fasele in word: return self.handle_nim_fasele_words(word, pos) # pass ignore words and accept as correct informal word! if word in self.ignore_words and not word in self.mapper: return [(word, word)] formal_prefix_obj = self.prefix_obj(word) if formal_prefix_obj: return formal_prefix_obj iword = InformalWord(lemma=word, pos=pos) expanded_candidates = [] candidates = self.parse_word(iword) #just verbs if any(c.is_verb for c in candidates): candidates = [c for c in candidates if c.is_verb] for cnd in candidates: expanded_candidates.extend(self.get_expand(cnd)) word_iwords = [] for ec in expanded_candidates: word_iwords.extend(self.to_formals(ec)) if any(f[1] and (f[1].is_mapper or f[1].is_verb) for f in word_iwords if f[1] is not None): word_iwords = [f for f in word_iwords if f[1] and (f[1].is_mapper or f[1].is_verb)] # else: word_lemmas_set = [(w, iword.lemma) for w, iword in word_iwords if iword is not None] word_lemmas_set = set(word_lemmas_set) out = self.filter_results(word_lemmas_set) # if type(out) == str: # out = [out] # out = set(out) return out if __name__ == '__main__': transformer = OneShotTransformer(None, None, None) candidates = transformer.match_postfixs('کارامم') print(candidates)