utilities_language_bert/esp_sentence_bert.py

import string
from random import random
from random import sample
from utilities_language_general.esp_constants import nlp
from utilities_language_general.morphology import inflect
from utilities_language_general.esp_constants import PHRASES
from utilities_language_general.esp_utils import check_token_bert
from utilities_language_general.esp_utils import fix_irregular_lemma
from utilities_language_general.esp_constants import BAD_USER_TARGET_WORDS
from utilities_language_general.esp_utils import get_distractors_from_model_bert


class SENTENCE:
    def __init__(self, original: str, n_sentence: int, max_num_distractors):
        self.original = original
        self.n_sentence = n_sentence
        self.max_num_distractors = max_num_distractors
        self.parsed = nlp(self.original)
        self.sentence_lemma_pos = []
        self.sentence_phrases = []
        self.target_words = []
        self.text_with_masked_task = ''

    def lemmatize_sentence(self):
        for token in self.parsed:
            lemma_pos = f'{token.lemma_}_{token.pos_}'
            if token.pos_ in ('AUX', 'VERB', 'ADJ'):
                lemma_pos = fix_irregular_lemma(lemma=lemma_pos)
            self.sentence_lemma_pos.append((lemma_pos, token))

    def bind_phrases(self):
        previous_was_phrase = False
        for i in range(len(self.sentence_lemma_pos) - 1):
            phrase_candidate = f'{self.sentence_lemma_pos[i][0]}_{self.sentence_lemma_pos[i + 1][0]}'
            if phrase_candidate in PHRASES and not previous_was_phrase:
                # phrase is {phrase: {original_token1: spacy.token, original_token2: spacy.token}}
                phrase = [
                    f'{self.sentence_lemma_pos[i][0]}_{self.sentence_lemma_pos[i + 1][0]}',
                    {
                        'original_token1': self.sentence_lemma_pos[i][1],
                        'original_token2': self.sentence_lemma_pos[i + 1][1]
                    }
                ]
                self.sentence_phrases.append(phrase)
                previous_was_phrase = True
            else:
                if not previous_was_phrase:
                    self.sentence_phrases.append(self.sentence_lemma_pos[i][1])
                previous_was_phrase = False

    def search_target_words_automatically(self, target_minimum: set, frequency_dict: dict = None):
        for token in self.sentence_phrases:
            if isinstance(token, list):  # if token is a phrase
                original_token1 = token[1]['original_token1']
                original_token2 = token[1]['original_token2']
                original_token1_tags = original_token1.morph.to_dict()
                original_token2_tags = original_token2.morph.to_dict()
                tags = dict()
                if ('haber_AUX' == f'{original_token1.lemma_}_{original_token1.pos_}'
                        and original_token2.pos_ in ('VERB', 'ADJ', 'AUX')):
                    tags['VerbForm'] = 'Compuesto'
                    tags['Mood'] = original_token1_tags.get('Mood')
                    tags['Tense'] = original_token1_tags.get('Tense')
                    tags['Person'] = original_token1_tags.get('Person')
                    tags['Number'] = original_token1_tags.get('Number')
                    tags['Gender'] = None
                else:
                    tags = {**original_token1_tags, **original_token2_tags}
                not_ner = True if (original_token1.ent_type == 0 and original_token2.ent_type == 0) else False
                target_word = {
                    'masked_sentence': self.original.replace(f'{original_token1.text} {original_token2.text}',
                                                             '[MASK]'),
                    'sentence_number': self.n_sentence,
                    'sentence_text': self.original,
                    'original_text': f'{original_token1.text} {original_token2.text}',
                    'lemma': token[0],
                    'pos': ('phrase', 'phrase'),
                    'gender': tags.get('Gender'),
                    'tags': tags,
                    'position_in_sentence': self.original.find(original_token1.text),
                    'not_named_entity': not_ner,
                    'frequency_in_text': 0
                }
                self.target_words.append(target_word)
            else:  # if token is just a spacy.nlp token
                if check_token_bert(token=token, current_minimum=target_minimum):
                    tags = token.morph.to_dict()
                    target_word = {
                        'masked_sentence': self.original.replace(token.text, '[MASK]'),
                        'sentence_number': self.n_sentence,
                        'sentence_text': self.original,
                        'original_text': token.text,
                        'lemma': token.lemma_,
                        'pos': ('simple', token.pos_),
                        'gender': tags.get('Gender'),
                        'number_children': len([child for child in token.children]),
                        'tags': tags,
                        'position_in_sentence': self.original.find(token.text),
                        'not_named_entity': True if token.ent_type == 0 else False,
                        'frequency_in_text': frequency_dict.get(token.lemma_, 1),
                    }
                    self.target_words.append(target_word)

    def search_user_target_words(self, user_target_words: set = None, frequency_dict: dict = None):
        for _utw in user_target_words:
            if _utw in self.original:
                parse_utw = nlp(_utw)
                if ' ' in _utw:
                    tags = dict()
                    if ('haber_AUX' == f'{parse_utw[0].lemma_}_{parse_utw[0].pos_}'
                            and parse_utw[1].pos_ in ('VERB', 'ADJ', 'AUX')):
                        tags['VerbForm'] = 'Compuesto'
                        tags['Mood'] = parse_utw[0].morph.to_dict().get('Mood')
                        tags['Tense'] = parse_utw[0].morph.to_dict().get('Tense')
                        tags['Person'] = parse_utw[0].morph.to_dict().get('Person')
                        tags['Number'] = parse_utw[0].morph.to_dict().get('Number')
                        tags['Gender'] = None
                    else:
                        tags = {**parse_utw[0].morph.to_dict(), **parse_utw[1].morph.to_dict()}
                    user_target_word_lemma = '_'.join([f'{token.lemma_}_{token.pos_}' for token in parse_utw])
                    user_target_word_pos = 'phrase'
                    user_target_word_tags = tags
                    not_ner = True if (parse_utw[0].ent_type == 0 and parse_utw[1].ent_type == 0) else False
                else:
                    user_target_word_lemma = f'{parse_utw[0].lemma_}_{parse_utw[0].pos_}'
                    user_target_word_pos = ('simple', parse_utw[0].pos_)
                    user_target_word_tags = parse_utw[0].morph.to_dict()
                    not_ner = parse_utw[0].ent_type == 0
                target_word = {
                    'masked_sentence': self.original.replace(_utw, '[MASK]'),
                    'sentence_number': self.n_sentence,
                    'sentence_text': self.original,
                    'original_text': _utw,
                    'lemma': user_target_word_lemma,
                    'pos': user_target_word_pos,
                    'gender': user_target_word_tags.get('Gender'),
                    'tags': user_target_word_tags,
                    'position_in_sentence': self.original.find(_utw),
                    'not_named_entity': not_ner,
                    'frequency_in_text': frequency_dict.get(user_target_word_lemma, 1)
                }
                self.target_words.append(target_word)

    def search_target_words(self, target_words_automatic_mode: bool, target_minimum,
                            user_target_words: set = None,
                            frequency_dict: dict = None):
        if target_words_automatic_mode:
            self.search_target_words_automatically(target_minimum=target_minimum,
                                                   frequency_dict=frequency_dict)
        else:
            self.search_user_target_words(user_target_words=user_target_words,
                                          frequency_dict=frequency_dict)

    def filter_target_words(self, target_words_automatic_mode):
        c_position = 0
        bad_target_words = []
        for target_word in self.target_words:
            position_difference = 3 if target_words_automatic_mode else 0
            if not (target_word['position_in_sentence'] == 0
                    or abs(target_word['position_in_sentence'] - c_position) >= position_difference):
                bad_target_words.append(target_word)
        for btw in bad_target_words:
            BAD_USER_TARGET_WORDS.append(btw['original_text'])
            self.target_words.remove(btw)


class TASK:
    def __init__(self, task_data, max_num_distractors):
        self.task_data = task_data
        self.distractors = None
        self.distractors_number = 0
        self.bad_target_word = False
        self.inflected_distractors = None
        self.pos = task_data['pos']
        self.tags = task_data['tags']
        self.lemma = task_data['lemma']
        self.gender = task_data['gender']
        self.max_num_distractors = max_num_distractors
        self.original_text = task_data['original_text']
        self.sentence_text = task_data['sentence_text']
        self.sentence_number = task_data['sentence_number']
        self.masked_sentence = task_data['masked_sentence']
        self.frequency_in_text = task_data['frequency_in_text']
        self.position_in_sentence = task_data['position_in_sentence']
        self.text_with_masked_task = task_data['text_with_masked_task']
        self.result = ''
        self.variants = []

    def __repr__(self):
        return '\n'.join([f'{key}\t=\t{value}' for key, value in self.__dict__.items()])

    def attach_distractors_to_target_word(self, model, global_distractors, distractor_minimum,
                                          level_name, max_frequency):
        pos = self.pos[0] if self.pos[0] == 'phrase' else self.pos[1]
        # distractors_full_text = get_distractors_from_model_bert(model=model, lemma=self.lemma, pos=pos,
        #                                                         gender=self.gender, level_name=level_name,
        #                                                         text_with_masked_task=self.text_with_masked_task,
        #                                                         global_distractors=global_distractors,
        #                                                         distractor_minimum=distractor_minimum,
        #                                                         max_num_distractors=self.max_num_distractors)
        distractors_sentence = get_distractors_from_model_bert(model=model, lemma=self.lemma, pos=pos,
                                                               gender=self.gender, level_name=level_name,
                                                               text_with_masked_task=self.masked_sentence,
                                                               global_distractors=global_distractors,
                                                               distractor_minimum=distractor_minimum,
                                                               max_num_distractors=self.max_num_distractors)
        if distractors_sentence is None or self.frequency_in_text > max_frequency:
            self.bad_target_word = True
            self.distractors = None
        else:
            self.distractors = [d[0] for i, d in enumerate(distractors_sentence) if i < 15]
            self.distractors_number = len(distractors_sentence) if distractors_sentence is not None else 0

    def inflect_distractors(self):
        inflected_distractors = []
        if self.distractors is None:
            self.bad_target_word = True
            return
        for distractor_lemma in self.distractors:
            if distractor_lemma.count('_') > 1:
                if distractor_lemma.startswith('haber_'):
                    distractor_lemma = distractor_lemma.split('_')[-2]
                    inflected = inflect(lemma=distractor_lemma, target_pos=self.pos[1], target_tags=self.tags)
                else:
                    continue
            else:
                inflected = inflect(lemma=distractor_lemma, target_pos=self.pos[1], target_tags=self.tags)
            if inflected is not None:
                inflected_distractors.append(inflected)
        num_distractors = min(4, self.max_num_distractors) if self.max_num_distractors >= 4 \
            else self.max_num_distractors
        if len(inflected_distractors) < num_distractors:
            self.bad_target_word = True
        else:
            self.distractors_number = num_distractors
            self.inflected_distractors = inflected_distractors

    def sample_distractors(self, num_distractors):
        if not self.bad_target_word:
            num_distractors = min(self.distractors_number, num_distractors) if num_distractors >= 4 else num_distractors
            self.inflected_distractors = sample(self.inflected_distractors[:min(self.distractors_number, 10)],
                                                num_distractors)

    def compile_task(self, max_num_distractors):
        len_distractors = len(self.inflected_distractors)
        len_variants = min(len_distractors, max_num_distractors) if max_num_distractors > 4 \
            else max_num_distractors
        letters = (f'({letter})' for letter in string.ascii_lowercase[:len_variants + 1])
        try:
            distractors = sample(self.inflected_distractors, len_variants) + [self.original_text, ]
        except ValueError:
            distractors = self.inflected_distractors + [self.original_text, ]
        tmp_vars = [f'{item[0]} {item[1].replace("_", " ")}'.lower()
                    for item in zip(letters, sorted(distractors, key=lambda _: random()))]
        self.variants.append((self.original_text, tmp_vars))