Delete utils.py

utils.py

@@ -1,415 +0,0 @@
-import numpy as np
-
-class Config:
-    def __init__(self):
-        super(Config, self).__init__()
-
-
-def read_conll_ner(path):
-    with open(path) as f:
-        lines = f.readlines()
-    unique_entries = []
-    sentences = []
-    curr_sentence = []
-    for line in lines:
-        if not line.strip():
-            if curr_sentence:
-                sentences.append(curr_sentence)
-            curr_sentence = []
-            continue
-        entry = line.split()
-        curr_sentence.append(entry)
-        if not len(unique_entries):
-            unique_entries = [[] for _ in entry[1:]]
-        for e, list in zip(entry[1:], unique_entries):
-            if e not in list:
-                list.append(e)
-    return [sentences] + unique_entries
-
-
-def read_pickled_conll(path):
-    with open(path, "rb") as f:
-        data = pickle.load(f)
-    return data
-
-
-def split_conll_docs(conll_sents, skip_docstart=True):
-    docs = []
-    curr_doc = []
-    for sent in conll_sents:
-        if sent[0][0] == '-DOCSTART-':
-            if curr_doc:
-                docs.append(curr_doc)
-                curr_doc = []
-            if skip_docstart:
-                continue
-        curr_doc.append(sent)
-    docs.append(curr_doc)
-    return docs
-
-
-def create_context_data(docs, pos_col_id=1, tag_col_id=3, context_length=1, **kwargs):
-    ctx_type = kwargs.get("ctx_type", "other")
-    sep_token = kwargs.get("sep_token", "[SEP]")
-    if ctx_type == "cand_titles":
-        # create context for candidate titles scenario
-        for doc in docs:
-            doc["ctx_sent"] = doc["query"] + [sep_token] + f"<split>{sep_token}<split>".join([cand["doc_title"] for cand in doc["BM25_cands"]]).split("<split>")
-        return docs
-    if ctx_type == "cand_links":
-        for doc in docs:
-            doc_titles_list = f"<split>{sep_token}<split>".join([cand["doc_title"] for cand in doc["BM25_cands"]]).split("<split>")
-            linked_titles_list = f"<split>{sep_token}<split>".join([linked for cand in doc["BM25_cands"] for linked in cand["linked_titles"]]).split("<split>")
-            doc["ctx_sent"] = doc["query"] + [sep_token] + doc_titles_list + [sep_token] + linked_titles_list
-        return docs
-    if ctx_type == "raw_text":
-        # create context for candidate raw text
-        for doc in docs:
-            doc["ctx_sent"] = [doc["query"] + [sep_token] + [cand["processed_text"]] for cand in doc["BM25_cands"]]
-        return docs
-    if ctx_type == 'matched_spans':
-        matched_spans = kwargs.get('matched_spans')
-        return [
-            [[t[0] for t in d] + [t for ms in ms for t in [sep_token] + ms[1]],     # sentence tokens + spans
-             None,                                                                  # pos tags
-             [s[tag_col_id] for s in d] if tag_col_id > 0 else None,                # ner tags
-             [len(d)]                                                               # sentence length
-             ]
-            for d, ms in zip(docs, matched_spans)]
-    if ctx_type == 'bm25_matched_spans':
-        matched_spans = kwargs.get('matched_spans')
-        pickled_data = kwargs.get('pickled_data')
-        docs = [[[t[0] for t in d] + [t for ms in ms for t in [sep_token] + ms[1]],  # sentence tokens + spans
-             None,  # pos tags
-             [s[tag_col_id] for s in d],  # ner tags
-             [len(d)]  # sentence length
-             ]
-            for d, ms in zip(docs, matched_spans)]
-        for ms, doc in zip(docs,pickled_data):
-            doc_titles_list = f"<split>{sep_token}<split>".join([cand["doc_title"] for cand in doc["BM25_cands"]]).split("<split>")
-            linked_titles_list = f"<split>{sep_token}<split>".join([linked for cand in doc["BM25_cands"] for linked in cand["linked_titles"]]).split("<split>")
-            ms[0] = ms[0] + [sep_token] + doc_titles_list + [sep_token] + linked_titles_list
-        return docs
-    if ctx_type == "infobox":
-        infobox_keys_path = kwargs.get("infobox_keys_path")
-        infobox_keys = read_pickled_conll(infobox_keys_path)
-        if 'pred_spans' in docs[0]:
-            docs = get_pred_ent_bounds(docs)
-        for doc in docs:
-            if 'pred_spans' in doc:
-                ents = [' '.join(doc['query'][bd[0]:bd[1] + 1]) for bd in doc['pred_ent_bounds']]
-                ents_wo_space = [''.join(doc['query'][bd[0]:bd[1] + 1]) for bd in doc['pred_ent_bounds']]
-            else:
-                ents = [' '.join(doc['query'][bd[0]:bd[1] + 1]) for bd in doc['ent_bounds']]
-                ents_wo_space = [''.join(doc['query'][bd[0]:bd[1] + 1]) for bd in doc['ent_bounds']]
-            ents = list(set(ents + ents_wo_space))
-            infobox = [infobox_keys[en] for en in ents if en in infobox_keys and infobox_keys[en]]
-            for ibs in infobox:
-                ibs[0] = '[INFO] ' + ibs[0]
-                ibs[-1] = ibs[-1] + ' [/INFO]'
-            infobox = [i for j in infobox for i in j]
-            doc["ctx_sent"] = doc["query"] + [sep_token] + infobox
-        return docs
-    # create context type for other scenarios
-    res = []
-    for doc in docs:
-        ctx_len = context_length if context_length > 0 else len(doc)
-        # for the last sentences loop around to the beginning for context
-        padded_doc = doc + doc[:ctx_len]
-        for i in range(len(doc)):
-            res.append((
-                [s[0] for sent in padded_doc[i:i+ctx_len] for s in sent],
-                [s[pos_col_id] for sent in padded_doc[i:i+ctx_len] for s in sent] if pos_col_id > 0 else None,
-                [s[tag_col_id] for sent in padded_doc[i:i+ctx_len] for s in sent],
-                [len(sent) for sent in padded_doc[i:i+ctx_len]],
-                {}  # dictionary for extra context
-            ))
-    return res
-
-
-def calc_correct(sentence):
-    gold_chunks = []
-    parallel_chunks = []
-    pred_chunks = []
-    curr_gold_chunk = []
-    curr_parallel_chunk = []
-    curr_pred_chunk = []
-    prev_tag = None
-    for line in sentence:
-        _, _, _, gt, pt = line
-        curr_tag = None
-        if '-' in pt:
-            curr_tag = pt.split('-')[1]
-        if gt.startswith('B'):
-            if curr_gold_chunk:
-                gold_chunks.append(curr_gold_chunk)
-                parallel_chunks.append(curr_parallel_chunk)
-            curr_gold_chunk = [gt]
-            curr_parallel_chunk = [pt]
-        elif gt.startswith('I') or (pt.startswith('I') and curr_tag == prev_tag
-                                    and curr_gold_chunk):
-            curr_gold_chunk.append(gt)
-            curr_parallel_chunk.append(pt)
-        elif gt.startswith('O') and pt.startswith('O'):
-            if curr_gold_chunk:
-                gold_chunks.append(curr_gold_chunk)
-                parallel_chunks.append(curr_parallel_chunk)
-                curr_gold_chunk = []
-                curr_parallel_chunk = []
-        if pt.startswith('O'):
-            if curr_pred_chunk:
-                pred_chunks.append(curr_pred_chunk)
-                curr_pred_chunk = []
-        elif pt.startswith('B'):
-            if curr_pred_chunk:
-                pred_chunks.append(curr_pred_chunk)
-            curr_pred_chunk = [pt]
-            prev_tag = curr_tag
-        else:
-            if prev_tag is not None and curr_tag != prev_tag:
-                prev_tag = curr_tag
-                if curr_pred_chunk:
-                    pred_chunks.append(curr_pred_chunk)
-                    curr_pred_chunk = []
-            curr_pred_chunk.append(pt)
-
-    if curr_gold_chunk:
-        gold_chunks.append(curr_gold_chunk)
-        parallel_chunks.append(curr_parallel_chunk)
-    if curr_pred_chunk:
-        pred_chunks.append(curr_pred_chunk)
-    correct = sum([1 for gc, pc in zip(gold_chunks, parallel_chunks)
-                   if not len([1 for g, p in zip(gc, pc) if g != p])])
-    correct_tagless = sum([1 for gc, pc in zip(gold_chunks, parallel_chunks)
-                   if not len([1 for g, p in zip(gc, pc) if g[0] != p[0]])])
-    # return correct, gold_chunks, parallel_chunks, pred_chunks, ob1_correct, correct_tagless
-    return {'correct': correct,
-            'correct_tagless': correct_tagless,
-            'gold_count': len(gold_chunks),
-            'pred_count': len(pred_chunks)}
-
-
-def tag_sentences(sentences):
-    nlp = stanza.Pipeline(lang='en', processors='tokenize,pos', logging_level='WARNING')
-    tagged_sents = []
-    for sentence in sentences:
-        n = nlp(sentence)
-        tagged_sent = []
-        for s in n.sentences:
-            for w in s.words:
-                tagged_sent.append([w.text, w.upos])
-        tagged_sents.append(tagged_sent)
-    return tagged_sents
-
-
-def extract_spans(sentence, tagless=False):
-    spans_positions = []
-    span_bounds = []
-    all_bounds = []
-    span_tags = []
-    curr_tag = None
-    curr_span = []
-    curr_span_start = -1
-    # span ids, span types
-    for i, token in enumerate(sentence):
-        if token.startswith('B'):
-            if curr_span:
-                spans_positions.append([curr_span, len(all_bounds)])
-                span_bounds.append([curr_span_start, i-1])
-                all_bounds.append([[curr_span_start, i - 1], 'E', len(all_bounds)])
-                if not tagless:
-                    span_tags.append(token.split('-')[1])
-                curr_span = []
-                curr_tag = None
-            curr_span.append(token)
-            curr_tag = None if tagless else token.split('-')[1]
-            curr_span_start = i
-        elif token.startswith('I'):
-            if not tagless:
-                tag = token.split('-')[1]
-                if tag != curr_tag and curr_tag is not None:
-                    spans_positions.append([curr_span, len(all_bounds)])
-                    span_bounds.append([curr_span_start, i - 1])
-                    span_tags.append(token.split('-')[1])
-                    all_bounds.append([[curr_span_start, i - 1], 'E', len(all_bounds)])
-                    curr_span = []
-                    curr_tag = tag
-                    curr_span_start = i
-                elif curr_tag is None:
-                    curr_span = []
-                    curr_tag = tag
-                    curr_span_start = i
-            elif not curr_span:
-                curr_span_start = i
-            curr_span.append(token)
-        elif token.startswith('O') or token.startswith('-'):
-            if curr_span:
-                spans_positions.append([curr_span, len(all_bounds)])
-                span_bounds.append([curr_span_start, i-1])
-                all_bounds.append([[curr_span_start, i-1], 'E', len(all_bounds)])
-                curr_span = []
-            curr_tag = None
-            all_bounds.append([[i], 'W', len(all_bounds)])
-    # check if sentence ended with a span
-    if curr_span:
-        spans_positions.append([curr_span, len(all_bounds)])
-        span_bounds.append([curr_span_start, len(sentence) - 1])
-        all_bounds.append([[curr_span_start, len(sentence) - 1], 'E', len(all_bounds)])
-    tagged_bounds = [[loc[0][0].split('-')[1] if '-' in loc[0][0] else loc[0][0], bound]
-                     for loc, bound in zip(spans_positions, span_bounds)]
-    return spans_positions, span_bounds, all_bounds, tagged_bounds
-
-
-def ner_corpus_stats(corpus_path):
-    onto_train_cols = read_conll_ner(corpus_path)
-    tags = list(set([t.split('-')[1] for t in onto_train_cols[3] if '-' in t]))
-    onto_train_spans = [extract_spans([t[3] for t in sent])[3] for sent in
-                        onto_train_cols[0]]
-    span_lens = [span[1][1] - span[1][0] + 1 for sent in onto_train_spans for
-                 span in sent]
-
-    len_stats = [span_lens.count(i + 1) / len(span_lens) for i in
-                 range(max(span_lens))]
-    flat_spans = [span for sent in onto_train_spans for span in sent]
-
-    tag_lens_dict = {k: [] for k in tags}
-    tag_counts_dict = {k: 0 for k in tags}
-    for span in flat_spans:
-        span_length = span[1][1] - span[1][0] + 1
-        span_tag = span[0][0].split('-')[1]
-        tag_lens_dict[span_tag].append(span_length)
-        tag_counts_dict[span_tag] += 1
-
-    x = list(tag_counts_dict.items())
-    x.sort(key=lambda l: l[1])
-    tag_counts = [list(l) for l in x]
-    for l in tag_counts:
-        l[1] = l[1] / len(span_lens)
-
-    tag_len_stats = {k: [v.count(i + 1) / len(v) for i in range(max(v))]
-                     for k, v in tag_lens_dict.items()}
-    span_texts = [sent[span[1][0]:span[1][1] + 1]
-                  for sent, spans in zip(onto_train_cols[0], onto_train_spans)
-                  for span in spans]
-    span_pos = [[span[0][-1].split('-')[1], '_'.join(t[1] for t in span)]
-                for span in span_texts]
-    unique_pos = list(set([span[1] for span in span_pos]))
-    pos_dict = {k: 0 for k in unique_pos}
-    for span in span_pos:
-        pos_dict[span[1]] += 1
-    unique_pos.sort(key=lambda l: pos_dict[l], reverse=True)
-    pos_stats = [[p, pos_dict[p] / len(span_pos)] for p in unique_pos]
-    tag_pos_dict = {kt: {kp: 0 for kp in unique_pos} for kt in tags}
-    for span in span_pos:
-        tag_pos_dict[span[0]][span[1]] += 1
-    tag_pos_stats = {kt: [[p, tag_pos_dict[kt][p] / tag_counts_dict[kt]]
-                          for p in unique_pos] for kt in tags}
-    for kt in tags:
-        tag_pos_stats[kt].sort(key=lambda l: l[1], reverse=True)
-
-    return len_stats, tag_len_stats, tag_counts, pos_stats, tag_pos_stats
-
-
-def filter_by_max_ents(sentences, max_ent_length):
-    """
-    Filters a given list of sentences and only returns the sentences that have
-    named entities shorter than or equal to the given max_ent_length.
-
-    :param sentences: sentences in conll format as extracted by read_conll_ner
-    :param max_ent_length: The maximum number of tokens in an entity
-    :return: a lits of sentences
-    """
-    filtered_sents = []
-    for sent in sentences:
-        sent_span_lens = [s[1] - s[0] + 1
-                          for s in extract_spans([t[3] for t in sent])[1]]
-        if not sent_span_lens or max(sent_span_lens) <= max_ent_length:
-            filtered_sents.append(sent)
-    return filtered_sents
-
-
-def get_pred_ent_bounds(docs):
-    for doc in docs:
-        eb = []
-        count = 0
-        for p_eb in doc['pred_spans']:
-            if p_eb == 'B':
-                eb.append([count,count])
-            elif p_eb == 'I' and len(eb) > 0:
-                eb[-1][1] = count
-            count += 1
-        doc['pred_ent_bounds'] = eb
-    return docs
-
-def enumerate_spans(batch):
-  
-  enumerated_spans_batch = []
-  
-  for idx in range(0, len(batch)):
-    sentence_length = batch[idx]
-    enumerated_spans = []
-    for x in range(len(sentence_length)):
-      for y in range(x, len(sentence_length)):
-        enumerated_spans.append([x,y])
-        
-    enumerated_spans_batch.append(enumerated_spans)
-  
-  return enumerated_spans_batch
-
-def compact_span_enumeration(batch):
-  sentence_lengths = [len(b) for b in batch]
-  enumerated_spans = [[[x, y] 
-                        for y in range(0, sentence_length) 
-                        for x in range(sentence_length)] 
-                    for sentence_length in sentence_lengths]
-  return enumerated_spans
-
-def preprocess_data(data):
-  clean_data = []
-  for sample in data:
-    clean_tokens = [araby.strip_tashkeel(token) for token in sample[0]]
-    clean_tokens = [araby.strip_tatweel(token) for token in clean_tokens]
-    clean_sample = [clean_tokens] 
-    clean_sample.extend(sample[1:])
-    clean_data.append(clean_sample)
-  return clean_data
-  
-
-def generate_targets(enumerated_spans, sentences):
-  #### could be refactored into a helper function ####
-  extracted_spans= [extract_spans(sentence,True)[3] for sentence in sentences]
-  target_locations = []
-
-  for span in extracted_spans:
-    sentence_locations = []
-    for location in span:
-      sentence_locations.append(location[1])
-    target_locations.append(sentence_locations)
-
-  #### could be refactored into a helper function ####
-  
-
-  targets= []
-  
-  for span, location_list in zip(enumerated_spans, target_locations):
-    span_arr = np.zeros_like(span).tolist()
-    target_indices = [span.index(span_location) for
-                      span_location in location_list]
-    
-
-    for idx in target_indices:
-      span_arr[idx] =1
-
-    span_arr = [0 if x!=1 else x for x in span_arr]
-    targets.append(list(span_arr))
-    
-  return targets
-
-def label_tags(tags):
-    output_tags = []
-    for tag in tags:
-        if (tag == "O"):
-            output_tags.append(0)
-        else:
-            output_tags.append(1)
-    return output_tags