src/textrank.py

import pandas as pd
import spacy
import math
from collections import Counter


try:
    from src.clean import clean_license_text
    from src.parameters import color, vocab
except:
    from clean import clean_license_text
    from parameters import color, vocab


GOLD_STANDARD_PATH = "../UBC-SAP_gold-corpus/UBC-SAP_capstone_corpus_labels_removed.xlsx"
LABELS_PATH = "data/choosealicense_appendix_labels.csv"
MIN_SENT_LEN = 3
SUMMARY_LEN = 0.3

nlp = spacy.load("en_core_web_sm")


def normalize_sentence_counter(counter):
    """
    Normalize sentence scores in the counter between 0 and 3

    Parameters
    ----------
    counter : dict
        A dictionary of scores with keys as sentence and values as raw scores.

    Returns
    -------
    counter : dict
        A dictionary of scores with keys as sentence and values as normalized
        scores.

    """
    vals = list(counter.values())

    if vals:
        min_val = min(vals)
        max_val = max(vals)
    else:
        return counter

    for sent in counter:
        try:
            counter[sent] = round(3 * (counter[sent] - min_val) / (max_val - min_val), 3)
        except:
            counter[sent] = 0
    return counter


def sent_tokenize_text(text, debug=False):
    """
    Tokenize a license text into sentences

    Parameters
    ----------
    text : str
        License text to be tokenized into sentences.
    debug : bool, optional
        Toggles debug mode. The default is False.

    Returns
    -------
    tokenized_sents : list
        A list of tokenized sentences.

    """
    tokenized_sents = list()
    paras = text.split("\n\n")
    for para in paras:
        for sent in nlp(para).sents:
            sent = sent.text.replace("\n", "").strip()
            if tokenized_sents and len(tokenized_sents[-1]) <= 30:
                tokenized_sents[-1] += f" {sent.strip()}"
            else:
                tokenized_sents.append(sent.strip())
        try:
            tokenized_sents[-1] += "\n\n"
        except:
            pass
    if debug:
        print("Segmented Sentences:")
        print("="*20)
        for i, sent in enumerate(tokenized_sents):
            print(f"Sent {i+1}")
            print("-"*20)
            print(sent)
            print("-"*50)
            print()
    return tokenized_sents


def lemmatize_tokens(sent):
    """
    Lemmatize tokens into the given sentence

    Parameters
    ----------
    sent : str
        A sentences whose tokens are to be lemmatized.

    Returns
    -------
    list
        A list of lemmatized tokens.

    """
    lemmas = list()

    nlp_sent = [token.lemma_.lower().strip() for token in nlp(sent)]

    for tok_i, token in enumerate(nlp_sent):
        if (token
            and token not in vocab.license_stopwords
            and token not in vocab.negation_words):
            if tok_i > 0 and nlp_sent[tok_i-1] in vocab.negation_words:
                lemmas.append(f"{nlp_sent[tok_i-1]}-{token}")
            elif (tok_i > 1
                  and nlp_sent[tok_i-1] in " -"
                  and nlp_sent[tok_i-2] in vocab.negation_words):
                lemmas.append(f"{nlp_sent[tok_i-2]}-{token}")
            else:
                lemmas.append(token)

    return [lemma for lemma in lemmas if len(lemma) > 2]


def get_license_summary_scores(license_text,
                               min_sent_len=MIN_SENT_LEN,
                               summary_len=SUMMARY_LEN,
                               summary_in_text_order=True,
                               return_summary_only=True,
                               debug=False,
                               cleaned_license_sentences=None):
    """
    Get sentence scores for all the cleaned sentences in a given license_text
    along with other extracted details such as definitions, exceptions, etc.
    and the cleaned license text itself.

    Parameters
    ----------
    license_text : str
        License text.
    min_sent_len : int, optional
        The minimum number of tokens in a sentence for it to be considered.
        The default is 3.
    summary_len : float, optional
        The proportion of length of the expected summary to the length of
        license text. The default is 0.3.
    summary_in_text_order : bool, optional
        Toggle to switch between summary in text order or in descending order
        by scores. The default is True.
    return_summary_only : bool, optional
        Toggle to return just the summary or entire license text with
        important sentences highlighted. The default is True.
    debug : bool, optional
        Toggles debug mode. The default is False.
    cleaned_license_sentences : list, optional
        A list of cleaned sentences. The default is None.

    Returns
    -------
    sent_scores : dict
        A dictionary of sentence scores with keys as tuples of sentence and
        sentence id and values as their normalized scores.
    cleaned_license_sentences : list
        A list of cleaned sentences.
    definitions : str
        Definitions extracted from license text.
    exceptions : str
        Exceptions extracted from license text.
    summary_len : float
        The proportion of length of the expected summary to the length of
        license text.

    """

    if not cleaned_license_sentences:
        cleaned_license_text, definitions, exceptions = clean_license_text(license_text)
        cleaned_license_sentences = sent_tokenize_text(cleaned_license_text, debug)
    else:
        definitions, exceptions = "", ""

    sent_scores = Counter()

    summary_len = math.ceil(summary_len * len(cleaned_license_sentences))

    if debug:
        print(f"summary length:{summary_len}")

    for sent_i, sent in enumerate(cleaned_license_sentences):

        if len(sent.split()) < min_sent_len:
            continue

        score = 0

        lemmatized_tokens = lemmatize_tokens(sent)

        if debug:
            print("-"*50)
            print(f"\nOriginal Sentence = {sent}")
            print(f"\n{sent_i}. Lemmatized_tokens = {lemmatized_tokens}")

        word_count = Counter([tok for tok in lemmatized_tokens])

        for prop, prop_words in vocab.properties_dict.items():
            prop_score = 0

            imp_words = list()

            for prop_word in prop_words:
                if prop_word in word_count.keys():
                    prop_score += vocab.properties_scores[prop] 
                    imp_words.append(prop_word)

            if debug:
                print(prop, "=", imp_words, "=", prop_score)

            score += prop_score

        # With normalization
        # sent_scores[(sent, sent_i)] = score  / len(lemmatized_tokens)

        # Without normalization
        sent_scores[(sent, sent_i)] = score

        if debug:
            print(f"Sentence score: {sent_scores[(sent, sent_i)]}")
            print()

    sent_scores = normalize_sentence_counter(sent_scores)

    if debug:
        print(sent_scores)

    return sent_scores, cleaned_license_sentences, definitions, exceptions, summary_len


def get_sent_scores(license_text,
                    min_sent_len=MIN_SENT_LEN,
                    summary_len=SUMMARY_LEN,
                    summary_in_text_order=True,
                    return_summary_only=True,
                    debug=False,
                    cleaned_license_sentences=None):
    """
    Get sentence scores for all the sentences in a given license_text along
    with their sentence ids.

    Parameters
    ----------
    license_text : str
        License text.
    min_sent_len : int, optional
        The minimum number of tokens in a sentence for it to be considered.
        The default is 3.
    summary_len : float, optional
        The proportion of length of the expected summary to the length of
        license text. The default is 0.3.
    summary_in_text_order : bool, optional
        Toggle to switch between summary in text order or in descending order
        by scores. The default is True.
    return_summary_only : bool, optional
        Toggle to return just the summary or entire license text with
        important sentences highlighted. The default is True.
    debug : bool, optional
        Toggles debug mode. The default is False.
    cleaned_license_sentences : list, optional
        A list of cleaned sentences. The default is None.

    Returns
    -------
    sent_id_scores : list(tuple)
        A list of tuples of sentence id and sentence score.

    """
    sent_scores, cleaned_license_sentences, definitions, exceptions, summary_len = get_license_summary_scores(
        license_text,
        min_sent_len=min_sent_len,
        summary_len=summary_len,
        summary_in_text_order=summary_in_text_order,
        return_summary_only=return_summary_only,
        debug=debug,
        cleaned_license_sentences=cleaned_license_sentences
    )

    sent_id_scores = [
        (sent_i, score) for (sent_id, sent_i), score in sent_scores.items()
    ]

    return sent_id_scores


def custom_textrank_summarizer(license_text,
                               min_sent_len=MIN_SENT_LEN,
                               summary_len=SUMMARY_LEN,
                               summary_in_text_order=True,
                               return_summary_only=True,
                               debug=False):
    """
    Returns summary / highlighted summary, definitions and exceptions for a
    given license_text.

    Parameters
    ----------
    license_text : str
        License text.
    min_sent_len : int, optional
        The minimum number of tokens in a sentence for it to be considered.
        The default is 3.
    summary_len : float, optional
        The proportion of length of the expected summary to the length of
        license text. The default is 0.3.
    summary_in_text_order : bool, optional
        Toggle to switch between summary in text order or in descending order
        by scores. The default is True.
    return_summary_only : bool, optional
        Toggle to return just the summary or entire license text with
        important sentences highlighted. The default is True.
    debug : bool, optional
        Toggles debug mode. The default is False.

    Returns
    -------
    str
        Summary or the highlighted license text.
    definitions : str
        Definitions extracted from license text.
    exceptions : str
        Exceptions extracted from license text.

    """

    sent_scores, cleaned_license_sentences, definitions, exceptions, summary_len = get_license_summary_scores(
        license_text,
        min_sent_len=min_sent_len,
        summary_len=summary_len,
        summary_in_text_order=summary_in_text_order,
        return_summary_only=return_summary_only,
        debug=debug
    )

    sorted_sent_scores = sent_scores.most_common()[:summary_len]

    if summary_in_text_order:
        sentences_in_text_order = sorted(sorted_sent_scores, key=lambda x: x[0][1])
        summary = "".join(sent.strip(". ") for (sent, sent_i), score in sentences_in_text_order)
        selected_sent_ids = set(sent_i for (_, sent_i), score in sentences_in_text_order)
    else:
        summary = "".join(sent.strip(". ") for (sent, sent_i), score in sorted_sent_scores)
        selected_sent_ids = set(sent_i for (_, sent_i), score in sorted_sent_scores)

    highlighted_license_text = " ".join(
        f"""<mark style="color: {color.BLACK}; background-color:{color.GREEN}">{sent}</mark>"""
        if sent_i in selected_sent_ids
        else sent
        for sent_i, sent in enumerate(cleaned_license_sentences)
    )

    if debug:
        print("="*50)
        print("License Text:")
        print("-"*30)
        print(highlighted_license_text)
        print("="*50)

    definitions = definitions.strip("\n.") + "."

    if return_summary_only:
        return summary, definitions, exceptions
    else:
        return highlighted_license_text, definitions, exceptions


def get_system_scores(attachment_id=None):
    """
    Get system sentence scores for all the sentences in all licenses in gold
    standard.

    Parameters
    ----------
    attachment_id : str, optional
        The attachment id of the document for which the sentence scores are to
        be calculated. If None, the sentence scores for all the documents will
        be returned. The default is None.

    Returns
    -------
    scores_dict : dict
        A dictionary of all the scores with keys as the attachment id of a
        document and values as a list of tuples of sentence id and scores for
        that attachment id.

    """
    gold_data = pd.read_excel(GOLD_STANDARD_PATH)
    gold_data = gold_data[["attachment_id", "sentence"]]
    sent_lists = gold_data.groupby("attachment_id")["sentence"].apply(list)

    scores_dict = dict()

    if attachment_id:
        scores_dict[attachment_id] = get_sent_scores(
            "",
            summary_len=SUMMARY_LEN,
            cleaned_license_sentences=sent_lists[attachment_id]
        )
        return scores_dict

    for attachment_id, cleaned_license_sentences in dict(sent_lists).items():
        
        scores_dict[attachment_id] = get_sent_scores(
            "",
            summary_len=SUMMARY_LEN,
            cleaned_license_sentences=cleaned_license_sentences
        )

    return scores_dict


def preprocess_properties(cell):
    """
    Converts licnse properties to title case and removes hyphens and
    underscores.

    Parameters
    ----------
    cell : str
        A cell string in properties dataframe of a license.

    Returns
    -------
    cell : TYPE
        DESCRIPTION.

    """
    try:
        cell = cell.replace("--", "$")
        cell = cell.replace("-", " ")
        cell = cell.replace("_", " ")
        cell = cell.replace("$", " - ").title()
    except:
        pass
    return cell

def get_labels_for_license(license_id, by_license_id=True):
    """
    Gets license properties for a given license_id.

    Parameters
    ----------
    license_id : str
        License id of the license for which properties are to be returned.
    by_license_id : bool, optional
        A flag to decide whether we fetch the license properties by license id
        or license name. The default is True.

    Returns
    -------
    properties : pandas.DataFrame
        Dataframe with properties of the license with id license_id.

    """
    index_col = 0 if by_license_id else 1
    columns = ["Property", "Label"]
    labels_data = pd.read_csv(LABELS_PATH, index_col=index_col)
    properties = pd.DataFrame(labels_data.loc[license_id]).reset_index()
    properties.columns = columns
    properties = properties.applymap(preprocess_properties)
    return properties