updating src module imports with try-except clause

src/clean.py

@@ -2,12 +2,62 @@ import re
 import json
 from bs4 import BeautifulSoup
 from striprtf.striprtf import rtf_to_text
+from collections import defaultdict
 
 
 PARA_BREAK = "para___break"
+seperator = "=" * 50
+verbosity = 0
+
+
+def extract_author_details(text, verbosity=0):
+    """
+    Extracts important author information from the license text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    text : str
+        License text with author details removed.
+    author_details : list
+        A list of important author details.
+
+    """
+    author_details_pattern = r"(@(author|license|copyright|package).*)"
+    author_details = list()
+    text = re.sub(author_details_pattern, lambda m: author_details.append(m.group(1)), text)
+    if author_details and verbosity != 0:
+        print(seperator)
+        print(seperator)
+        print("Following author details were extracted:")
+        print(seperator)
+        print(author_details)
+        print()
+        
+    return text, author_details
 
 
 def php_cleaner(text):
+    """
+    Cleans the license file in PHP format.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with PHP script removed.
+
+    """
     try:
         return re.findall("\/\*[\S\s]*?\*\/", text)[0]
     except:
@@ -16,6 +66,20 @@ def php_cleaner(text):
 
 
 def html_cleaner(text):
+    """
+    Cleans the license file in HTML format.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with HTML script removed.
+
+    """
     soup = BeautifulSoup(text)
     text = soup.body.text
     if not text:
@@ -24,6 +88,20 @@ def html_cleaner(text):
 
 
 def json_cleaner(text_dict):
+    """
+    Cleans the license file in JSON format.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with JSON format normalized to text.
+
+    """
     out = ""
     for key in text_dict.keys():
         if key in ("description", "license"):
@@ -34,60 +112,101 @@ def json_cleaner(text_dict):
     return out
 
 
-def discard_text_after_tnc(text):
-    return text.split("END OF TERMS AND CONDITIONS")[0]
+def rtf_cleaner(text):
+    """
+    Cleans the license file in RTF format.
 
+    Parameters
+    ----------
+    text : str
+        Raw License text.
 
-def gnu_cleaner(text):
-    t = text.split('END OF TERMS AND CONDITIONS')[0]
-    definitions = ""
-    if 'Preamble' in text:
-        if len(t.split('Preamble')[0])>100:
-            t0 = t.split('Preamble')[0]
-            try:
-                t1 = t.split('Preamble')[1].split('distribution and\nmodification follow')[1]
-            except:
-                try:
-                    t1 = t.split('Preamble')[1].split('distribution and\n\nmodification follow')[1]
-                except:
-                    t1 = t.split('Preamble')[1].split('distribution and modification follow')[1]
-            t = t0+t1
-        else: 
-            t = t.split('Preamble')[1].split('distribution and\nmodification follow')[1]
-    if 'Definitions' in text:
-        try:
-            def_pos = re.search(r"[0-9]\.? (Additional )?Definitions",t).span()
-            other_start_pos = re.search(r"[0-9]\.? [A-Z][a-z]+",t[def_pos[1]:]).span()[0]
-            definitions = t[def_pos[0]: def_pos[1] + other_start_pos]
-            t = t[:def_pos[0]] + t[def_pos[1]+other_start_pos:]
-        except:
-            t = t
-    return t, definitions
-
+    Returns
+    -------
+    str
+        Cleaned License text with RTF script removed.
 
-def rtf_cleaner(text):
+    """
     return rtf_to_text(text)
 
 
 def url_cleaner(text):
-    return re.sub(r"http\S+", "", text)
+    """
+    Removes URLs from the License text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with URLs removed.
+
+    """
+    return re.sub(r"\(?http\S+\)?", "", text)
 
 
 def email_cleaner(text):
-    return re.sub(r"\S*@\S*", "", text)
+    """
+    Removes emails from the License text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with emails removed.
+
+    """
+    return re.sub(r"\S{3,}@\S{2,}\.\S+", "", text)
 
 
 def var_cleaner(text):
+    """
+    Removes potential variable names from the License text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with variable names removed.
+
+    """
     text = re.sub(r"\$\w+", "", text)
     text = re.sub(r"{[{}()\w\s._,\[\]'\"]+}", "", text)
+    # text = re.sub(r"[a-zA-Z\(\)_'\"]+\.[a-zA-Z_]+", "", text)
     return text
 
 
 def character_cleaner(text):
+    """
+    Removes unnecessary special characters from the License text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with some special characters removed.
+
+    """
     text = url_cleaner(text)
     text = email_cleaner(text)
     text = var_cleaner(text)
 
+    text = re.sub("\s*(;quot;|&amp)\s*", " ", text)
     text = re.sub("[\n]{2,}", ". ", text)
     text = re.sub("[:%#<>=*\-/·\s{}]+", " ", text)
     text = re.sub("[\. ]{2,}", ". ", text)
@@ -95,6 +214,20 @@ def character_cleaner(text):
 
 
 def isEnglish(s):
+    """
+    Checks whether the License text is in English or not.
+
+    Parameters
+    ----------
+    s : str
+        Raw License text.
+
+    Returns
+    -------
+    bool
+        True if complete License text is in English, False otherwise.
+
+    """
     try:
         s.encode(encoding="utf-8").decode("ascii")
     except UnicodeDecodeError:
@@ -103,15 +236,227 @@ def isEnglish(s):
         return True
 
 
-def preprocess_text(text):
+def split_definitions_exceptions(text, verbosity=0):
+    """
+    Extract definitions from the License text
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    paras : list
+        A list of paragraphs from License text with definitions and exceptions
+        removed.
+    definitions : str
+        Definitions extracted from the License text.
+    exceptions : list
+        A list of paragraphs which contain exceptions .
+
+    """
     definitions = ""
+
+    if "Definitions" in text:
+        try:
+            def_pattern = r"([S|s]ection )?[0-9] ?[\.|-|–]? ?([A|a]dditional )?[D|d]efinitions"
+            after_def_pattern = r"\s+(Section )?[0-9]\.? [\.|-|–]? ?[A-Z][a-z]+"
+            def_pos = re.search(def_pattern, text).span()
+            other_start_pos = re.search(after_def_pattern, text[def_pos[1]:]).span()[0]
+            definitions = text[def_pos[0]: def_pos[1] + other_start_pos].strip() + "\n\n"
+            text = text[:def_pos[0]] + text[def_pos[1] + other_start_pos:]
+        except:
+            pass
+
+    paras, more_defs = extract_relevant_paras(
+        split_paras(text, verbosity=verbosity),
+        verbosity=verbosity
+    )
+
+    definitions += more_defs.strip()
+    paras, exceptions = get_exeptions(paras, verbosity=verbosity)
+
+    return paras, definitions, exceptions
+
+
+def discard_text_after_end_tnc(text):
+    """
+    Discards text after "END OF TERMS AND CONDITIONS"
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        License text with irrelavant information after "END OF TERMS AND CONDITIONS" removed.
+
+    """
+    return text.split("END OF TERMS AND CONDITIONS")[0]
+
+
+def clear_preamble(text):
+    """
+    Cleans Preamble from the License text
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    text : str
+        License text with Preamble removed.
+
+    """
+    preamble_pattern = "Preamble"
+    dist_and_mod_pattern = "distribution\s+and\s+modification\s+follow\.?"
+
+    if preamble_pattern in text:
+        preamble_split = text.split(preamble_pattern)
+
+        if len(preamble_split) != 2:
+            return text
+
+        try:
+            after_preamble_end = re.split(dist_and_mod_pattern, preamble_split[1])[1]
+
+            # TODO Why do we need this condition?
+            if len(preamble_split[0]) > 100:
+                text = preamble_split[0] + after_preamble_end.strip()
+        except:
+            pass
+    return text
+
+
+def gnu_cleaner(text):
+    """
+    Cleans GNU text such as discarding Preamble and text after end of terms
+    and conditions.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    preamble_cleared_text : str
+        License text with irrelavant information in Preamble and text after end
+        of terms and conditions removed.
+
+    """
+
+    before_end_tnc = discard_text_after_end_tnc(text)
+    preamble_cleared_text = clear_preamble(before_end_tnc)
+
+    return preamble_cleared_text
+
+
+def preprocess_text(text):
+    """
+    Preprocesses License text considering different License types.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    text : str
+        Cleaned License text.
+
+    """
+
+    # if most_likely_license_type in [
+    #     "GPL-3.0-only",
+    #     "AGPL-3.0-only",
+    #     "GPL-2.0-only",
+    #     "LGPL-3.0-only",
+    #     "LGPL-2.1-only",
+    # ]:
+
+    # # We need to take care of these cases too: 
+        # # https://choosealicense.com/licenses/ofl-1.1/
+        # # https://choosealicense.com/licenses/lodbl-1.0/
+        # # https://choosealicense.com/licenses/odbl-1.0/
+        # # https://choosealicense.com/licenses/lms-rl/
+        # # https://choosealicense.com/licenses/lms-pl/
+        # # https://choosealicense.com/licenses/lmpl-2.0/
+        # # https://choosealicense.com/licenses/lppl-1.3c/
+        # # https://choosealicense.com/licenses/eupl-1.2/
+        # # https://choosealicense.com/licenses/eupl-1.1/
+        # # https://choosealicense.com/licenses/epl-2.0/
+        # # https://choosealicense.com/licenses/epl-1.0/
+        # # https://choosealicense.com/licenses/ecl-2.0/
+        # # https://choosealicense.com/licenses/cecill-2.1/
+        # # https://choosealicense.com/licenses/cc-by-sa-4.0/
+        # # https://choosealicense.com/licenses/cc-by-4.0/
+        # # https://choosealicense.com/licenses/artistic-2.0/
+        # # https://choosealicense.com/licenses/apache-2.0/
+
+    # TODO This condition will not work, fix it:
     if "GNU" in text or "Apache" in text:
-        text, definitions = gnu_cleaner(text)
-        definitions = definitions.strip()
-    return text, definitions
+        text = gnu_cleaner(text)
+    return text
+
+
+def clean_if_else(text):
+    """
+    Removes specific if-else conditions from the License text
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with if-else conditions removed.
+
+    """
+    return re.sub(r"#\bif[\s\S]+?#endif\s*", "", text).strip()
+
+
+def clean_comments(text):
+    """
+    Cleans specific comment formats from the License texts
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text with comments conditions removed.
+
+    """
+    return re.sub(r"[\`'\"]{3,}[\s\S]*?[\`'\"]{3,}", "", text).strip()
 
 
 def script_cleaner(text):
+    """
+    Cleans the script text from License text to extract the main content.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+
+    Returns
+    -------
+    str
+        Cleaned License text without scripts.
+
+    """
     if "<?php" in text:
         text = php_cleaner(text)
     elif "</html>" in text:
@@ -122,50 +467,300 @@ def script_cleaner(text):
         text = rtf_cleaner(text)
     if not text:
         return ""
+    text = clean_if_else(text)
+    text = clean_comments(text)
     return text
 
 
-def split_paras(text):
-    if "\n\n\n\n" in text:
+def split_paras(text, verbosity=0):
+    """
+    Splits the text into paragraphs.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    paras : list
+        A list of split paragraphs.
+
+    """
+    text = re.sub(r"\n{4,}", "\n"*4, text)
+    if len(re.findall("\n\n\n\n", text)) >= 2:
         paras = text.split("\n\n\n\n")
-    elif "\n\n\n" in text:
-        paras = text.split("\n\n\n")
-    elif "\n\n" in text:
+        paras = [re.sub(r"\n{1,3}", " ", para) for para in paras]
+    elif len(re.findall("\n\n", text)) >= 2:
         paras = text.split("\n\n")
+        paras = [re.sub(r"\n", " ", para) for para in paras]
+    elif len(re.findall("\n", text)) >= 2:
+        paras = text.split("\n")
     else:
         paras = [text]
+    if verbosity != 0:
+        print(seperator)
+        print(seperator)
+        print("These are the split paras in the text:")
+        for para in paras:
+            if not para.strip():
+                continue
+            print(seperator)
+            print(para)
+        print()
     return paras
 
 
-def clean_paras(paras):
-    return paras
+def extract_relevant_paras(paras, verbosity=0):
+    """
+    Extracts relevant paragraphs from the list of all paragraphs.
+
+    Parameters
+    ----------
+    paras : list
+        A list of split paragraphs.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    cleaned_paras : list
+        A list of relevant paragraphs.
+    definitions : str
+        Definition text as extracted by the "clean_definitions_pattern", which
+        is to be appended to other definitons in the License text if any.
 
+    """
+    cleaned_paras = list()
+    definitions = ""
+
+    # TODO This might be interesting to look into: 
+    # https://choosealicense.com/licenses/eupl-1.2/
+
+    clean_definitions_pattern = r"""\".{0,20}\".{0,40}(mean|include|refer)s?"""
+
+    if verbosity != 0:
+        print(seperator)
+        print(seperator)
+        print("Following paragraphs were considered unnecessary and removed:")
+    for para in paras:
+        if not para.strip():
+            continue
+        if re.search(clean_definitions_pattern, para):
+            definitions += para + "\n\n"
+            if verbosity != 0:
+                print(seperator)
+                print(para)
+        else:
+            cleaned_paras.append(para)
+    if verbosity != 0:
+        print()
+
+    definitions = definitions.strip()
+
+    return cleaned_paras, definitions
+
+
+def get_all_caps(text, verbosity=0):
+    """
+    Extracts text with all caps content from the License text.
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    text : str
+        License text with all caps sentences removed.
+    all_caps : list
+        A list of all caps sentences from the License text.
+
+    """
+    all_caps_pattern = r"([^a-z\n]{50,})"
+    all_caps = list()
+    text = re.sub(all_caps_pattern, lambda m: all_caps.append(m.group(1)), text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    if all_caps and verbosity != 0:
+        print(seperator)
+        print(seperator)
+        print("Following all caps were removed from the text:")
+        print(all_caps)
+        print()
+    return text, all_caps
+
+
+def get_exeptions(paras, verbosity=0):
+    """
+    Extracts a list of exceptions from the License text.
+
+    Parameters
+    ----------
+    paras : list
+        A list of paragraphs from the License text.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    non_exception_paras : list
+        A list of all paragraphs not containing exceptions from the License text.
+    exceptions : list
+        A list of all paragraphs containing exceptions from the License text.
+
+    """
+    non_exception_paras = list()
+    exceptions = list()
+
+    for para in paras:
+        if re.search("exception", para.lower()):
+            exceptions.append(para)
+        else:
+            non_exception_paras.append(para)
+
+    if exceptions and verbosity != 0:
+        print(seperator)
+        print(seperator)
+        print("Following exceptions were found in the text:")
+        for exception in exceptions:
+            print(seperator)
+            print(exception)
+        print()
+
+    return non_exception_paras, exceptions
+
+
+def get_MIT_content(text):
+    """
+    Returns the content of the MIT-like-licenses segregated into categories like
+    Copyright, main content, etc.
+
+    Parameters
+    ----------
+    text : str
+        Cleaned MIT License text.
+    
+    Returns
+    -------
+    dictionary
+        A dictionary of content from the MIT license. Keys are the type of
+        content and values are the License contents from License text.
+    """
+    paras = split_paras(text)
+
+    mit_content = defaultdict(list)
+
+    for para in paras:
+        para = para.strip()
+        if len(para) < 1:
+            continue
+        if len(para.split()) <= 10 and ("Licens" in para or "licens" in para) and "Copyright" not in para:
+            mit_content["header"].append(para)
+        elif "Copyright" in para:
+            if "is hereby granted" in para:
+                mit_content["copyright+content"].append(para)
+            else:
+                mit_content["copyright"].append(para)
+        elif "Permission is hereby granted" in para:
+            mit_content["content"].append(para)
+        elif "The above copyright notice" in para or len(para.split()) < 18:
+            mit_content["sentence"].append(para)
+        elif get_all_caps(para)[1]:
+            mit_content["all_cap"].append(para)
+        else:
+            mit_content["content"].append(para)
+
+    for key, value in mit_content.items():
+        mit_content[key] = "\n\n".join(value)
+
+    return mit_content
+
+
+def get_most_likely_license_type(text):
+    """
+    Returns the most likely license type based on Doc2Vec scores
+    (similarity > 0.9).
+
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+    
+    Returns
+    -------
+    str
+        The type of the most likely license. "Not found" if no license score is
+        above 0.9
+    """
+    from src.doc2vec import inference
+
+    top1_result = inference(text).loc[0, :]
+
+    if top1_result["Scores"] > 0.9:
+        return top1_result["License"]
+    else:
+        return "Not Found"
+
+
+def clean_license_text(text, verbosity=0):
+    """
+    Cleans License text.
 
-def clean_license_text(text):
+    Parameters
+    ----------
+    text : str
+        Raw License text.
+    verbosity : int, optional
+        The level of print statements on the output console. The default is 0.
+
+    Returns
+    -------
+    text : str
+        Cleaned License text.
+    definitions : str
+        Definitions extracted from the License text.
+
+    """
 
     if len(text) == 0:
         return text
 
+    most_likely_license_type = get_most_likely_license_type(text)
+
+    text, author_details = extract_author_details(text, verbosity=verbosity)
     text = script_cleaner(text)
-    text, definitions = preprocess_text(text)
-    paras = clean_paras(split_paras(text))
+    text = preprocess_text(text)
+    paras, definitions, exceptions = split_definitions_exceptions(
+        text, verbosity=verbosity
+    )
     text = PARA_BREAK.join(paras)
     text = character_cleaner(text)
     text = re.sub(PARA_BREAK, "\n\n", text)
     text = text.strip()
 
+    # TODO Need to update this too:
     if not isEnglish(text):
         if not isEnglish(" ".join(text.split()[-5:-1])):
             return "", ""
-    
+
+    if "MIT" in most_likely_license_type:
+        mit_content = get_MIT_content(text)
+        if verbosity != 0:
+            print("This is likely an MIT License!")
+            print(mit_content)
+
     return text, definitions
 
 
 """
 Notes: 
     
-    1. Regex for other definitions: -------->  ".{0,20}".{0,40}means
-    2. Try splitting each para by "\n", if len == 1 and len(para) < 100 (or something)
+    1. Try splitting each para by "\n", if len == 1 and len(para) < 100 (or something)
        -> Merge with the next para
        Ex. "8. Termination."
 """

src/doc2vec.py

@@ -4,7 +4,10 @@ from gensim.models.doc2vec import Doc2Vec, TaggedDocument
 import pandas as pd
 import json
 
-from clean import preprocess_text, script_cleaner
+try:
+    from src.clean import preprocess_text, script_cleaner
+except:
+    from clean import preprocess_text, script_cleaner
 
 MODEL_PATH = 'models/d2v.model'
 LICENSE_INDEX_PATH = 'data/index_license_map.json'