Custom textrank, changes to UI

app.py

@@ -1,13 +1,13 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
+import os
 import nltk 
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 import torch
-import os
+import streamlit as st
 
 from src.doc2vec import inference
 from src.abstractive_sum import summarize_text_with_model
+from src.textrank import custom_textrank_summarizer
+from src.clean import clean_license_text
 
 CUSTOM_MODEL_NAME = "utkarshsaboo45/ClearlyDefinedLicenseSummarizer"
 
@@ -19,12 +19,37 @@ with st.spinner('Loading...'):
     model = AutoModelForSeq2SeqLM.from_pretrained(CUSTOM_MODEL_NAME).to(device)
     tokenizer = AutoTokenizer.from_pretrained(CUSTOM_MODEL_NAME)
 
+summarization_type = st.sidebar.selectbox(
+    "Select summarization type.",
+    ("Abstractive", "Extractive", "Both")
+)
+if summarization_type == 'Abstractive':
+    st.sidebar.caption('Summary will be generated by the T5 Transformer Model')
+elif summarization_type == 'Extractive':
+    st.sidebar.caption('Summary will be generated by a custom TextRank Algorithm')
+    summary_len = st.sidebar.slider('Summary length percentage', 1, 10, 3)
+elif summarization_type == 'Both':
+    st.sidebar.caption('The License text will be first passed through the custom TextRank algorithm and then passed on to the T5 Transformer Model to generate a summary.')
+
+clean_text = st.sidebar.checkbox('Show cleaned license text')
+
 st.title('Clearly Defined: License Summarizer')
 input = st.text_area('Enter contents of the license')
 
 if len(input) > 0:
     with st.spinner('Loading...'):
-        summary = summarize_text_with_model(input, model, tokenizer)
+        if summarization_type == 'Abstractive':
+            summary, definitions = summarize_text_with_model(input, model, tokenizer)
+        if summarization_type == 'Extractive':
+                summary, definitions = custom_textrank_summarizer(input, summary_len = summary_len/10)
+        if summarization_type == 'Both':
+            summary, definitions = summarize_text_with_model(input, model, tokenizer)
+            summary, _ = custom_textrank_summarizer(summary, summary_len = 1)
+        
+        if clean_text:
+            st.header('Cleaned License Text')
+            st.write(clean_license_text(input)[0])
+
         st.header('Summary')
         st.write(summary)
         
@@ -32,3 +57,7 @@ if len(input) > 0:
         st.header('Similarity Index')
         st.dataframe(prediction_scores)
 
+        if definitions:
+            st.header('Definitions')
+            st.write(definitions) 
+

data/choosealicense_appendix_labels.csv

@@ -1,42 +1,42 @@
-spdx_id,license_name,commercial-use,disclose-source,distribution,document-changes,include-copyright,include-copyright--source,liability,modifications,network-use-disclose,patent-use,private-use,same-license,same-license--file,same-license--library,trademark-use,warranty,GTLC_Permissive,GTLC_Notes
-0bsd,BSD Zero Clause License,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-afl-3.0,Academic Free License v3.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations,permissive,
-agpl-3.0,GNU Affero General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,,limitations,not_permissive,
-apache-2.0,Apache License 2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations,permissive,
-artistic-2.0,Artistic License 2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations,not_permissive,
-bsd-2-clause,BSD 2-Clause Simplified License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-bsd-3-clause,BSD 3-Clause New or Revised License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-bsd-3-clause-clear,BSD 3-Clause Clear License,permissions,,permissions,,conditions,,limitations,permissions,,limitations,permissions,,,,,limitations,permissive,
-bsd-4-clause,BSD 4-Clause Original or Old License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-bsl-1.0,Boost Software License 1.0,permissions,,permissions,,,conditions,limitations,permissions,,,permissions,,,,,limitations,permissive,
-cc-by-4.0,Creative Commons Attribution 4.0 International,permissions,,permissions,conditions,conditions,,limitations,permissions,,limitations,permissions,,,,limitations,limitations,permissive,
-cc-by-sa-4.0,Creative Commons Attribution Share Alike 4.0 International,permissions,,permissions,conditions,conditions,,limitations,permissions,,limitations,permissions,conditions,,,limitations,limitations,not_permissive,
-cc0-1.0,Creative Commons Zero v1.0 Universal,permissions,,permissions,,,,limitations,permissions,,limitations,permissions,,,,limitations,limitations,permissive,
-cecill-2.1,CeCILL Free Software License Agreement v2.1,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations,not_permissive,
-ecl-2.0,Educational Community License v2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations,,unfamiliar
-epl-1.0,Eclipse Public License 1.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations,not_permissive,weak copyleft
-epl-2.0,Eclipse Public License 2.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations,not_permissive,weak copyleft
-eupl-1.1,European Union Public License 1.1,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations,not_permissive,
-eupl-1.2,European Union Public License 1.2,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations,not_permissive,
-gpl-2.0,GNU General Public License v2.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,conditions,,,,limitations,not_permissive,
-gpl-3.0,GNU General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations,not_permissive,
-isc,ISC License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-lgpl-2.1,GNU Lesser General Public License v2.1,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,,,conditions,,limitations,not_permissive,
-lgpl-3.0,GNU Lesser General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,conditions,,limitations,not_permissive,
-lppl-1.3c,LaTeX Project Public License v1.3c,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,,,,,limitations,,unfamiliar
-mit,MIT License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-mit-0,MIT No Attribution,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-mpl-2.0,Mozilla Public License 2.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,,conditions,,limitations,limitations,not_permissive,weak copyleft
-ms-pl,Microsoft Public License,permissions,,permissions,,conditions,,,permissions,,permissions,permissions,,,,limitations,limitations,not_permissive,
-ms-rl,Microsoft Reciprocal License,permissions,conditions,permissions,,conditions,,,permissions,,permissions,permissions,,conditions,,limitations,limitations,not_permissive,
-mulanpsl-2.0,"Mulan Permissive Software License, Version 2",permissions,,permissions,,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations,,unfamiliar
-ncsa,University of IllinoisNCSA Open Source License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-odbl-1.0,Open Data Commons Open Database License v1.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,limitations,permissions,conditions,,,limitations,limitations,,unfamiliar
-ofl-1.1,SIL Open Font License 1.1,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,conditions,,,,limitations,not_permissive,
-osl-3.0,Open Software License 3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations,not_permissive,
-postgresql,PostgreSQL License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-unlicense,The Unlicense,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations,permissive,
-upl-1.0,Universal Permissive License v1.0,permissions,,permissions,,conditions,,limitations,permissions,,permissions,permissions,,,,,limitations,permissive,
-vim,Vim License,permissions,conditions,permissions,conditions,conditions,,,permissions,,,permissions,conditions,,,,,not_permissive,
-wtfpl,Do What The Fck You Want To Public License,permissions,,permissions,,,,,permissions,,,permissions,,,,,,,unfamiliar
-zlib,zlib License,permissions,,permissions,conditions,,conditions,limitations,permissions,,,permissions,,,,,limitations,permissive,
+spdx_id,license_name,commercial-use,disclose-source,distribution,document-changes,include-copyright,include-copyright--source,liability,modifications,network-use-disclose,patent-use,private-use,same-license,same-license--file,same-license--library,trademark-use,warranty
+0bsd,BSD Zero Clause License,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations
+afl-3.0,Academic Free License v3.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations
+agpl-3.0,GNU Affero General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,,limitations
+apache-2.0,Apache License 2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations
+artistic-2.0,Artistic License 2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations
+bsd-2-clause,BSD 2-Clause Simplified License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+bsd-3-clause,BSD 3-Clause New or Revised License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+bsd-3-clause-clear,BSD 3-Clause Clear License,permissions,,permissions,,conditions,,limitations,permissions,,limitations,permissions,,,,,limitations
+bsd-4-clause,BSD 4-Clause Original or Old License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+bsl-1.0,Boost Software License 1.0,permissions,,permissions,,,conditions,limitations,permissions,,,permissions,,,,,limitations
+cc-by-4.0,Creative Commons Attribution 4.0 International,permissions,,permissions,conditions,conditions,,limitations,permissions,,limitations,permissions,,,,limitations,limitations
+cc-by-sa-4.0,Creative Commons Attribution Share Alike 4.0 International,permissions,,permissions,conditions,conditions,,limitations,permissions,,limitations,permissions,conditions,,,limitations,limitations
+cc0-1.0,Creative Commons Zero v1.0 Universal,permissions,,permissions,,,,limitations,permissions,,limitations,permissions,,,,limitations,limitations
+cecill-2.1,CeCILL Free Software License Agreement v2.1,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations
+ecl-2.0,Educational Community License v2.0,permissions,,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations
+epl-1.0,Eclipse Public License 1.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations
+epl-2.0,Eclipse Public License 2.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations
+eupl-1.1,European Union Public License 1.1,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations
+eupl-1.2,European Union Public License 1.2,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations
+gpl-2.0,GNU General Public License v2.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,conditions,,,,limitations
+gpl-3.0,GNU General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,conditions,,,,limitations
+isc,ISC License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+lgpl-2.1,GNU Lesser General Public License v2.1,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,,,conditions,,limitations
+lgpl-3.0,GNU Lesser General Public License v3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,permissions,permissions,,,conditions,,limitations
+lppl-1.3c,LaTeX Project Public License v1.3c,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,,,permissions,,,,,limitations
+mit,MIT License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+mit-0,MIT No Attribution,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations
+mpl-2.0,Mozilla Public License 2.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,permissions,permissions,,conditions,,limitations,limitations
+ms-pl,Microsoft Public License,permissions,,permissions,,conditions,,,permissions,,permissions,permissions,,,,limitations,limitations
+ms-rl,Microsoft Reciprocal License,permissions,conditions,permissions,,conditions,,,permissions,,permissions,permissions,,conditions,,limitations,limitations
+mulanpsl-2.0,"Mulan Permissive Software License, Version 2",permissions,,permissions,,conditions,,limitations,permissions,,permissions,permissions,,,,limitations,limitations
+ncsa,University of IllinoisNCSA Open Source License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+odbl-1.0,Open Data Commons Open Database License v1.0,permissions,conditions,permissions,,conditions,,limitations,permissions,,limitations,permissions,conditions,,,limitations,limitations
+ofl-1.1,SIL Open Font License 1.1,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,conditions,,,,limitations
+osl-3.0,Open Software License 3.0,permissions,conditions,permissions,conditions,conditions,,limitations,permissions,conditions,permissions,permissions,conditions,,,limitations,limitations
+postgresql,PostgreSQL License,permissions,,permissions,,conditions,,limitations,permissions,,,permissions,,,,,limitations
+unlicense,The Unlicense,permissions,,permissions,,,,limitations,permissions,,,permissions,,,,,limitations
+upl-1.0,Universal Permissive License v1.0,permissions,,permissions,,conditions,,limitations,permissions,,permissions,permissions,,,,,limitations
+vim,Vim License,permissions,conditions,permissions,conditions,conditions,,,permissions,,,permissions,conditions,,,,
+wtfpl,Do What The Fck You Want To Public License,permissions,,permissions,,,,,permissions,,,permissions,,,,,
+zlib,zlib License,permissions,,permissions,conditions,,conditions,limitations,permissions,,,permissions,,,,,limitations

requirements.txt

@@ -10,15 +10,19 @@ base58==2.1.1
 beautifulsoup4==4.11.1
 bleach==5.0.0
 blinker==1.4
+blis==0.7.7
 cachetools==5.1.0
+catalogue==2.0.7
 certifi==2021.10.8
 cffi==1.15.0
 charset-normalizer==2.0.12
 click==8.0.4
 cycler==0.11.0
+cymem==2.0.6
 debugpy==1.6.0
 decorator==5.1.1
 defusedxml==0.7.1
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl
 entrypoints==0.4
 executing==0.8.3
 fastjsonschema==2.15.3
@@ -43,10 +47,12 @@ jupyter-core==4.10.0
 jupyterlab-pygments==0.2.2
 jupyterlab-widgets==1.1.0
 kiwisolver==1.4.2
+langcodes==3.3.0
 MarkupSafe==2.1.1
 matplotlib==3.5.0
 matplotlib-inline==0.1.3
 mistune==0.8.4
+murmurhash==1.0.7
 nbclient==0.6.3
 nbconvert==6.5.0
 nbformat==5.4.0
@@ -58,10 +64,12 @@ packaging==21.3
 pandas==1.3.4
 pandocfilters==1.5.0
 parso==0.8.3
+pathy==0.6.1
 pexpect==4.8.0
 pickleshare==0.7.5
 Pillow==9.1.1
 pip==22.1
+preshed==3.0.6
 prometheus-client==0.14.1
 prompt-toolkit==3.0.29
 protobuf==3.20.1
@@ -70,6 +78,7 @@ ptyprocess==0.7.0
 pure-eval==0.2.2
 pyarrow==8.0.0
 pycparser==2.21
+pydantic==1.8.2
 pydeck==0.7.1
 Pygments==2.12.0
 Pympler==1.0.1
@@ -90,13 +99,18 @@ setuptools==62.3.1
 setuptools-scm==6.4.2
 six==1.16.0
 sklearn==0.0
-smart-open==6.0.0
+smart-open==5.2.1
 smmap==5.0.0
 soupsieve==2.3.2.post1
+spacy==3.3.0
+spacy-legacy==3.0.9
+spacy-loggers==1.0.2
+srsly==2.4.3
 stack-data==0.2.0
 streamlit==1.9.0
 striprtf==0.0.20
 terminado==0.15.0
+thinc==8.0.16
 threadpoolctl==3.1.0
 tinycss2==1.1.1
 tokenizers==0.12.1
@@ -108,11 +122,13 @@ tornado==6.1
 tqdm==4.64.0
 traitlets==5.2.1.post0
 transformers==4.19.2
+typer==0.4.1
 typing_extensions==4.2.0
 tzdata==2022.1
 tzlocal==4.2
 urllib3==1.26.9
 validators==0.19.0
+wasabi==0.9.1
 watchdog==2.1.8
 wcwidth==0.2.5
 webencodings==0.5.1

src/abstractive_sum.py

@@ -12,9 +12,8 @@ import pandas as pd
 import torch
 from torch.utils.data import Dataset, DataLoader
 from sklearn.model_selection import train_test_split
+from src.clean import clean_license_text
 
-
-import torch.nn as nn
 from tqdm.auto import tqdm
 
 from transformers import (
@@ -32,7 +31,7 @@ MODEL_PATH = "models/"
 MODEL_FILENAME = "t5-base.model"
 
 MODEL_NAME = "t5-base"
-# TOKENIZER = T5Tokenizer.from_pretrained(MODEL_NAME)
+TOKENIZER = None
 
 TEXT_MAX_TOKEN_LEN = 512
 SUMMARY_MAX_TOKEN_LEN = 128
@@ -56,13 +55,14 @@ class LicenseSummaryDataset(Dataset):
         self.text_max_token_len = text_max_token_len
         self.summary_max_token_len = summary_max_token_len
 
+
     def __len__(self):
         return len(self.data)
 
     def __getitem__(self, index: int):
         data_row = self.data.iloc[index]
         text = data_row["text"]
-        text_encoding = TOKENIZER(
+        text_encoding = self.tokenizer(
             text,
             max_length=self.text_max_token_len,
             padding="max_length",
@@ -72,7 +72,7 @@ class LicenseSummaryDataset(Dataset):
             return_tensors="pt"
         )
 
-        summary_encoding = TOKENIZER(
+        summary_encoding = self.tokenizer(
             data_row["summary"],
             max_length=self.summary_max_token_len,
             padding="max_length",
@@ -111,6 +111,8 @@ def prepare_dataloaders():
 
     train_df, dev_df = train_test_split(license_summary_data, test_size=0.1)
 
+    TOKENIZER = T5Tokenizer.from_pretrained(MODEL_NAME)
+
     train_dataset = LicenseSummaryDataset(
         train_df,
         TOKENIZER,
@@ -239,6 +241,8 @@ def summarize_text_with_model(text, model, tokenizer):
         Summary of the License text from the given model.
 
     """
+    text, definitions = clean_license_text(text)
+
     text_encoding = tokenizer(
         text,
         max_length=TEXT_MAX_TOKEN_LEN,
@@ -267,10 +271,10 @@ def summarize_text_with_model(text, model, tokenizer):
         ) for gen_id in generated_ids
     ]
 
-    return "".join(preds)
+    return "".join(preds), definitions
 
 
-def summarize(text, load_from_huggingface=False):
+def summarize(text, load_from_huggingface=True):
     """
     Summarizes the given License text
 
@@ -295,6 +299,7 @@ def summarize(text, load_from_huggingface=False):
         if os.path.exists(MODEL_PATH + MODEL_FILENAME):
             print("Loading Model...")
             model = T5ForConditionalGeneration.from_pretrained(MODEL_NAME, return_dict=True).to(device)
+            TOKENIZER = T5Tokenizer.from_pretrained(MODEL_NAME)
             model.load_state_dict(torch.load(MODEL_PATH + MODEL_FILENAME))
             model.eval()
         else:
@@ -305,8 +310,7 @@ def summarize(text, load_from_huggingface=False):
             model = train_and_save_model(train_dataloader, MODEL_PATH + MODEL_FILENAME)
         tokenizer = TOKENIZER
 
-    summary = summarize_text_with_model(text, model, tokenizer)
-    return summary
+    return summarize_text_with_model(text, model, tokenizer)
 
 
 def summarize_license_files(path):
@@ -322,6 +326,6 @@ def summarize_license_files(path):
     paths = glob.glob(path + "*.txt")
     for license_path in paths:
         with open(license_path, "r", encoding="utf-8") as f:
-            summary = summarize(f.read())
+            summary, _ = summarize(f.read())
         with open(license_path.replace(".txt", "") + "__summary.txt", "w", encoding="utf-8") as f:
             f.write(summary)

src/clean.py

@@ -1,27 +1,46 @@
 import re
-import os
+import json
 from bs4 import BeautifulSoup
 from striprtf.striprtf import rtf_to_text
-import json
-import nltk as nltk
+
+
+PARA_BREAK = "para___break"
 
 
 def php_cleaner(text):
-    return re.findall(r"(?<=<\?php\\n\\n\/\*\*\\n \*).*(?=\\n \*\/)", text)[0]
+    try:
+        return re.findall("\/\*[\S\s]*?\*\/", text)[0]
+    except:
+        return ""
+    # return re.findall(r"(?<=<\?php\\n\\n\/\*\*\\n \*).*(?=\\n \*\/)", text)[0]
+
+
 def html_cleaner(text):
     soup = BeautifulSoup(text)
-    return soup.body.text
-def json_cleaner(text):
-    out = "" 
-    for keys in text:
-        if keys in ('description', 'license'):
-            out+=keys
-            out+=": "
-            out+=str(text[keys])
-            out+=", "
+    text = soup.body.text
+    if not text:
+        return ""
+    return text
+
+
+def json_cleaner(text_dict):
+    out = ""
+    for key in text_dict.keys():
+        if key in ("description", "license"):
+            out += key
+            out += ": "
+            out += str(text_dict[key])
+            out += ", "
     return out
+
+
+def discard_text_after_tnc(text):
+    return text.split("END OF TERMS AND CONDITIONS")[0]
+
+
 def gnu_cleaner(text):
     t = text.split('END OF TERMS AND CONDITIONS')[0]
+    definitions = ""
     if 'Preamble' in text:
         if len(t.split('Preamble')[0])>100:
             t0 = t.split('Preamble')[0]
@@ -32,49 +51,121 @@ def gnu_cleaner(text):
                     t1 = t.split('Preamble')[1].split('distribution and\n\nmodification follow')[1]
                 except:
                     t1 = t.split('Preamble')[1].split('distribution and modification follow')[1]
-            return t0+t1
+            t = t0+t1
         else: 
-            return t.split('Preamble')[1].split('distribution and\nmodification follow')[1]
-    else:
-        return t
+            t = t.split('Preamble')[1].split('distribution and\nmodification follow')[1]
+    if 'Definitions' in text:
+        try:
+            def_pos = re.search(r"[0-9]\.? (Additional )?Definitions",t).span()
+            other_start_pos = re.search(r"[0-9]\.? [A-Z][a-z]+",t[def_pos[1]:]).span()[0]
+            definitions = t[def_pos[0]: def_pos[1] + other_start_pos]
+            t = t[:def_pos[0]] + t[def_pos[1]+other_start_pos:]
+        except:
+            t = t
+    return t, definitions
+
+
 def rtf_cleaner(text):
     return rtf_to_text(text)
-def character_cleaner(text):
-    return re.sub("[=*-/·\n]+", "", text)
+
+
 def url_cleaner(text):
-    return re.sub(r'http\S+', '', text)
+    return re.sub(r"http\S+", "", text)
+
+
+def email_cleaner(text):
+    return re.sub(r"\S*@\S*", "", text)
+
+
+def var_cleaner(text):
+    text = re.sub(r"\$\w+", "", text)
+    text = re.sub(r"{[{}()\w\s._,\[\]'\"]+}", "", text)
+    return text
+
+
+def character_cleaner(text):
+    text = url_cleaner(text)
+    text = email_cleaner(text)
+    text = var_cleaner(text)
+
+    text = re.sub("[\n]{2,}", ". ", text)
+    text = re.sub("[:%#<>=*\-/·\s{}]+", " ", text)
+    text = re.sub("[\. ]{2,}", ". ", text)
+    return text
+
+
 def isEnglish(s):
     try:
-        s.encode(encoding='utf-8').decode('ascii')
+        s.encode(encoding="utf-8").decode("ascii")
     except UnicodeDecodeError:
         return False
     else:
         return True
-    
-# input as a text
-def clean_license_text(text):
-    text = text.strip()
-    if text[:5] == '<?php':
-        try:
-            t = php_cleaner(text)
-        except:
-            return ""
+
+
+def preprocess_text(text):
+    definitions = ""
+    if "GNU" in text or "Apache" in text:
+        text, definitions = gnu_cleaner(text)
+        definitions = definitions.strip()
+    return text, definitions
+
+
+def script_cleaner(text):
+    if "<?php" in text:
+        text = php_cleaner(text)
     elif "</html>" in text:
-        t = html_cleaner(text)
-    elif text[0] == '{' and text[-1] == '}':
-        with open(file, 'r') as f:
-            t = json_cleaner(json.load(f))
-    elif "GNU" in text or "Apache" in text:
-        t = gnu_cleaner(text)
+        text = html_cleaner(text)
+    elif text[0] == "{" and text[-1] == "}":
+        text = json_cleaner(json.loads(text))
     elif "\\rtf" in text:
-        t = rtf_cleaner(text)
+        text = rtf_cleaner(text)
+    if not text:
+        return ""
+    return text
+
+
+def split_paras(text):
+    if "\n\n\n\n" in text:
+        paras = text.split("\n\n\n\n")
+    elif "\n\n\n" in text:
+        paras = text.split("\n\n\n")
+    elif "\n\n" in text:
+        paras = text.split("\n\n")
     else:
-        t = text
+        paras = [text]
+    return paras
+
+
+def clean_paras(paras):
+    return paras
+
+
+def clean_license_text(text):
 
-    t = url_cleaner(t)
-    t = character_cleaner(t)
+    if len(text) == 0:
+        return text
 
-    if not isEnglish(t):
-        if not isEnglish(' '.join(t.split()[-5:-1])):
-            return ""
-    return t
+    text = script_cleaner(text)
+    text, definitions = preprocess_text(text)
+    paras = clean_paras(split_paras(text))
+    text = PARA_BREAK.join(paras)
+    text = character_cleaner(text)
+    text = re.sub(PARA_BREAK, "\n\n", text)
+    text = text.strip()
+
+    if not isEnglish(text):
+        if not isEnglish(" ".join(text.split()[-5:-1])):
+            return "", ""
+    
+    return text, definitions
+
+
+"""
+Notes: 
+    
+    1. Regex for other definitions: -------->  ".{0,20}".{0,40}means
+    2. Try splitting each para by "\n", if len == 1 and len(para) < 100 (or something)
+       -> Merge with the next para
+       Ex. "8. Termination."
+"""

src/read_data.py

@@ -129,7 +129,7 @@ def augment_summary(license_data):
     return license_data
 
 
-def read_license_data(labels_file="choosealicense_appendix_labels.csv"):
+def read_license_data(labels_file="choosealicense_appendix_labels.csv", drop_summary=False):
     """
     Reads data from Text and Summary File and stores it as a dictionary of
     dictionaries.
@@ -142,21 +142,46 @@ def read_license_data(labels_file="choosealicense_appendix_labels.csv"):
         corresponding summaries and license texts respectively.
 
     """
+
     files = glob.glob(gold_licenses_data + "*")
+    if not files:
+        files = glob.glob(f"../{gold_licenses_data}" + "*")
+    if not files:
+        print("Gold licenses not found, please check the path again!")
+        return None
+
     data_dict = defaultdict(dict)
+
     for file_path in files:
+        if "\\" in file_path:
+            split_by = "\\"
+        else:
+            split_by = "/"
+
         if file_path.endswith(".summary"):
-            file_name = file_path.split("/")[-1][:-8]
+            file_name = file_path.split(split_by)[-1][:-8]
             data_dict[file_name]["summary"] = read_file(file_path)
         elif file_path.endswith(".txt"):
-            file_name = file_path.split("/")[-1][:-4]
+            file_name = file_path.split(split_by)[-1][:-4]
             data_dict[file_name]["text"] = clean_data(read_file(file_path))
     
     summary_df = pd.DataFrame(data_dict).T
-    labels_df = pd.read_csv(data_directory + labels_file, index_col=index_col)
 
-    merged_data = labels_df.join(summary_df).drop(columns=["spdx_id", "GTLC_Notes"])
-    return merged_data[:5]
+    try:
+        labels_df = pd.read_csv(data_directory + labels_file, index_col=index_col)
+    except:
+        try:
+            labels_df = pd.read_csv(f"../{data_directory}" + labels_file, index_col=index_col)
+        except:
+            print("Labels file not found, please check the path again!")
+            return None
+
+    merged_data = labels_df.join(summary_df).drop(columns=["spdx_id"])
+
+    if drop_summary:
+        merged_data = merged_data.drop(columns=["summary"])
+
+    return merged_data
 
 
 def read_license_summary_data(aug_summary=False):
@@ -193,11 +218,11 @@ def fix_labels(license_data):
         "limitations": 2
     }
 
-    permissive_not_permissive_map = {
-        np.nan: 0,
-        "permissive": 1,
-        "not_permissive": 2
-    }
+    # permissive_not_permissive_map = {
+    #     np.nan: 0,
+    #     "permissive": 1,
+    #     "not_permissive": 2
+    # }
 
     permissions_columns = [
         "commercial-use",
@@ -226,14 +251,14 @@ def fix_labels(license_data):
         "patent-use"
     ]
 
-    permissive_not_permissive_columns = [
-        "GTLC_Permissive"
-    ]
+    # permissive_not_permissive_columns = [
+    #     "GTLC_Permissive"
+    # ]
 
     license_data[permissions_columns] = license_data[permissions_columns].replace(permissions_map)
     license_data[conditions_columns] = license_data[conditions_columns].replace(conditions_map)
     license_data[limitations_columns] = license_data[limitations_columns].replace(limitations_map)
     license_data[permissions_limitations_columns] = license_data[permissions_limitations_columns].replace(permissions_limitations_map)
-    license_data[permissive_not_permissive_columns] = license_data[permissive_not_permissive_columns].replace(permissive_not_permissive_map)
+    # license_data[permissive_not_permissive_columns] = license_data[permissive_not_permissive_columns].replace(permissive_not_permissive_map)
 
     return license_data

src/textrank.py

@@ -0,0 +1,69 @@
+import nltk
+import numpy as np
+import gensim
+import spacy
+import math
+from collections import Counter
+
+from src.clean import clean_license_text
+from src.read_data import read_file
+
+nltk.download('punkt')
+
+properties_dict = {
+    "modify":['modify', 'modification', 'change'],
+    "distribute":['distribute', 'distribution'],
+    "copy":['copy'],
+    "copyright": ['copyright']
+#     "exception"
+}
+
+properties_scores = {
+    "modify": 0.8,
+    "distribute": 0.8,
+    "copy": 0.8,
+    "copyright": 0.9 
+}
+
+nlp = spacy.load('en_core_web_sm')
+
+def lemmatize_tokens(sent):
+    #TODO: Docstrings
+    '''each word in input sentence is converted to lemma'''
+    return [token.lemma_.lower() for token in nlp(sent)]
+
+
+def custom_textrank_summarizer(license_text, min_sent_len=2, summary_len=0.3, debug=False):
+    '''
+    TODO: Doctrings
+    '''
+    summary_len = math.ceil(summary_len*len(license_text.split('.')))
+    sent_scores = {}
+    cleaned_license_text, definitions = clean_license_text(license_text)
+    for i in cleaned_license_text.split('.'):
+        if debug:
+            print(i.split())
+        if len(i.split()) < min_sent_len:
+            break
+        score = 0
+        for prop, prop_words in properties_dict.items():
+            prop_score = 0
+            lemmatized_tokens = lemmatize_tokens(i)
+            word_count = Counter([tok for tok in lemmatized_tokens])
+            for prop_word in prop_words:
+                if prop_word in word_count.keys():
+                    prop_score += properties_scores[prop_word] 
+            if debug:
+                print(prop, "=", prop_score)
+            score += prop_score
+        sent_scores[i] = score/len(lemmatized_tokens)
+        if debug:
+            print(f'Sentence score: {sent_scores[i]}')
+            print()
+    if debug:
+        print(sent_scores)
+    sorted_sent_scores = dict(sorted(sent_scores.items(), key=lambda item: item[1], reverse=True))
+    summary = '.\n'.join(list(sorted_sent_scores.keys())[:summary_len])
+    return summary, definitions
+
+