Updating clean function

src/clean.py

@@ -236,7 +236,7 @@ def isEnglish(s):
         return True
 
 
-def split_definitions_exceptions(text, verbosity=0):
+def split_definitions_exceptions(text, remove_exceptions, verbosity=0):
     """
     Extract definitions from the License text
 
@@ -275,7 +275,7 @@ def split_definitions_exceptions(text, verbosity=0):
     )
 
     definitions += more_defs.strip()
-    paras, exceptions = get_exeptions(paras, verbosity=verbosity)
+    paras, exceptions = get_exeptions(paras, remove_exceptions, verbosity=verbosity)
 
     return paras, definitions, exceptions
 
@@ -595,7 +595,7 @@ def get_all_caps(text, verbosity=0):
     return text, all_caps
 
 
-def get_exeptions(paras, verbosity=0):
+def get_exeptions(paras, remove_exceptions, verbosity=0):
     """
     Extracts a list of exceptions from the License text.
 
@@ -620,6 +620,8 @@ def get_exeptions(paras, verbosity=0):
     for para in paras:
         if re.search("exception", para.lower()):
             exceptions.append(para)
+            if not remove_exceptions:
+                non_exception_paras.append(para)
         else:
             non_exception_paras.append(para)
 
@@ -697,7 +699,11 @@ def get_most_likely_license_type(text):
         The type of the most likely license. "Not found" if no license score is
         above 0.9
     """
-    from src.doc2vec import inference
+
+    try:
+        from src.doc2vec import inference
+    except:
+        from doc2vec import inference
 
     top1_result = inference(text).loc[0, :]
 
@@ -707,7 +713,7 @@ def get_most_likely_license_type(text):
         return "Not Found"
 
 
-def clean_license_text(text, verbosity=0):
+def clean_license_text(text, remove_exceptions=False, verbosity=0):
     """
     Cleans License text.
 
@@ -736,7 +742,7 @@ def clean_license_text(text, verbosity=0):
     text = script_cleaner(text)
     text = preprocess_text(text)
     paras, definitions, exceptions = split_definitions_exceptions(
-        text, verbosity=verbosity
+        text, remove_exceptions, verbosity=verbosity
     )
     text = PARA_BREAK.join(paras)
     text = character_cleaner(text)
@@ -756,7 +762,6 @@ def clean_license_text(text, verbosity=0):
 
     return text, definitions
 
-
 """
 Notes: