speed up plagiarism

plagiarism.py

@@ -33,8 +33,13 @@ months = {
 
 color_map = [
     "#cf2323",
+    "#d65129",
+    "#d66329",
+    "#d67129",
     "#eb9d59",
     "#c2ad36",
+    "#d6ae29",
+    "#d6b929",
     "#e1ed72",
     "#c2db76",
     "#a2db76",
@@ -114,25 +119,25 @@ def matching_score(sentence_content_tuple):
     sentence, content, score = sentence_content_tuple
     if sentence in content:
         return 1
-    # if score > 0.9:
-    #     return score
+    if score > 0.9:
+        return score
     else:
         n = 5
 
-        ngrams = split_ngrams(sentence, n)
-        if len(ngrams) == 0:
-            return 0
-        matched = [x for x in ngrams if " ".join(x) in content]
-        return len(matched) / len(ngrams)
-
-        # ngrams_sentence = split_ngrams(sentence, n)
-        # if len(ngrams_sentence) == 0:
+        # ngrams = split_ngrams(sentence, n)
+        # if len(ngrams) == 0:
         #     return 0
-        # ngrams_content = set(tuple(ngram) for ngram in split_ngrams(content, n))
-        # matched_count = sum(
-        #     1 for ngram in ngrams_sentence if tuple(ngram) in ngrams_content
-        # )
-        # return matched_count / len(ngrams_sentence)
+        # matched = [x for x in ngrams if " ".join(x) in content]
+        # return len(matched) / len(ngrams)
+
+        ngrams_sentence = split_ngrams(sentence, n)
+        if len(ngrams_sentence) == 0:
+            return 0
+        ngrams_content = set(tuple(ngram) for ngram in split_ngrams(content, n))
+        matched_count = sum(
+            1 for ngram in ngrams_sentence if tuple(ngram) in ngrams_content
+        )
+        return matched_count / len(ngrams_sentence)
 
 
 def process_with_multiprocessing(input_data):
@@ -174,6 +179,7 @@ def google_search(
     **kwargs,
 ):
     service = build("customsearch", "v1", developerKey=api_key)
+    num_pages = 3
     for i, sentence in enumerate(sentences):
         results = (
             service.cse()
@@ -182,8 +188,7 @@ def google_search(
         )
         if "items" in results and len(results["items"]) > 0:
             for count, link in enumerate(results["items"]):
-                # stop after 3 pages
-                if count >= 3:
+                if count >= num_pages:
                     break
                 # skip user selected domains
                 if (domains_to_skip is not None) and any(
@@ -228,11 +233,11 @@ def plagiarism_check(
     domains_to_skip,
     source_block_size,
 ):
-    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     # api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
-    # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
+    api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
     # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
     # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     cse_id = "851813e81162b4ed4"