updated

plagiarism.py

@@ -155,10 +155,25 @@ def matching_score(sentence_content_tuple):
         # matched = [x for x in ngrams if " ".join(x) in content]
         # return len(matched) / len(ngrams)
 
-        ngrams_sentence = split_ngrams(sentence, n)
+        # list comprehension matching
+        # ngrams_sentence = split_ngrams(sentence, n)
+        # ngrams_content = [tuple(ngram) for ngram in split_ngrams(content, n)]
+        # if len(ngrams_sentence) == 0:
+        #     return 0, ""
+        # matched_ngrams = [
+        #     1 for ngram in ngrams_sentence if tuple(ngram) in ngrams_content
+        # ]
+        # matched_count = sum(matched_ngrams)
+
+        # set intersection matching
+        ngrams_sentence = set(split_ngrams(sentence, n))
+        ngrams_content = set(split_ngrams(content, n))
         if len(ngrams_sentence) == 0:
             return 0, ""
-        ngrams_content = [tuple(ngram) for ngram in split_ngrams(content, n)]
+        matched_ngrams = ngrams_sentence.intersection(ngrams_content)
+        matched_count = len(matched_ngrams)
+
+        # matched content
         matched_content_ngrams = []
         found = False
         last_found = None
@@ -174,10 +189,6 @@ def matching_score(sentence_content_tuple):
         )
         matched_content = merge_ngrams_into_sentence(matched_content_ngrams)
 
-        matched_ngrams = [
-            1 for ngram in ngrams_sentence if tuple(ngram) in ngrams_content
-        ]
-        matched_count = sum(matched_ngrams)
         return matched_count / len(ngrams_sentence), matched_content
 
 
@@ -229,7 +240,7 @@ def google_search(
     **kwargs,
 ):
     service = build("customsearch", "v1", developerKey=api_key)
-    num_pages = 3
+    num_pages = 1
     for i, sentence in enumerate(sentences):
         results = (
             service.cse()
@@ -287,10 +298,10 @@ def plagiarism_check(
 ):
     # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
-    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
     # api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
     # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
-    api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
+    # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
     # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     cse_id = "851813e81162b4ed4"
 
@@ -305,6 +316,7 @@ def plagiarism_check(
     date_to = build_date(year_to, month_to, day_to)
     sort_date = f"date:r:{date_from}:{date_to}"
     # get list of URLS to check
+    start_time = time.perf_counter()
     url_count, score_array = google_search(
         plag_option,
         sentences,
@@ -317,15 +329,20 @@ def plagiarism_check(
         api_key,
         cse_id,
     )
+    print("GOOGLE SEARCH PROCESSING TIME: ", time.perf_counter() - start_time)
     # Scrape URLs in list
+    start_time = time.perf_counter()
     soups = asyncio.run(parallel_scrap(url_list))
+    print("SCRAPING PROCESSING TIME: ", time.perf_counter() - start_time)
     input_data = []
     for i, soup in enumerate(soups):
         if soup:
             page_content = soup.text
             for j, sent in enumerate(sentences):
                 input_data.append((sent, page_content, score_array[i][j]))
+    start_time = time.perf_counter()
     scores = process_with_multiprocessing(input_data)
+    print("MATCHING SCORE PROCESSING TIME: ", time.perf_counter() - start_time)
     matched_sentence_array = [
         ["" for _ in range(len(score_array[0]))]
         for _ in range(len(score_array))