Update app.py

app.py

@@ -23,6 +23,8 @@ import nltk
 from unidecode import unidecode
 import time
 from utils import cos_sim_torch, embed_text
+import multiprocessing
+from functools import partial
 
 nltk.download('punkt')
 
@@ -113,15 +115,37 @@ def plagiarism_check(
            source_embeddings.append(None)
            
     # Populate matching scores for scrapped pages
-    for i, soup in enumerate(soups):
-        print(f"Analyzing {i+1} of {len(soups)} soups........................")
-        if soup:
-            page_content = soup.text
-            for j, sent in enumerate(sentences):
-                # score = matchingScore(sent, page_content)
-                # score = matchingScoreWithTimeout(sent, page_content)
-                score = cos_sim_torch(embed_text(sent), source_embeddings[i])
-                ScoreArray[i][j] = score
+    # for i, soup in enumerate(soups):
+    #     print(f"Analyzing {i+1} of {len(soups)} soups........................")
+    #     if soup:
+    #         page_content = soup.text
+    #         for j, sent in enumerate(sentences):
+    #             # score = matchingScore(sent, page_content)
+    #             # score = matchingScoreWithTimeout(sent, page_content)
+    #             score = cos_sim_torch(embed_text(sent), source_embeddings[i])
+    #             ScoreArray[i][j] = score
+    
+    def calculate_score(params):
+        i, sent, source_embedding = params
+        return cos_sim_torch(embed_text(sent), source_embedding)
+
+    
+    def worker(soups, sentences, source_embeddings):
+        params_list = []
+        for i, soup in enumerate(soups):
+            if soup:
+                for j, sent in enumerate(sentences):
+                    params_list.append((i, sent, source_embeddings[i]))
+        num_processes = multiprocessing.cpu_count()
+        pool = multiprocessing.Pool(processes=num_processes)
+        scores = pool.map(calculate_score, params_list)
+        pool.close()
+        pool.join()
+        for k, score in enumerate(scores):
+            i, j = divmod(k, len(sentences))
+            ScoreArray[i][j] = score
+
+    worker(soups, sentences, source_embeddings)
 
     print(f"Time for matching score: {time.perf_counter()-time1}")
     time1 = time.perf_counter()