Update app.py

app.py

@@ -25,6 +25,7 @@ import time
 from utils import cos_sim_torch, embed_text
 import multiprocessing
 from functools import partial
+import concurrent.futures
 
 nltk.download('punkt')
 
@@ -124,28 +125,30 @@ def plagiarism_check(
     #             # score = matchingScoreWithTimeout(sent, page_content)
     #             score = cos_sim_torch(embed_text(sent), source_embeddings[i])
     #             ScoreArray[i][j] = score
-    
-    def calculate_score(params):
-        i, sent, source_embedding = params
-        return cos_sim_torch(embed_text(sent), source_embedding)
 
-    
-    def worker(soups, sentences, source_embeddings):
-        params_list = []
-        for i, soup in enumerate(soups):
-            if soup:
-                for j, sent in enumerate(sentences):
-                    params_list.append((i, sent, source_embeddings[i]))
-        num_processes = multiprocessing.cpu_count()
-        pool = multiprocessing.Pool(processes=num_processes)
-        scores = pool.map(calculate_score, params_list)
-        pool.close()
-        pool.join()
-        for k, score in enumerate(scores):
-            i, j = divmod(k, len(sentences))
+
+def compute_cosine_similarity(args):
+    sent, source_embedding, i, j = args
+    score = cos_sim_torch(embed_text(sent), source_embedding)
+    return i, j, score
+
+def main(soups, sentences):
+    source_embeddings = [preprocess(soup) for soup in soups]
+    ScoreArray = [[0 for _ in sentences] for _ in soups]
+    args_list = []
+    for i, soup in enumerate(soups):
+        if soup:
+            for j, sent in enumerate(sentences):
+                args_list.append((sent, source_embeddings[i], i, j))
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        results = executor.map(compute_cosine_similarity, args_list)
+        for i, j, score in results:
             ScoreArray[i][j] = score
+    return ScoreArray
+
+    ScoreArray = main(soups, sentences)
+
 
-    worker(soups, sentences, source_embeddings)
 
     print(f"Time for matching score: {time.perf_counter()-time1}")
     time1 = time.perf_counter()