refactored plagiarism

plagiarism.py

@@ -16,37 +16,36 @@ WORD = re.compile(r"\w+")
 model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
 
-# returns cosine similarity of two vectors
-# input: two vectors
-# output: integer between 0 and 1.
-def get_cosine(vec1, vec2):
-    intersection = set(vec1.keys()) & set(vec2.keys())
-
-    # calculating numerator
-    numerator = sum([vec1[x] * vec2[x] for x in intersection])
-
-    # calculating denominator
-    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
-    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
-    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+months = {
+    "January": "01",
+    "February": "02",
+    "March": "03",
+    "April": "04",
+    "May": "05",
+    "June": "06",
+    "July": "07",
+    "August": "08",
+    "September": "09",
+    "October": "10",
+    "November": "11",
+    "December": "12",
+}
 
-    # checking for divide by zero
-    if denominator == 0:
-        return 0.0
-    else:
-        return float(numerator) / denominator
+color_map = [
+    "#cf2323",
+    "#eb9d59",
+    "#c2ad36",
+    "#e1ed72",
+    "#c2db76",
+    "#a2db76",
+]
 
 
-# converts given text into a vector
 def text_to_vector(text):
-    # uses the Regular expression above and gets all words
     words = WORD.findall(text)
-    # returns a counter of all the words (count of number of occurences)
     return Counter(words)
 
 
-# returns cosine similarity of two words
-# uses: text_to_vector(text) and get_cosine(v1,v2)
 def cosineSim(text1, text2):
     vector1 = text_to_vector(text1)
     vector2 = text_to_vector(text2)
@@ -55,75 +54,16 @@ def cosineSim(text1, text2):
     return cosine
 
 
-def cos_sim_torch(embedding_1, embedding_2):
-    return util.pytorch_cos_sim(embedding_1, embedding_2).item()
-
-
-def embed_text(text):
-    return model.encode(text, convert_to_tensor=True)
-
-
-def sentence_similarity(text1, text2):
-    embedding_1 = model.encode(text1, convert_to_tensor=True)
-    embedding_2 = model.encode(text2, convert_to_tensor=True)
-
-    o = util.pytorch_cos_sim(embedding_1, embedding_2)
-    return o.item()
-
-
-def google_search(
-    plag_option,
-    sentences,
-    url_count,
-    score_array,
-    url_list,
-    sorted_date,
-    domains_to_skip,
-    api_key,
-    cse_id,
-    **kwargs,
-):
-    service = build("customsearch", "v1", developerKey=api_key)
-    for i, sentence in enumerate(sentences):
-        results = (
-            service.cse()
-            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
-            .execute()
-        )
-        if "items" in results and len(results["items"]) > 0:
-            for count, link in enumerate(results["items"]):
-                # stop after 3 pages
-                if count >= 3:
-                    break
-                # skip user selected domains
-                if any(
-                    ("." + domain) in link["link"] for domain in domains_to_skip
-                ):
-                    continue
-                # clean up snippet of '...'
-                snippet = link["snippet"]
-                ind = snippet.find("...")
-                if ind < 20 and ind > 9:
-                    snippet = snippet[ind + len("... ") :]
-                ind = snippet.find("...")
-                if ind > len(snippet) - 5:
-                    snippet = snippet[:ind]
-
-                # update cosine similarity between snippet and given text
-                url = link["link"]
-                if url not in url_list:
-                    url_list.append(url)
-                    score_array.append([0] * len(sentences))
-                url_count[url] = url_count[url] + 1 if url in url_count else 1
-                if plag_option == "Standard":
-                    score_array[url_list.index(url)][i] = cosineSim(
-                        sentence, snippet
-                    )
-                else:
-                    score_array[url_list.index(url)][i] = sentence_similarity(
-                        sentence, snippet
-                    )
-    return url_count, score_array
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+    if denominator == 0:
+        return 0.0
+    else:
+        return float(numerator) / denominator
 
 
 def split_sentence_blocks(text):
@@ -138,49 +78,32 @@ def split_sentence_blocks(text):
     return two_sents
 
 
-months = {
-    "January": "01",
-    "February": "02",
-    "March": "03",
-    "April": "04",
-    "May": "05",
-    "June": "06",
-    "July": "07",
-    "August": "08",
-    "September": "09",
-    "October": "10",
-    "November": "11",
-    "December": "12",
-}
-
-
 def build_date(year=2024, month="March", day=1):
     return f"{year}{months[month]}{day}"
 
 
+def split_ngrams(text, n):
+    words = text.split()
+    return [words[i : i + n] for i in range(len(words) - n + 1)]
+
+
+def sentence_similarity(text1, text2):
+    embedding_1 = model.encode(text1, convert_to_tensor=True)
+    embedding_2 = model.encode(text2, convert_to_tensor=True)
+    o = util.pytorch_cos_sim(embedding_1, embedding_2)
+    return o.item()
+
+
 async def get_url_data(url, client):
     try:
         r = await client.get(url)
-        # print(r.status_code)
         if r.status_code == 200:
-            # print("in")
             soup = BeautifulSoup(r.content, "html.parser")
             return soup
     except Exception:
         return None
 
 
-def remove_punc(text):
-    res = re.sub(r"[^\w\s]", "", text)
-    return res
-
-
-def split_ngrams(text, n):
-    # return n-grams of size n
-    words = text.split()
-    return [words[i : i + n] for i in range(len(words) - n + 1)]
-
-
 async def parallel_scrap(urls):
     async with httpx.AsyncClient(timeout=30) as client:
         tasks = []
@@ -209,11 +132,6 @@ def process_with_multiprocessing(input_data):
     return scores
 
 
-def print2d(array):
-    for row in array:
-        print(row)
-
-
 def map_sentence_url(sentences, score_array):
     sentenceToMaxURL = [-1] * len(sentences)
     for j in range(len(sentences)):
@@ -234,65 +152,59 @@ def map_sentence_url(sentences, score_array):
     return sentenceToMaxURL
 
 
-def html_highlight(
+def google_search(
     plag_option,
-    input,
-    year_from,
-    month_from,
-    day_from,
-    year_to,
-    month_to,
-    day_to,
+    sentences,
+    url_count,
+    score_array,
+    url_list,
+    sorted_date,
     domains_to_skip,
+    api_key,
+    cse_id,
+    **kwargs,
 ):
-    sentence_scores, url_scores = plagiarism_check(
-        plag_option,
-        input,
-        year_from,
-        month_from,
-        day_from,
-        year_to,
-        month_to,
-        day_to,
-        domains_to_skip,
-    )
-    color_map = [
-        "#cf2323",
-        "#eb9d59",
-        "#c2ad36",
-        "#e1ed72",
-        "#c2db76",
-        "#a2db76",
-    ]
-    font = "Roboto"
-    html_content = "<link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet'>\n<div style='font-family: {font}; border: 2px solid black; background-color: #333333; padding: 10px; color: #FFFFFF;'>"
-    prev_idx = None
-    combined_sentence = ""
-    for sentence, _, _, idx in sentence_scores:
-        if idx != prev_idx and prev_idx is not None:
-            color = color_map[prev_idx - 1]
-            index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
-            formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
-            html_content += formatted_sentence
-            combined_sentence = ""
-        combined_sentence += " " + sentence
-        prev_idx = idx
-
-    if combined_sentence:
-        color = color_map[prev_idx - 1]
-        index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
-        formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
-        html_content += formatted_sentence
-
-    html_content += "<hr>"
-    for url, score, idx in url_scores:
-        color = color_map[idx - 1]
-        formatted_url = f'<p style="background-color: {color}; padding: 5px;">({idx}) <b>{url}</b></p><p> --- Matching Score: {score}%</p>'
-        html_content += formatted_url
-
-    html_content += "</div>"
+    service = build("customsearch", "v1", developerKey=api_key)
+    for i, sentence in enumerate(sentences):
+        results = (
+            service.cse()
+            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+            .execute()
+        )
+        if "items" in results and len(results["items"]) > 0:
+            for count, link in enumerate(results["items"]):
+                # stop after 3 pages
+                if count >= 3:
+                    break
+                # skip user selected domains
+                if any(
+                    ("." + domain) in link["link"] for domain in domains_to_skip
+                ):
+                    continue
+                # clean up snippet of '...'
+                snippet = link["snippet"]
+                ind = snippet.find("...")
+                if ind < 20 and ind > 9:
+                    snippet = snippet[ind + len("... ") :]
+                ind = snippet.find("...")
+                if ind > len(snippet) - 5:
+                    snippet = snippet[:ind]
 
-    return html_content
+                # update cosine similarity between snippet and given text
+                url = link["link"]
+                if url not in url_list:
+                    url_list.append(url)
+                    score_array.append([0] * len(sentences))
+                url_count[url] = url_count[url] + 1 if url in url_count else 1
+                if plag_option == "Standard":
+                    score_array[url_list.index(url)][i] = cosineSim(
+                        sentence, snippet
+                    )
+                else:
+                    score_array[url_list.index(url)][i] = sentence_similarity(
+                        sentence, snippet
+                    )
+    return url_count, score_array
 
 
 def plagiarism_check(
@@ -306,11 +218,11 @@ def plagiarism_check(
     day_to,
     domains_to_skip,
 ):
-    api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
-    api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
+    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    # api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
     # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
     # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
-    # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
     cse_id = "851813e81162b4ed4"
 
     url_scores = []
@@ -384,3 +296,56 @@ def plagiarism_check(
             )
 
     return sentence_scores, url_scores
+
+
+def html_highlight(
+    plag_option,
+    input,
+    year_from,
+    month_from,
+    day_from,
+    year_to,
+    month_to,
+    day_to,
+    domains_to_skip,
+):
+    sentence_scores, url_scores = plagiarism_check(
+        plag_option,
+        input,
+        year_from,
+        month_from,
+        day_from,
+        year_to,
+        month_to,
+        day_to,
+        domains_to_skip,
+    )
+
+    html_content = "<link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet'>\n<div style='font-family: {font}; border: 2px solid black; background-color: #333333; padding: 10px; color: #FFFFFF;'>"
+    prev_idx = None
+    combined_sentence = ""
+    for sentence, _, _, idx in sentence_scores:
+        if idx != prev_idx and prev_idx is not None:
+            color = color_map[prev_idx - 1]
+            index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
+            formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
+            html_content += formatted_sentence
+            combined_sentence = ""
+        combined_sentence += " " + sentence
+        prev_idx = idx
+
+    if combined_sentence:
+        color = color_map[prev_idx - 1]
+        index_part = f'<span style="background-color: {color}; padding: 2px;">[{prev_idx}]</span>'
+        formatted_sentence = f"<p>{combined_sentence} {index_part}</p>"
+        html_content += formatted_sentence
+
+    html_content += "<hr>"
+    for url, score, idx in url_scores:
+        color = color_map[idx - 1]
+        formatted_url = f'<p style="background-color: {color}; padding: 5px;">({idx}) <b>{url}</b></p><p> --- Matching Score: {score}%</p>'
+        html_content += formatted_url
+
+    html_content += "</div>"
+
+    return html_content

predictors.py

@@ -1,23 +1,11 @@
-import requests
-import httpx
 import torch
-import re
-from bs4 import BeautifulSoup
 import numpy as np
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import asyncio
-from evaluate import load
-from datetime import date
 import nltk
-from transformers import GPT2LMHeadModel, GPT2TokenizerFast
-import plotly.graph_objects as go
 import torch.nn.functional as F
 import nltk
-from unidecode import unidecode
-import time
 from scipy.special import softmax
 import yaml
-import os
 from utils import *
 import joblib
 
@@ -51,9 +39,9 @@ tokenizers_1on1 = {}
 models_1on1 = {}
 for model_name, model in zip(mc_label_map, text_1on1_models):
     tokenizers_1on1[model_name] = AutoTokenizer.from_pretrained(model)
-    models_1on1[model_name] = AutoModelForSequenceClassification.from_pretrained(
-        model
-    ).to(device)
+    models_1on1[model_name] = (
+        AutoModelForSequenceClassification.from_pretrained(model).to(device)
+    )
 
 # proxy models for explainability
 mini_bc_model_name = "polygraf-ai/bc-model-bert-mini"
@@ -62,7 +50,9 @@ bc_model_mini = AutoModelForSequenceClassification.from_pretrained(
     mini_bc_model_name
 ).to(device)
 mini_humanizer_model_name = "polygraf-ai/quillbot-detector-bert-mini-9K"
-humanizer_tokenizer_mini = AutoTokenizer.from_pretrained(mini_humanizer_model_name)
+humanizer_tokenizer_mini = AutoTokenizer.from_pretrained(
+    mini_humanizer_model_name
+)
 humanizer_model_mini = AutoModelForSequenceClassification.from_pretrained(
     mini_humanizer_model_name
 ).to(device)
@@ -232,7 +222,9 @@ def predict_mc_scores(input):
     bc_scores = []
     mc_scores = []
 
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det="bc"))
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
     segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
     for i in range(samples_len_bc):
         cleaned_text_bc = remove_special_characters(segments_bc[i])
@@ -243,7 +235,9 @@ def predict_mc_scores(input):
     bc_score_list = average_bc_scores.tolist()
     bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
     segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
-    samples_len_mc = len(split_text_allow_complete_sentences_nltk(input, type_det="mc"))
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
     for i in range(samples_len_mc):
         cleaned_text_mc = remove_special_characters(segments_mc[i])
         mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
@@ -266,7 +260,9 @@ def predict_mc_scores(input):
 
 def predict_bc_scores(input):
     bc_scores = []
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det="bc"))
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
     segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
     for i in range(samples_len_bc):
         cleaned_text_bc = remove_special_characters(segments_bc[i])
@@ -275,7 +271,9 @@ def predict_bc_scores(input):
     bc_scores_array = np.array(bc_scores)
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
-    print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
+    print(
+        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
+    )
     # isotonic regression calibration
     ai_score = iso_reg.predict([bc_score_list[1]])[0]
     human_score = 1 - ai_score
@@ -309,7 +307,9 @@ def predict_1on1_combined(input):
 
 
 def predict_1on1_single(input, model):
-    predictions = predict_1on1(models_1on1[model], tokenizers_1on1[model], input)[1]
+    predictions = predict_1on1(
+        models_1on1[model], tokenizers_1on1[model], input
+    )[1]
     return predictions
 
 
@@ -321,7 +321,9 @@ def predict_1on1_scores(input, models):
     print(f"Models to Test: {models}")
     # BC SCORE
     bc_scores = []
-    samples_len_bc = len(split_text_allow_complete_sentences_nltk(input, type_det="bc"))
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
     segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
     for i in range(samples_len_bc):
         cleaned_text_bc = remove_special_characters(segments_bc[i])
@@ -330,24 +332,30 @@ def predict_1on1_scores(input, models):
     bc_scores_array = np.array(bc_scores)
     average_bc_scores = np.mean(bc_scores_array, axis=0)
     bc_score_list = average_bc_scores.tolist()
-    print(f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}")
+    print(
+        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
+    )
     # isotonic regression calibration
     ai_score = iso_reg.predict([bc_score_list[1]])[0]
     human_score = 1 - ai_score
     bc_score = {"AI": ai_score, "HUMAN": human_score}
     print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
-    
+
     # MC SCORE
     if len(models) > 1:
         print("Starting MC")
         mc_scores = []
-        segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+        segments_mc = split_text_allow_complete_sentences_nltk(
+            input, type_det="mc"
+        )
         samples_len_mc = len(
             split_text_allow_complete_sentences_nltk(input, type_det="mc")
         )
         for i in range(samples_len_mc):
             cleaned_text_mc = remove_special_characters(segments_mc[i])
-            mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
+            mc_score = predict_mc(
+                text_mc_model, text_mc_tokenizer, cleaned_text_mc
+            )
             mc_scores.append(mc_score)
         mc_scores_array = np.array(mc_scores)
         average_mc_scores = np.mean(mc_scores_array, axis=0)
@@ -357,7 +365,9 @@ def predict_1on1_scores(input, models):
             mc_score[label.upper()] = score
 
         mc_score = {
-            key: mc_score[key.upper()] for key in models if key.upper() in mc_score
+            key: mc_score[key.upper()]
+            for key in models
+            if key.upper() in mc_score
         }
         total = sum(mc_score.values())
         # Normalize each value by dividing it by the total
@@ -365,14 +375,16 @@ def predict_1on1_scores(input, models):
         sum_prob = 1 - bc_score["HUMAN"]
         for key, value in mc_score.items():
             mc_score[key] = value * sum_prob
-        print('MC Score:',mc_score)
+        print("MC Score:", mc_score)
         if sum_prob < 0.01:
             mc_score = {}
 
     elif len(models) == 1:
         print("Starting 1on1")
         mc_scores = []
-        segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+        segments_mc = split_text_allow_complete_sentences_nltk(
+            input, type_det="mc"
+        )
         samples_len_mc = len(
             split_text_allow_complete_sentences_nltk(input, type_det="mc")
         )

utils.py

@@ -1,28 +1,11 @@
-from urllib.request import urlopen, Request
-from googleapiclient.discovery import build
-import requests
-import httpx
 import re
-from bs4 import BeautifulSoup
-import re, math
-from collections import Counter
-import numpy as np
-import asyncio
-import nltk
+import re
 from sentence_transformers import SentenceTransformer, util
-import threading
-import torch
 import re
-import numpy as np
-import asyncio
-from datetime import date
-import nltk
 from unidecode import unidecode
-from scipy.special import softmax
 from transformers import AutoTokenizer
 import yaml
 import fitz
-import os
 
 
 def remove_accents(input_str):
@@ -48,9 +31,6 @@ def update_character_count(text):
     return f"{len(text)} characters"
 
 
-nltk.download("punkt")
-
-
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 
@@ -77,4 +57,4 @@ def extract_text_from_pdf(pdf_path):
 
 
 WORD = re.compile(r"\w+")
-model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")