Merge branch 'main' into demo

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/analysis.cpython-311.pyc
+__pycache__/app.cpython-311.pyc
+__pycache__/explainability.cpython-311.pyc
+__pycache__/plagiarism.cpython-311.pyc
+__pycache__/predictors.cpython-311.pyc
+__pycache__/utils.cpython-311.pyc

analysis.py

@@ -22,12 +22,10 @@ import yaml
 import nltk
 import os
 from explainability import *
-from dotenv import load_dotenv
 import subprocess
 
 nltk.download("punkt")
 nltk.download("stopwords")
-load_dotenv()
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 device = "cuda" if torch.cuda.is_available() else "cpu"

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import numpy as np
 from datetime import date
-from predictors import predict_bc_scores, predict_mc_scores
+from predictors import predict_bc_scores, predict_mc_scores, predict_1on1_scores
 from analysis import depth_analysis
 from predictors import predict_quillbot
 from plagiarism import plagiarism_check, build_date
@@ -13,11 +13,12 @@ np.set_printoptions(suppress=True)
 def ai_generated_test(option, input):
     if option == "Human vs AI":
         return predict_bc_scores(input), None
-    else:
-        return (
-            predict_bc_scores(input),
-            predict_mc_scores(input),
-        )
+    elif option == "Human vs AI Source Models":
+        return predict_bc_scores(input), predict_mc_scores(input)
+    # elif option == "Human vs AI Source Models (1 on 1)":
+    #     return predict_bc_scores(input), None, predict_1on1_scores(input)
+
+    return None, None
 
 
 # COMBINED
@@ -49,11 +50,13 @@ def main(
     depth_analysis_plot = depth_analysis(input)
     bc_score = predict_bc_scores(input)
     mc_score = predict_mc_scores(input)
+    mc_1on1_score = predict_1on1_scores(input)
     quilscore = predict_quillbot(input)
 
     return (
         bc_score,
         mc_score,
+        mc_1on1_score,
         formatted_tokens,
         depth_analysis_plot,
         quilscore,
@@ -105,7 +108,11 @@ with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column():
             ai_option = gr.Radio(
-                ["Human vs AI", "Human vs AI Source Models"],
+                [
+                    "Human vs AI",
+                    "Human vs AI Source Models",
+                    # "Human vs AI Source Models (1 on 1)",
+                ],
                 label="Choose an option please.",
             )
         with gr.Column():
@@ -147,6 +154,8 @@ with gr.Blocks() as demo:
             bcLabel = gr.Label(label="Source")
         with gr.Column():
             mcLabel = gr.Label(label="Creator")
+        # with gr.Column():
+        #     mc1on1Label = gr.Label(label="Creator(1 on 1 Approach)")
     with gr.Row():
         QLabel = gr.Label(label="Humanized")
     with gr.Group():
@@ -213,6 +222,7 @@ with gr.Blocks() as demo:
         outputs=[
             bcLabel,
             mcLabel,
+            # mc1on1Label,
             sentenceBreakdown,
             writing_analysis_plot,
             QLabel,
@@ -223,10 +233,8 @@ with gr.Blocks() as demo:
     only_ai_btn.click(
         fn=ai_generated_test,
         inputs=[ai_option, input_text],
-        outputs=[
-            bcLabel,
-            mcLabel,
-        ],
+        # outputs=[bcLabel, mcLabel, mc1on1Label],
+        outputs=[bcLabel, mcLabel],
         api_name="ai_check",
     )
 

plagiarism.py

@@ -9,6 +9,7 @@ import httpx
 from bs4 import BeautifulSoup
 import numpy as np
 import concurrent
+from multiprocessing import Pool
 
 
 WORD = re.compile(r"\w+")
@@ -18,6 +19,7 @@ model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 # returns cosine similarity of two vectors
 # input: two vectors
 # output: integer between 0 and 1.
+
 def get_cosine(vec1, vec2):
     intersection = set(vec1.keys()) & set(vec2.keys())
 
@@ -130,7 +132,7 @@ def split_sentence_blocks(text):
     sents = sent_tokenize(text)
     two_sents = []
     for i in range(len(sents)):
-        if (i % 4) == 0:
+        if (i % 2) == 0:
             two_sents.append(sents[i])
         else:
             two_sents[len(two_sents) - 1] += " " + sents[i]
@@ -189,9 +191,9 @@ async def parallel_scrap(urls):
     return results
 
 
-def matching_score(args_list):
-    sentence = remove_punc(args_list[0])
-    content = remove_punc(args_list[1])
+
+def matching_score(sentence_content_tuple):
+    sentence, content = sentence_content_tuple
     if sentence in content:
         return 1
     else:
@@ -200,9 +202,13 @@ def matching_score(args_list):
         if len(ngrams) == 0:
             return 0
         matched = [x for x in ngrams if " ".join(x) in content]
-    return len(matched) / len(ngrams)
-
+        return len(matched) / len(ngrams)
 
+def process_with_multiprocessing(input_data):
+    with Pool(processes=4) as pool:
+        scores = pool.map(matching_score, input_data)
+    return scores
+    
 def plagiarism_check(
     plag_option,
     input,
@@ -244,55 +250,36 @@ def plagiarism_check(
     # Scrape URLs in list
     formatted_tokens = []
     soups = asyncio.run(parallel_scrap(urlList))
-
-    # Populate matching scores for scrapped pages
-    for i, soup in enumerate(soups):
-        print(f"Analyzing {i+1} of {len(soups)} soups........................")
-        if soup:
-            page_content = soup.text
-            for j, sent in enumerate(sentences):
-                args_list = (sent, page_content)
-                score = matching_score(args_list)
-                # score = cos_sim_torch(embed_text(sent), source_embeddings[i])
-                ScoreArray[i][j] = score
-
-    # with concurrent.futures.ProcessPoolExecutor() as executor:
-    #     results = executor.map(matching_score, args_list)
-
-    # *****IF THIS IS TO BE USED, PLEASE PROVIDE "preprocess()" FUNCTION IN LINE 248**************
-    # source_embeddings = []
+    
+    # # Populate matching scores for scrapped pages
     # for i, soup in enumerate(soups):
+    #     print(f"Analyzing {i+1} of {len(soups)} soups........................")
     #     if soup:
     #         page_content = soup.text
-    #         source_embeddings.append(embed_text(page_content))
-    #     else:
-    #        source_embeddings.append(None)
-
-    # def compute_cosine_similarity(args):
-    #     sent, source_embedding, i, j = args
-    #     score = cos_sim_torch(embed_text(sent), source_embedding)
-    #     return i, j, score
-
-    # def main(soups, sentences):
-    #     source_embeddings = [preprocess(soup) for soup in soups]
-    #     ScoreArray = [[0 for _ in sentences] for _ in soups]
-    #     args_list = []
-    #     for i, soup in enumerate(soups):
-    #         if soup:
-    #             for j, sent in enumerate(sentences):
-    #                 args_list.append((sent, source_embeddings[i], i, j))
-    #     with concurrent.futures.ProcessPoolExecutor() as executor:
-    #         results = executor.map(compute_cosine_similarity, args_list)
-    #         for i, j, score in results:
+            
+    #         for j, sent in enumerate(sentences):
+    #             args_list = (sent, page_content)
+    #             score = matching_score(args_list)
+    #             # score = cos_sim_torch(embed_text(sent), source_embeddings[i])
     #             ScoreArray[i][j] = score
-    #     return ScoreArray
 
-    # # Populate matching scores for scrapped pages
-    # ScoreArray = main(soups, sentences)
-    # *******************************************************************************************
+    input_data = []
+    for i, soup in enumerate(soups):
+        if soup:
+            page_content = soup.text
+            for j, sent in enumerate(sentences):
+                input_data.append((sent, page_content))
 
-    # Calculate URL of max matching score for each sentence chunk
+    scores = process_with_multiprocessing(input_data)
+    k = 0 
+    for i, soup in enumerate(soups):
+        if soup:
+            for j, _ in enumerate(sentences):
+                ScoreArray[i][j] = scores[k]
+                k += 1 
+            
     sentenceToMaxURL = [-1] * len(sentences)
+    
     for j in range(len(sentences)):
         if j > 0:
             maxScore = ScoreArray[sentenceToMaxURL[j - 1]][j]
@@ -326,10 +313,16 @@ def plagiarism_check(
     urlMap = {}
     for count, i in enumerate(index_descending):
         urlMap[i] = count + 1
+        
     for i, sent in enumerate(sentences):
         formatted_tokens.append(
             (sent, "[" + str(urlMap[sentenceToMaxURL[i]]) + "]")
         )
+        
+    formatted_tokens.append(("\n", None))
+    formatted_tokens.append(("\n", None))
+    formatted_tokens.append(("\n", None))
+    
     for ind in index_descending:
         formatted_tokens.append(
             (

predictors.py

@@ -19,19 +19,19 @@ from scipy.special import softmax
 import yaml
 import os
 from utils import *
-from dotenv import load_dotenv
 
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 nltk.download("punkt")
 nltk.download("stopwords")
-load_dotenv()
 device = "cuda" if torch.cuda.is_available() else "cpu"
 text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
 text_mc_model_path = params["TEXT_MC_MODEL_PATH"]
 text_quillbot_model_path = params["TEXT_QUILLBOT_MODEL_PATH"]
+text_1on1_models = params["TEXT_1ON1_MODEL"]
 quillbot_labels = params["QUILLBOT_LABELS"]
 mc_label_map = params["MC_OUTPUT_LABELS"]
+text_1on1_label_map = params["1ON1_OUTPUT_LABELS"]
 mc_token_size = int(params["MC_TOKEN_SIZE"])
 bc_token_size = int(params["BC_TOKEN_SIZE"])
 text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
@@ -46,6 +46,13 @@ quillbot_tokenizer = AutoTokenizer.from_pretrained(text_quillbot_model_path)
 quillbot_model = AutoModelForSequenceClassification.from_pretrained(
     text_quillbot_model_path
 ).to(device)
+# tokenizers_1on1 = {}
+# models_1on1 = {}
+# for model in text_1on1_models:
+#     tokenizers_1on1[model] = AutoTokenizer.from_pretrained(model)
+#     models_1on1[model] = AutoModelForSequenceClassification.from_pretrained(
+#         model
+#     ).to(device)
 
 
 def split_text_allow_complete_sentences_nltk(
@@ -234,13 +241,68 @@ def predict_bc_scores(input):
     return bc_score
 
 
-# def predict_1on1(input):
-#     models = ['bard', 'claude', 'gpt4', 'mistral_ai', 'llama2']
-#     text = str(row["text"])
-#     predictions = {}
-#     prediction = predict(text, bard_model, bard_tokenizer) predictions['bard'] = prediction[1]
-#     prediction = predict(text, claude_model, claude_tokenizer) predictions['claude'] = prediction[1]
-#     prediction = predict(text, gpt4_model, gpt4_tokenizer) predictions['gpt4'] = prediction[1]
-#     prediction = predict(text, mistral_ai_model, mistral_ai_tokenizer) predictions['mistral_ai'] = prediction[1]
-#     prediction = predict(text, llama2_model, llama2_tokenizer) predictions['llama2'] = prediction[1]
-#     max_key = max(predictions, key=predictions.get)
+def predict_1on1(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt",
+            max_length=mc_token_size,
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_1on1_combined(input):
+    predictions = []
+    for i, model in enumerate(text_1on1_models):
+        predictions.append(
+            predict_1on1(models_1on1[model], tokenizers_1on1[model], input)[1]
+        )
+    return predictions
+
+
+def predict_1on1_scores(input):
+    # BC SCORE
+    bc_scores = []
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+
+    # MC SCORE
+    mc_scores = []
+    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
+    for i in range(samples_len_mc):
+        cleaned_text_mc = remove_special_characters(segments_mc[i])
+        mc_score = predict_1on1_combined(cleaned_text_mc)
+        mc_scores.append(mc_score)
+    mc_scores_array = np.array(mc_scores)
+    average_mc_scores = np.mean(mc_scores_array, axis=0)
+    normalized_mc_scores = average_mc_scores / np.sum(average_mc_scores)
+    mc_score_list = normalized_mc_scores.tolist()
+    mc_score = {}
+    for score, label in zip(mc_score_list, text_1on1_label_map):
+        mc_score[label.upper()] = score
+
+    sum_prob = 1 - bc_score["HUMAN"]
+    for key, value in mc_score.items():
+        mc_score[key] = value * sum_prob
+    if sum_prob < 0.01:
+        mc_score = {}
+
+    return mc_score