added 1on1 model results to "Human vs AI Source Models"

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/analysis.cpython-311.pyc
+__pycache__/app.cpython-311.pyc
+__pycache__/explainability.cpython-311.pyc
+__pycache__/plagiarism.cpython-311.pyc
+__pycache__/predictors.cpython-311.pyc
+__pycache__/utils.cpython-311.pyc

analysis.py

@@ -22,12 +22,10 @@ import yaml
 import nltk
 import os
 from explainability import *
-from dotenv import load_dotenv
 import subprocess
 
 nltk.download("punkt")
 nltk.download("stopwords")
-load_dotenv()
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 device = "cuda" if torch.cuda.is_available() else "cpu"

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import numpy as np
 from datetime import date
-from predictors import predict_bc_scores, predict_mc_scores
+from predictors import predict_bc_scores, predict_mc_scores, predict_1on1_scores
 from analysis import depth_analysis
 from predictors import predict_quillbot
 from plagiarism import plagiarism_check, build_date
@@ -12,11 +12,12 @@ np.set_printoptions(suppress=True)
 
 def ai_generated_test(option, input):
     if option == "Human vs AI":
-        return predict_bc_scores(input), None
+        return predict_bc_scores(input), None, None
     else:
         return (
             predict_bc_scores(input),
             predict_mc_scores(input),
+            predict_1on1_scores(input),
         )
 
 
@@ -49,11 +50,13 @@ def main(
     depth_analysis_plot = depth_analysis(input)
     bc_score = predict_bc_scores(input)
     mc_score = predict_mc_scores(input)
+    mc_1on1_score = predict_1on1_scores(input)
     quilscore = predict_quillbot(input)
 
     return (
         bc_score,
         mc_score,
+        mc_1on1_score,
         formatted_tokens,
         depth_analysis_plot,
         quilscore,
@@ -147,6 +150,8 @@ with gr.Blocks() as demo:
             bcLabel = gr.Label(label="Source")
         with gr.Column():
             mcLabel = gr.Label(label="Creator")
+        with gr.Column():
+            mc1on1Label = gr.Label(label="Creator(1 on 1 Approach)")
     with gr.Row():
         QLabel = gr.Label(label="Humanized")
     with gr.Group():
@@ -213,6 +218,7 @@ with gr.Blocks() as demo:
         outputs=[
             bcLabel,
             mcLabel,
+            mc1on1Label,
             sentenceBreakdown,
             writing_analysis_plot,
             QLabel,
@@ -223,10 +229,7 @@ with gr.Blocks() as demo:
     only_ai_btn.click(
         fn=ai_generated_test,
         inputs=[ai_option, input_text],
-        outputs=[
-            bcLabel,
-            mcLabel,
-        ],
+        outputs=[bcLabel, mcLabel, mc1on1Label],
         api_name="ai_check",
     )
 
@@ -266,4 +269,6 @@ with gr.Blocks() as demo:
     date_from = ""
     date_to = ""
 
-demo.launch(share=True, auth=("polygraf-admin", "test@aisd"))
+
+if __name__ == "__main__":
+    demo.launch(share=True, auth=("polygraf-admin", "test@aisd"))

predictors.py

@@ -19,19 +19,19 @@ from scipy.special import softmax
 import yaml
 import os
 from utils import *
-from dotenv import load_dotenv
 
 with open("config.yaml", "r") as file:
     params = yaml.safe_load(file)
 nltk.download("punkt")
 nltk.download("stopwords")
-load_dotenv()
 device = "cuda" if torch.cuda.is_available() else "cpu"
 text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
 text_mc_model_path = params["TEXT_MC_MODEL_PATH"]
 text_quillbot_model_path = params["TEXT_QUILLBOT_MODEL_PATH"]
+text_1on1_models = params["TEXT_1ON1_MODEL"]
 quillbot_labels = params["QUILLBOT_LABELS"]
 mc_label_map = params["MC_OUTPUT_LABELS"]
+text_1on1_label_map = params["1ON1_OUTPUT_LABELS"]
 mc_token_size = int(params["MC_TOKEN_SIZE"])
 bc_token_size = int(params["BC_TOKEN_SIZE"])
 text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
@@ -46,6 +46,13 @@ quillbot_tokenizer = AutoTokenizer.from_pretrained(text_quillbot_model_path)
 quillbot_model = AutoModelForSequenceClassification.from_pretrained(
     text_quillbot_model_path
 ).to(device)
+tokenizers_1on1 = {}
+models_1on1 = {}
+for model in text_1on1_models:
+    tokenizers_1on1[model] = AutoTokenizer.from_pretrained(model)
+    models_1on1[model] = AutoModelForSequenceClassification.from_pretrained(
+        model
+    ).to(device)
 
 
 def split_text_allow_complete_sentences_nltk(
@@ -234,13 +241,69 @@ def predict_bc_scores(input):
     return bc_score
 
 
-# def predict_1on1(input):
-#     models = ['bard', 'claude', 'gpt4', 'mistral_ai', 'llama2']
-#     text = str(row["text"])
-#     predictions = {}
-#     prediction = predict(text, bard_model, bard_tokenizer) predictions['bard'] = prediction[1]
-#     prediction = predict(text, claude_model, claude_tokenizer) predictions['claude'] = prediction[1]
-#     prediction = predict(text, gpt4_model, gpt4_tokenizer) predictions['gpt4'] = prediction[1]
-#     prediction = predict(text, mistral_ai_model, mistral_ai_tokenizer) predictions['mistral_ai'] = prediction[1]
-#     prediction = predict(text, llama2_model, llama2_tokenizer) predictions['llama2'] = prediction[1]
-#     max_key = max(predictions, key=predictions.get)
+def predict_1on1(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt",
+            max_length=mc_token_size,
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_1on1_combined(input):
+    predictions = []
+    for i, model in enumerate(text_1on1_models):
+        predictions.append(
+            predict_1on1(models_1on1[model], tokenizers_1on1[model], input)[1]
+        )
+    return predictions
+
+
+def predict_1on1_scores(input):
+    # BC SCORE
+    bc_scores = []
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+
+    # MC SCORE
+    mc_scores = []
+    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
+    for i in range(samples_len_mc):
+        cleaned_text_mc = remove_special_characters(segments_mc[i])
+        mc_score = predict_1on1_combined(cleaned_text_mc)
+        mc_scores.append(mc_score)
+    mc_scores_array = np.array(mc_scores)
+    average_mc_scores = np.mean(mc_scores_array, axis=0)
+    normalized_mc_scores = average_mc_scores / np.sum(average_mc_scores)
+    mc_score_list = normalized_mc_scores.tolist()
+    mc_score = {}
+    for score, label in zip(mc_score_list, text_1on1_label_map):
+        mc_score[label.upper()] = score
+
+    print(mc_score)
+    sum_prob = 1 - bc_score["HUMAN"]
+    for key, value in mc_score.items():
+        mc_score[key] = value * sum_prob
+    if sum_prob < 0.01:
+        mc_score = {}
+
+    return mc_score