refactoring

analysis.py

@@ -0,0 +1,98 @@
+import requests
+import httpx
+import torch
+import re
+from bs4 import BeautifulSoup
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import asyncio
+from scipy.special import softmax
+from evaluate import load
+from datetime import date
+import nltk
+import fitz
+from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+import nltk, spacy, subprocess, torch
+import plotly.graph_objects as go
+import torch.nn.functional as F
+import nltk
+from unidecode import unidecode
+import time
+import yaml
+import nltk
+import os
+from explainability import *
+from dotenv import load_dotenv
+import subprocess
+
+nltk.download("punkt")
+nltk.download("stopwords")
+load_dotenv()
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+device = "cuda" if torch.cuda.is_available() else "cpu"
+readability_model_id = params["READABILITY_MODEL_ID"]
+gpt2_model = GPT2LMHeadModel.from_pretrained(readability_model_id).to(device)
+gpt2_tokenizer = GPT2TokenizerFast.from_pretrained(readability_model_id)
+
+command = ["python", "-m", "spacy", "download", "en_core_web_sm"]
+subprocess.run(command)
+nlp = spacy.load("en_core_web_sm")
+
+
+def depth_analysis(input_text):
+    processed_words = preprocess_text1(input_text)
+    ttr_value = vocabulary_richness_ttr(processed_words)
+    gunning_fog = calculate_gunning_fog(input_text)
+    gunning_fog_norm = normalize(gunning_fog, min_value=0, max_value=20)
+    words, sentences = preprocess_text2(input_text)
+    average_sentence_length = calculate_average_sentence_length(sentences)
+    average_word_length = calculate_average_word_length(words)
+    average_sentence_length_norm = normalize(
+        average_sentence_length, min_value=0, max_value=40
+    )
+    average_word_length_norm = normalize(
+        average_word_length, min_value=0, max_value=8
+    )
+    average_tree_depth = calculate_syntactic_tree_depth(nlp, input_text)
+    average_tree_depth_norm = normalize(
+        average_tree_depth, min_value=0, max_value=10
+    )
+    perplexity = calculate_perplexity(
+        input_text, gpt2_model, gpt2_tokenizer, device
+    )
+    perplexity_norm = normalize(perplexity, min_value=0, max_value=30)
+
+    features = {
+        "readability": gunning_fog_norm,
+        "syntactic tree depth": average_tree_depth_norm,
+        "vocabulary richness": ttr_value,
+        "perplexity": perplexity_norm,
+        "average sentence length": average_sentence_length_norm,
+        "average word length": average_word_length_norm,
+    }
+    fig = go.Figure()
+    fig.add_trace(
+        go.Scatterpolar(
+            r=list(features.values()),
+            theta=list(features.keys()),
+            fill="toself",
+            name="Radar Plot",
+        )
+    )
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],
+            )
+        ),
+        showlegend=False,
+        margin=dict(
+            l=10,
+            r=20,
+            b=10,
+            t=10,
+        ),
+    )
+    return fig

app.py

@@ -1,286 +1,23 @@
-from utils import (
-    cosineSim,
-    googleSearch,
-    getSentences,
-    parallel_scrap,
-    matchingScore,
-)
 import gradio as gr
-from urllib.request import urlopen, Request
-from googleapiclient.discovery import build
-import requests
-import httpx
-import torch
-import re
-from bs4 import BeautifulSoup
 import numpy as np
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import asyncio
-from scipy.special import softmax
-from evaluate import load
 from datetime import date
-import nltk
-import fitz
-from transformers import GPT2LMHeadModel, GPT2TokenizerFast
-import nltk, spacy, subprocess, torch
-import plotly.graph_objects as go
-import torch.nn.functional as F
-import nltk
-from unidecode import unidecode
-import time
-from utils import cos_sim_torch, embed_text
-import multiprocessing
-from functools import partial
-import concurrent.futures
-from plagiarism import plagiarism_check
-
-nltk.download("punkt")
-
-from writing_analysis import (
-    normalize,
-    preprocess_text1,
-    preprocess_text2,
-    vocabulary_richness_ttr,
-    calculate_gunning_fog,
-    calculate_average_sentence_length,
-    calculate_average_word_length,
-    calculate_syntactic_tree_depth,
-    calculate_perplexity,
-)
+from predictors import predict_bc_scores, predict_mc_scores
+from analysis import depth_analysis
+from predictors import predict_quillbot
+from plagiarism import plagiarism_check, build_date
+from utils import extract_text_from_pdf, len_validator
 
 np.set_printoptions(suppress=True)
 
 
-"""
-AI DETECTION SECTION
-"""
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-text_bc_model_path = "polygraf-ai/text-detect-bc-v11-4m"
-text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
-text_bc_model = AutoModelForSequenceClassification.from_pretrained(
-    text_bc_model_path
-).to(device)
-
-text_mc_model_path = (
-    "polygraf-ai/ai-text-detection-mc-robert-open-ai-detector-v4"
-)
-text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
-text_mc_model = AutoModelForSequenceClassification.from_pretrained(
-    text_mc_model_path
-).to(device)
-
-quillbot_labels = ["Original", "QuillBot"]
-quillbot_tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base")
-quillbot_model = AutoModelForSequenceClassification.from_pretrained(
-    "polygraf-ai/quillbot-detector-28k"
-).to(device)
-
-
-def remove_accents(input_str):
-    text_no_accents = unidecode(input_str)
-    return text_no_accents
-
-
-def remove_special_characters(text):
-    text = remove_accents(text)
-    pattern = r'[^\w\s\d.,!?\'"()-;]+'
-    text = re.sub(pattern, "", text)
-    return text
-
-
-def remove_special_characters_2(text):
-    pattern = r"[^a-zA-Z0-9 ]+"
-    text = re.sub(pattern, "", text)
-    return text
-
-
-def update_character_count(text):
-    return f"{len(text)} characters"
-
-
-def split_text_allow_complete_sentences_nltk(
-    text,
-    max_length=256,
-    tolerance=30,
-    min_last_segment_length=100,
-    type_det="bc",
-):
-    sentences = nltk.sent_tokenize(text)
-    segments = []
-    current_segment = []
-    current_length = 0
-
-    if type_det == "bc":
-        tokenizer = text_bc_tokenizer
-        max_length = 333
-
-    elif type_det == "mc":
-        tokenizer = text_mc_tokenizer
-        max_length = 256
-
-    for sentence in sentences:
-        tokens = tokenizer.tokenize(sentence)
-        sentence_length = len(tokens)
-
-        if current_length + sentence_length <= max_length + tolerance - 2:
-            current_segment.append(sentence)
-            current_length += sentence_length
-        else:
-            if current_segment:
-                encoded_segment = tokenizer.encode(
-                    " ".join(current_segment),
-                    add_special_tokens=True,
-                    max_length=max_length + tolerance,
-                    truncation=True,
-                )
-                segments.append((current_segment, len(encoded_segment)))
-            current_segment = [sentence]
-            current_length = sentence_length
-
-    if current_segment:
-        encoded_segment = tokenizer.encode(
-            " ".join(current_segment),
-            add_special_tokens=True,
-            max_length=max_length + tolerance,
-            truncation=True,
-        )
-        segments.append((current_segment, len(encoded_segment)))
-
-    final_segments = []
-    for i, (seg, length) in enumerate(segments):
-        if i == len(segments) - 1:
-            if length < min_last_segment_length and len(final_segments) > 0:
-                prev_seg, prev_length = final_segments[-1]
-                combined_encoded = tokenizer.encode(
-                    " ".join(prev_seg + seg),
-                    add_special_tokens=True,
-                    max_length=max_length + tolerance,
-                    truncation=True,
-                )
-                if len(combined_encoded) <= max_length + tolerance:
-                    final_segments[-1] = (prev_seg + seg, len(combined_encoded))
-                else:
-                    final_segments.append((seg, length))
-            else:
-                final_segments.append((seg, length))
-        else:
-            final_segments.append((seg, length))
-
-    decoded_segments = []
-    encoded_segments = []
-    for seg, _ in final_segments:
-        encoded_segment = tokenizer.encode(
-            " ".join(seg),
-            add_special_tokens=True,
-            max_length=max_length + tolerance,
-            truncation=True,
-        )
-        decoded_segment = tokenizer.decode(encoded_segment)
-        decoded_segments.append(decoded_segment)
-    return decoded_segments
-
-
-def predict_quillbot(text):
-    with torch.no_grad():
-        quillbot_model.eval()
-        tokenized_text = quillbot_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            max_length=256,
-            return_tensors="pt",
-        ).to(device)
-        output = quillbot_model(**tokenized_text)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        q_score = {
-            "QuillBot": output_norm[1].item(),
-            "Original": output_norm[0].item(),
-        }
-        return q_score
-
-
-def predict_bc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_bc_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            max_length=333,
-            return_tensors="pt",
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        print("BC Score: ", output_norm)
-        return output_norm
-
-
-def predict_mc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_mc_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            return_tensors="pt",
-            max_length=256,
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        print("MC Score: ", output_norm)
-        return output_norm
-
-
-def ai_generated_test(ai_option, input):
-
-    bc_scores = []
-    mc_scores = []
-    samples_len_bc = len(
-        split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    )
-    samples_len_mc = len(
-        split_text_allow_complete_sentences_nltk(input, type_det="mc")
-    )
-    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
-
-    for i in range(samples_len_bc):
-        cleaned_text_bc = remove_special_characters(segments_bc[i])
-        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
-        bc_scores.append(bc_score)
-
-    for i in range(samples_len_mc):
-        cleaned_text_mc = remove_special_characters(segments_mc[i])
-        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
-        mc_scores.append(mc_score)
-
-    bc_scores_array = np.array(bc_scores)
-    mc_scores_array = np.array(mc_scores)
-    average_bc_scores = np.mean(bc_scores_array, axis=0)
-    average_mc_scores = np.mean(mc_scores_array, axis=0)
-    bc_score_list = average_bc_scores.tolist()
-    mc_score_list = average_mc_scores.tolist()
-
-    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
-    mc_score = {}
-    label_map = ["OpenAI GPT", "Mistral", "CLAUDE", "Gemini", "LLAMA 2"]
-
-    for score, label in zip(mc_score_list, label_map):
-        mc_score[label.upper()] = score
-
-    sum_prob = 1 - bc_score["HUMAN"]
-    for key, value in mc_score.items():
-        mc_score[key] = value * sum_prob
-
-    if ai_option == "Human vs AI":
-        mc_score = {}
-
-    if sum_prob < 0.01:
-        mc_score = {}
-        return bc_score, mc_score
+def ai_generated_test(option, input):
+    if option == "Human vs AI":
+        return predict_bc_scores(input), None
     else:
-        return bc_score, mc_score
+        return (
+            predict_bc_scores(input),
+            predict_mc_scores(input),
+        )
 
 
 # COMBINED
@@ -310,7 +47,8 @@ def main(
         domains_to_skip,
     )
     depth_analysis_plot = depth_analysis(input)
-    bc_score, mc_score = ai_generated_test(ai_option, input)
+    bc_score = predict_bc_scores(input)
+    mc_score = predict_mc_scores(input)
     quilscore = predict_quillbot(input)
 
     return (
@@ -322,120 +60,6 @@ def main(
     )
 
 
-def build_date(year, month, day):
-    return f"{year}{months[month]}{day}"
-
-
-def len_validator(text):
-    min_tokens = 200
-    lengt = len(text_bc_tokenizer.tokenize(text=text, return_tensors="pt"))
-    if lengt < min_tokens:
-        return f"Warning! Input length is {lengt}. Please input a text that is greater than {min_tokens} tokens long. Recommended length {min_tokens*2} tokens."
-    else:
-        return f"Input length ({lengt}) is satisified."
-
-
-def extract_text_from_pdf(pdf_path):
-    doc = fitz.open(pdf_path)
-    text = ""
-    for page in doc:
-        text += page.get_text()
-    return text
-
-
-# DEPTH ANALYSIS
-print("loading depth analysis")
-nltk.download("stopwords")
-nltk.download("punkt")
-command = ["python3", "-m", "spacy", "download", "en_core_web_sm"]
-# Execute the command
-subprocess.run(command)
-nlp = spacy.load("en_core_web_sm")
-
-# for perplexity
-model_id = "gpt2"
-gpt2_model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
-gpt2_tokenizer = GPT2TokenizerFast.from_pretrained(model_id)
-
-
-def depth_analysis(input_text):
-
-    # vocanulary richness
-    processed_words = preprocess_text1(input_text)
-    ttr_value = vocabulary_richness_ttr(processed_words)
-
-    # readability
-    gunning_fog = calculate_gunning_fog(input_text)
-    gunning_fog_norm = normalize(gunning_fog, min_value=0, max_value=20)
-
-    # average sentence length and average word length
-    words, sentences = preprocess_text2(input_text)
-    average_sentence_length = calculate_average_sentence_length(sentences)
-    average_word_length = calculate_average_word_length(words)
-    average_sentence_length_norm = normalize(
-        average_sentence_length, min_value=0, max_value=40
-    )
-    average_word_length_norm = normalize(
-        average_word_length, min_value=0, max_value=8
-    )
-
-    # syntactic_tree_depth
-    average_tree_depth = calculate_syntactic_tree_depth(nlp, input_text)
-    average_tree_depth_norm = normalize(
-        average_tree_depth, min_value=0, max_value=10
-    )
-
-    # perplexity
-    perplexity = calculate_perplexity(
-        input_text, gpt2_model, gpt2_tokenizer, device
-    )
-    perplexity_norm = normalize(perplexity, min_value=0, max_value=30)
-
-    features = {
-        "readability": gunning_fog_norm,
-        "syntactic tree depth": average_tree_depth_norm,
-        "vocabulary richness": ttr_value,
-        "perplexity": perplexity_norm,
-        "average sentence length": average_sentence_length_norm,
-        "average word length": average_word_length_norm,
-    }
-
-    print(features)
-
-    fig = go.Figure()
-
-    fig.add_trace(
-        go.Scatterpolar(
-            r=list(features.values()),
-            theta=list(features.keys()),
-            fill="toself",
-            name="Radar Plot",
-        )
-    )
-
-    fig.update_layout(
-        polar=dict(
-            radialaxis=dict(
-                visible=True,
-                range=[0, 100],
-            )
-        ),
-        showlegend=False,
-        # autosize=False,
-        # width=600,
-        # height=600,
-        margin=dict(
-            l=10,
-            r=20,
-            b=10,
-            t=10,
-            # pad=100
-        ),
-    )
-
-    return fig
-
-
 # START OF GRADIO
 
 title = "Copyright Checker"
@@ -497,7 +121,7 @@ with gr.Blocks() as demo:
             only_plagiarism_btn = gr.Button("Source Check")
 
     with gr.Row():
-        quillbot_check = gr.Button("Humanized Text Check (Quillbot)")
+        quillbot_check = gr.Button("Humanized Text Check")
 
     with gr.Row():
         depth_analysis_btn = gr.Button("Detailed Writing Analysis")
@@ -642,6 +266,4 @@ with gr.Blocks() as demo:
     date_from = ""
     date_to = ""
 
-demo.launch(
-    share=True, server_name="0.0.0.0", auth=("polygraf-admin", "test@aisd")
-)
+demo.launch(share=True, auth=("polygraf-admin", "test@aisd"))

explainability.py

@@ -0,0 +1,119 @@
+import re, textstat
+from nltk import FreqDist
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize, sent_tokenize
+import torch
+import nltk
+from tqdm import tqdm
+
+nltk.download("punkt")
+
+
+def normalize(value, min_value, max_value):
+    normalized_value = ((value - min_value) * 100) / (max_value - min_value)
+    return max(0, min(100, normalized_value))
+
+
+def preprocess_text1(text):
+    text = text.lower()
+    text = re.sub(r"[^\w\s]", "", text)  # remove punctuation
+    stop_words = set(stopwords.words("english"))  # remove stopwords
+    words = [word for word in text.split() if word not in stop_words]
+    words = [word for word in words if not word.isdigit()]  # remove numbers
+    return words
+
+
+def vocabulary_richness_ttr(words):
+    unique_words = set(words)
+    ttr = len(unique_words) / len(words) * 100
+    return ttr
+
+
+def calculate_gunning_fog(text):
+    """range 0-20"""
+    gunning_fog = textstat.gunning_fog(text)
+    return gunning_fog
+
+
+def calculate_automated_readability_index(text):
+    """range 1-20"""
+    ari = textstat.automated_readability_index(text)
+    return ari
+
+
+def calculate_flesch_reading_ease(text):
+    """range 0-100"""
+    fre = textstat.flesch_reading_ease(text)
+    return fre
+
+
+def preprocess_text2(text):
+    sentences = sent_tokenize(text)
+    words = [
+        word.lower()
+        for sent in sentences
+        for word in word_tokenize(sent)
+        if word.isalnum()
+    ]
+    stop_words = set(stopwords.words("english"))
+    words = [word for word in words if word not in stop_words]
+    return words, sentences
+
+
+def calculate_average_sentence_length(sentences):
+    """range 0-40 or 50 based on the histogram"""
+    total_words = sum(len(word_tokenize(sent)) for sent in sentences)
+    average_sentence_length = total_words / (len(sentences) + 0.0000001)
+    return average_sentence_length
+
+
+def calculate_average_word_length(words):
+    """range 0-8 based on the histogram"""
+    total_characters = sum(len(word) for word in words)
+    average_word_length = total_characters / (len(words) + 0.0000001)
+    return average_word_length
+
+
+def calculate_max_depth(sent):
+    return max(len(list(token.ancestors)) for token in sent)
+
+
+def calculate_syntactic_tree_depth(nlp, text):
+    """0-10 based on the histogram"""
+    doc = nlp(text)
+    sentence_depths = [calculate_max_depth(sent) for sent in doc.sents]
+    average_depth = (
+        sum(sentence_depths) / len(sentence_depths) if sentence_depths else 0
+    )
+    return average_depth
+
+
+def calculate_perplexity(text, model, tokenizer, device, stride=512):
+    """range 0-30 based on the histogram"""
+    encodings = tokenizer(text, return_tensors="pt")
+    max_length = model.config.n_positions
+    seq_len = encodings.input_ids.size(1)
+
+    nlls = []
+    prev_end_loc = 0
+    for begin_loc in tqdm(range(0, seq_len, stride)):
+        end_loc = min(begin_loc + max_length, seq_len)
+        trg_len = (
+            end_loc - prev_end_loc
+        )  # may be different from stride on last loop
+        input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
+        target_ids = input_ids.clone()
+        target_ids[:, :-trg_len] = -100
+
+        with torch.no_grad():
+            outputs = model(input_ids, labels=target_ids)
+            neg_log_likelihood = outputs.loss
+
+        nlls.append(neg_log_likelihood)
+
+        prev_end_loc = end_loc
+        if end_loc == seq_len:
+            break
+
+    ppl = torch.exp(torch.stack(nlls).mean())
+    return ppl.item()

plagiarism.py

@@ -8,6 +8,7 @@ import asyncio
 import httpx
 from bs4 import BeautifulSoup
 import numpy as np
+import concurrent
 
 
 WORD = re.compile(r"\w+")
@@ -129,7 +130,7 @@ def split_sentence_blocks(text):
     sents = sent_tokenize(text)
     two_sents = []
     for i in range(len(sents)):
-        if (i % 2) == 0:
+        if (i % 4) == 0:
             two_sents.append(sents[i])
         else:
             two_sents[len(two_sents) - 1] += " " + sents[i]
@@ -188,9 +189,9 @@ async def parallel_scrap(urls):
     return results
 
 
-def matching_score(sentence, content):
-    sentence = remove_punc(sentence)
-    content = remove_punc(content)
+def matching_score(args_list):
+    sentence = remove_punc(args_list[0])
+    content = remove_punc(args_list[1])
     if sentence in content:
         return 1
     else:
@@ -250,11 +251,14 @@ def plagiarism_check(
         if soup:
             page_content = soup.text
             for j, sent in enumerate(sentences):
-                score = matching_score(sent, page_content)
-                score = matching_score(sent, page_content)
+                args_list = (sent, page_content)
+                score = matching_score(args_list)
                 # score = cos_sim_torch(embed_text(sent), source_embeddings[i])
                 ScoreArray[i][j] = score
 
+    # with concurrent.futures.ProcessPoolExecutor() as executor:
+    #     results = executor.map(matching_score, args_list)
+
     # *****IF THIS IS TO BE USED, PLEASE PROVIDE "preprocess()" FUNCTION IN LINE 248**************
     # source_embeddings = []
     # for i, soup in enumerate(soups):

predictors.py

@@ -0,0 +1,246 @@
+import requests
+import httpx
+import torch
+import re
+from bs4 import BeautifulSoup
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import asyncio
+from evaluate import load
+from datetime import date
+import nltk
+from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+import plotly.graph_objects as go
+import torch.nn.functional as F
+import nltk
+from unidecode import unidecode
+import time
+from scipy.special import softmax
+import yaml
+import os
+from utils import *
+from dotenv import load_dotenv
+
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+nltk.download("punkt")
+nltk.download("stopwords")
+load_dotenv()
+device = "cuda" if torch.cuda.is_available() else "cpu"
+text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
+text_mc_model_path = params["TEXT_MC_MODEL_PATH"]
+text_quillbot_model_path = params["TEXT_QUILLBOT_MODEL_PATH"]
+quillbot_labels = params["QUILLBOT_LABELS"]
+mc_label_map = params["MC_OUTPUT_LABELS"]
+mc_token_size = int(params["MC_TOKEN_SIZE"])
+bc_token_size = int(params["BC_TOKEN_SIZE"])
+text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
+text_bc_model = AutoModelForSequenceClassification.from_pretrained(
+    text_bc_model_path
+).to(device)
+text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
+text_mc_model = AutoModelForSequenceClassification.from_pretrained(
+    text_mc_model_path
+).to(device)
+quillbot_tokenizer = AutoTokenizer.from_pretrained(text_quillbot_model_path)
+quillbot_model = AutoModelForSequenceClassification.from_pretrained(
+    text_quillbot_model_path
+).to(device)
+
+
+def split_text_allow_complete_sentences_nltk(
+    text,
+    max_length=256,
+    tolerance=30,
+    min_last_segment_length=100,
+    type_det="bc",
+):
+    sentences = nltk.sent_tokenize(text)
+    segments = []
+    current_segment = []
+    current_length = 0
+    if type_det == "bc":
+        tokenizer = text_bc_tokenizer
+        max_length = bc_token_size
+    elif type_det == "mc":
+        tokenizer = text_mc_tokenizer
+        max_length = mc_token_size
+    for sentence in sentences:
+        tokens = tokenizer.tokenize(sentence)
+        sentence_length = len(tokens)
+
+        if current_length + sentence_length <= max_length + tolerance - 2:
+            current_segment.append(sentence)
+            current_length += sentence_length
+        else:
+            if current_segment:
+                encoded_segment = tokenizer.encode(
+                    " ".join(current_segment),
+                    add_special_tokens=True,
+                    max_length=max_length + tolerance,
+                    truncation=True,
+                )
+                segments.append((current_segment, len(encoded_segment)))
+            current_segment = [sentence]
+            current_length = sentence_length
+
+    if current_segment:
+        encoded_segment = tokenizer.encode(
+            " ".join(current_segment),
+            add_special_tokens=True,
+            max_length=max_length + tolerance,
+            truncation=True,
+        )
+        segments.append((current_segment, len(encoded_segment)))
+
+    final_segments = []
+    for i, (seg, length) in enumerate(segments):
+        if i == len(segments) - 1:
+            if length < min_last_segment_length and len(final_segments) > 0:
+                prev_seg, prev_length = final_segments[-1]
+                combined_encoded = tokenizer.encode(
+                    " ".join(prev_seg + seg),
+                    add_special_tokens=True,
+                    max_length=max_length + tolerance,
+                    truncation=True,
+                )
+                if len(combined_encoded) <= max_length + tolerance:
+                    final_segments[-1] = (prev_seg + seg, len(combined_encoded))
+                else:
+                    final_segments.append((seg, length))
+            else:
+                final_segments.append((seg, length))
+        else:
+            final_segments.append((seg, length))
+
+    decoded_segments = []
+    encoded_segments = []
+    for seg, _ in final_segments:
+        encoded_segment = tokenizer.encode(
+            " ".join(seg),
+            add_special_tokens=True,
+            max_length=max_length + tolerance,
+            truncation=True,
+        )
+        decoded_segment = tokenizer.decode(encoded_segment)
+        decoded_segments.append(decoded_segment)
+    return decoded_segments
+
+
+def predict_quillbot(text):
+    with torch.no_grad():
+        quillbot_model.eval()
+        tokenized_text = quillbot_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            max_length=256,
+            return_tensors="pt",
+        ).to(device)
+        output = quillbot_model(**tokenized_text)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        q_score = {
+            "Humanized": output_norm[1].item(),
+            "Original": output_norm[0].item(),
+        }
+        return q_score
+
+
+def predict_bc(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = text_bc_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            max_length=bc_token_size,
+            return_tensors="pt",
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_mc(model, tokenizer, text):
+    with torch.no_grad():
+        model.eval()
+        tokens = text_mc_tokenizer(
+            text,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt",
+            max_length=mc_token_size,
+        ).to(device)
+        output = model(**tokens)
+        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
+        return output_norm
+
+
+def predict_mc_scores(input):
+    bc_scores = []
+    mc_scores = []
+
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+    segments_mc = split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    samples_len_mc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    )
+    for i in range(samples_len_mc):
+        cleaned_text_mc = remove_special_characters(segments_mc[i])
+        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
+        mc_scores.append(mc_score)
+    mc_scores_array = np.array(mc_scores)
+    average_mc_scores = np.mean(mc_scores_array, axis=0)
+    mc_score_list = average_mc_scores.tolist()
+    mc_score = {}
+    for score, label in zip(mc_score_list, mc_label_map):
+        mc_score[label.upper()] = score
+
+    sum_prob = 1 - bc_score["HUMAN"]
+    for key, value in mc_score.items():
+        mc_score[key] = value * sum_prob
+    if sum_prob < 0.01:
+        mc_score = {}
+
+    return mc_score
+
+
+def predict_bc_scores(input):
+    bc_scores = []
+    mc_scores = []
+    samples_len_bc = len(
+        split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    )
+    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
+    for i in range(samples_len_bc):
+        cleaned_text_bc = remove_special_characters(segments_bc[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
+        bc_scores.append(bc_score)
+    bc_scores_array = np.array(bc_scores)
+    average_bc_scores = np.mean(bc_scores_array, axis=0)
+    bc_score_list = average_bc_scores.tolist()
+    bc_score = {"AI": bc_score_list[1], "HUMAN": bc_score_list[0]}
+    return bc_score
+
+
+# def predict_1on1(input):
+#     models = ['bard', 'claude', 'gpt4', 'mistral_ai', 'llama2']
+#     text = str(row["text"])
+#     predictions = {}
+#     prediction = predict(text, bard_model, bard_tokenizer) predictions['bard'] = prediction[1]
+#     prediction = predict(text, claude_model, claude_tokenizer) predictions['claude'] = prediction[1]
+#     prediction = predict(text, gpt4_model, gpt4_tokenizer) predictions['gpt4'] = prediction[1]
+#     prediction = predict(text, mistral_ai_model, mistral_ai_tokenizer) predictions['mistral_ai'] = prediction[1]
+#     prediction = predict(text, llama2_model, llama2_tokenizer) predictions['llama2'] = prediction[1]
+#     max_key = max(predictions, key=predictions.get)

requirements.txt

@@ -6,8 +6,8 @@ BeautifulSoup4
 scrapingbee
 requests
 numpy
-torch==1.13.0
-transformers==4.25.1
+torch
+transformers
 transformers-interpret 
 textstat
 scipy

utils.py

@@ -11,284 +11,354 @@ import asyncio
 import nltk
 from sentence_transformers import SentenceTransformer, util
 import threading
+import torch
+import re
+import numpy as np
+import asyncio
+from datetime import date
+import nltk
+from unidecode import unidecode
+from scipy.special import softmax
+from transformers import AutoTokenizer
+import yaml
+import fitz
+import os
+
+
+def remove_accents(input_str):
+    text_no_accents = unidecode(input_str)
+    return text_no_accents
+
+
+def remove_special_characters(text):
+    text = remove_accents(text)
+    pattern = r'[^\w\s\d.,!?\'"()-;]+'
+    text = re.sub(pattern, "", text)
+    return text
+
+
+def remove_special_characters_2(text):
+    pattern = r"[^a-zA-Z0-9 ]+"
+    text = re.sub(pattern, "", text)
+    return text
+
+
+def update_character_count(text):
+    return f"{len(text)} characters"
+
+
+nltk.download("punkt")
+
+
+with open("config.yaml", "r") as file:
+    params = yaml.safe_load(file)
+
+text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
+
+text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
+
+
+def len_validator(text):
+    min_tokens = 200
+    lengt = len(text_bc_tokenizer.tokenize(text=text, return_tensors="pt"))
+    if lengt < min_tokens:
+        return f"Warning! Input length is {lengt}. Please input a text that is greater than {min_tokens} tokens long. Recommended length {min_tokens*2} tokens."
+    else:
+        return f"Input length ({lengt}) is satisified."
+
+
+def extract_text_from_pdf(pdf_path):
+    doc = fitz.open(pdf_path)
+    text = ""
+    for page in doc:
+        text += page.get_text()
+    return text
 
-nltk.download('punkt')
 
 WORD = re.compile(r"\w+")
-model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
 
 # returns cosine similarity of two vectors
 # input: two vectors
 # output: integer between 0 and 1.
-def get_cosine(vec1, vec2):
-    intersection = set(vec1.keys()) & set(vec2.keys())
+# def get_cosine(vec1, vec2):
+#     intersection = set(vec1.keys()) & set(vec2.keys())
 
-    # calculating numerator
-    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+#     # calculating numerator
+#     numerator = sum([vec1[x] * vec2[x] for x in intersection])
 
-    # calculating denominator
-    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
-    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
-    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+#     # calculating denominator
+#     sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+#     sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+#     denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+#     # checking for divide by zero
+#     if denominator == 0:
+#         return 0.0
+#     else:
+#         return float(numerator) / denominator
+
+
+# # converts given text into a vector
+# def text_to_vector(text):
+#     # uses the Regular expression above and gets all words
+#     words = WORD.findall(text)
+#     # returns a counter of all the words (count of number of occurences)
+#     return Counter(words)
 
-    # checking for divide by zero
-    if denominator == 0:
-        return 0.0
-    else:
-        return float(numerator) / denominator
-
-
-# converts given text into a vector
-def text_to_vector(text):
-    # uses the Regular expression above and gets all words
-    words = WORD.findall(text)
-    # returns a counter of all the words (count of number of occurences)
-    return Counter(words)
-
-
-# returns cosine similarity of two words
-# uses: text_to_vector(text) and get_cosine(v1,v2)
-def cosineSim(text1, text2):
-    vector1 = text_to_vector(text1)
-    vector2 = text_to_vector(text2)
-    # print vector1,vector2
-    cosine = get_cosine(vector1, vector2)
-    return cosine
-
-def cos_sim_torch(embedding_1, embedding_2):
-    return util.pytorch_cos_sim(embedding_1, embedding_2).item()
-
-def embed_text(text):
-    return model.encode(text, convert_to_tensor=True)
-
-def sentence_similarity(text1, text2):
-    embedding_1= model.encode(text1, convert_to_tensor=True)
-    embedding_2 = model.encode(text2, convert_to_tensor=True)
-
-    o = util.pytorch_cos_sim(embedding_1, embedding_2)
-    return o.item()
-
-def get_soup_requests(url):
-    page = requests.get(url)
-    if page.status_code == 200:
-        soup = BeautifulSoup(page.content, "html.parser")
-        return soup
-    print("HTML soup failed")
-    return None
-
-
-def get_soup_httpx(url):
-    client = httpx.Client(timeout=30)
-    try:
-        page = client.get(url)
-        if page.status_code == httpx.codes.OK:
-            soup = BeautifulSoup(page.content, "html.parser")
-            return soup
-    except:
-        print("HTTPx soup failed")
-        return None
-
-def getSentences(text):
-    from nltk.tokenize import sent_tokenize
-
-    sents = sent_tokenize(text)
-    two_sents = []
-    for i in range(len(sents)):
-        if (i % 2) == 0:
-            two_sents.append(sents[i])
-        else:
-            two_sents[len(two_sents) - 1] += " " + sents[i]
-    return two_sents
-
-
-def googleSearch(
-    plag_option,
-    sentences,
-    urlCount,
-    scoreArray,
-    urlList,
-    sorted_date,
-    domains_to_skip,
-    api_key,
-    cse_id,
-    **kwargs,
-):
-    service = build("customsearch", "v1", developerKey=api_key)
-    for i, sentence in enumerate(sentences):
-        results = (
-            service.cse()
-            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
-            .execute()
-        )
-        if "items" in results and len(results["items"]) > 0:
-            for count, link in enumerate(results["items"]):
-                # stop after 3 pages
-                if count >= 3:
-                    break
-                # skip user selected domains
-                if any(
-                    ("." + domain) in link["link"]
-                    for domain in domains_to_skip
-                ):
-                    continue
-                # clean up snippet of '...'
-                snippet = link["snippet"]
-                ind = snippet.find("...")
-                if ind < 20 and ind > 9:
-                    snippet = snippet[ind + len("... ") :]
-                ind = snippet.find("...")
-                if ind > len(snippet) - 5:
-                    snippet = snippet[:ind]
-
-                # update cosine similarity between snippet and given text
-                url = link["link"]
-                if url not in urlList:
-                    urlList.append(url)
-                    scoreArray.append([0] * len(sentences))
-                urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
-                if plag_option == 'Standard':
-                    scoreArray[urlList.index(url)][i] = cosineSim(
-                        sentence, snippet)
-                else :     
-                    scoreArray[urlList.index(url)][i] = sentence_similarity(
-                        sentence, snippet
-                )
-        else:
-            print("Google Search failed")
-    return urlCount, scoreArray
-
-
-def getQueries(text, n):
-    # return n-grams of size n
-    words = text.split()
-    return [words[i : i + n] for i in range(len(words) - n + 1)]
-
-
-def print2D(array):
-    print(np.array(array))
-
-
-def removePunc(text):
-    res = re.sub(r"[^\w\s]", "", text)
-    return res
-
-
-async def get_url_data(url, client):
-    try:
-        r = await client.get(url)
-        # print(r.status_code)
-        if r.status_code == 200:
-            # print("in")
-            soup = BeautifulSoup(r.content, "html.parser")
-            return soup
-    except Exception:
-        print("HTTPx parallel soup failed")
-        return None
-
-
-async def parallel_scrap(urls):
-    async with httpx.AsyncClient(timeout=30) as client:
-        tasks = []
-        for url in urls:
-            tasks.append(get_url_data(url=url, client=client))
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-    return results
-
-
-class TimeoutError(Exception):
-    pass
-
-    
-
-def matchingScore(sentence, content):
-    if sentence in content:
-        return 1
-    sentence = removePunc(sentence)
-    content = removePunc(content)
-    if sentence in content:
-        return 1
-    else:
-        n = 5
-        ngrams = getQueries(sentence, n)
-        if len(ngrams) == 0:
-            return 0
-        matched = [x for x in ngrams if " ".join(x) in content]
-    return len(matched) / len(ngrams)
 
+# # returns cosine similarity of two words
+# # uses: text_to_vector(text) and get_cosine(v1,v2)
+# def cosineSim(text1, text2):
+#     vector1 = text_to_vector(text1)
+#     vector2 = text_to_vector(text2)
+#     # print vector1,vector2
+#     cosine = get_cosine(vector1, vector2)
+#     return cosine
 
-# def matchingScoreWithTimeout(sentence, content):
-#     def timeout_handler():
-#         raise TimeoutError("Function timed out")
 
-#     timer = threading.Timer(10, timeout_handler)  # Set a timer for 2 seconds
-#     timer.start()
+# def cos_sim_torch(embedding_1, embedding_2):
+#     return util.pytorch_cos_sim(embedding_1, embedding_2).item()
+
+
+# def embed_text(text):
+#     return model.encode(text, convert_to_tensor=True)
+
+
+# def sentence_similarity(text1, text2):
+#     embedding_1 = model.encode(text1, convert_to_tensor=True)
+#     embedding_2 = model.encode(text2, convert_to_tensor=True)
+
+#     o = util.pytorch_cos_sim(embedding_1, embedding_2)
+#     return o.item()
+
+
+# def get_soup_requests(url):
+#     page = requests.get(url)
+#     if page.status_code == 200:
+#         soup = BeautifulSoup(page.content, "html.parser")
+#         return soup
+#     print("HTML soup failed")
+#     return None
+
+
+# def get_soup_httpx(url):
+#     client = httpx.Client(timeout=30)
 #     try:
-#         score = sentence_similarity(sentence, content)
-#         # score = matchingScore(sentence, content)
-#         timer.cancel()  # Cancel the timer if calculation completes before timeout
-#         return score
-#     except TimeoutError:
-#         return 0    
+#         page = client.get(url)
+#         if page.status_code == httpx.codes.OK:
+#             soup = BeautifulSoup(page.content, "html.parser")
+#             return soup
+#     except:
+#         print("HTTPx soup failed")
+#         return None
+
+
+# def getSentences(text):
+#     from nltk.tokenize import sent_tokenize
+
+#     sents = sent_tokenize(text)
+#     two_sents = []
+#     for i in range(len(sents)):
+#         if (i % 2) == 0:
+#             two_sents.append(sents[i])
+#         else:
+#             two_sents[len(two_sents) - 1] += " " + sents[i]
+#     return two_sents
+
+
+# def googleSearch(
+#     plag_option,
+#     sentences,
+#     urlCount,
+#     scoreArray,
+#     urlList,
+#     sorted_date,
+#     domains_to_skip,
+#     api_key,
+#     cse_id,
+#     **kwargs,
+# ):
+#     service = build("customsearch", "v1", developerKey=api_key)
+#     for i, sentence in enumerate(sentences):
+#         results = (
+#             service.cse()
+#             .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+#             .execute()
+#         )
+#         if "items" in results and len(results["items"]) > 0:
+#             for count, link in enumerate(results["items"]):
+#                 # stop after 3 pages
+#                 if count >= 3:
+#                     break
+#                 # skip user selected domains
+#                 if any(
+#                     ("." + domain) in link["link"] for domain in domains_to_skip
+#                 ):
+#                     continue
+#                 # clean up snippet of '...'
+#                 snippet = link["snippet"]
+#                 ind = snippet.find("...")
+#                 if ind < 20 and ind > 9:
+#                     snippet = snippet[ind + len("... ") :]
+#                 ind = snippet.find("...")
+#                 if ind > len(snippet) - 5:
+#                     snippet = snippet[:ind]
+
+#                 # update cosine similarity between snippet and given text
+#                 url = link["link"]
+#                 if url not in urlList:
+#                     urlList.append(url)
+#                     scoreArray.append([0] * len(sentences))
+#                 urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
+#                 if plag_option == "Standard":
+#                     scoreArray[urlList.index(url)][i] = cosineSim(
+#                         sentence, snippet
+#                     )
+#                 else:
+#                     scoreArray[urlList.index(url)][i] = sentence_similarity(
+#                         sentence, snippet
+#                     )
+#         else:
+#             print("Google Search failed")
+#     return urlCount, scoreArray
+
+
+# def getQueries(text, n):
+#     # return n-grams of size n
+#     words = text.split()
+#     return [words[i : i + n] for i in range(len(words) - n + 1)]
+
 
+# def print2D(array):
+#     print(np.array(array))
 
-# async def matchingScoreAsync(sentences, content, content_idx, ScoreArray):
+
+# def removePunc(text):
+#     res = re.sub(r"[^\w\s]", "", text)
+#     return res
+
+
+# async def get_url_data(url, client):
+#     try:
+#         r = await client.get(url)
+#         # print(r.status_code)
+#         if r.status_code == 200:
+#             # print("in")
+#             soup = BeautifulSoup(r.content, "html.parser")
+#             return soup
+#     except Exception:
+#         print("HTTPx parallel soup failed")
+#         return None
+
+
+# async def parallel_scrap(urls):
+#     async with httpx.AsyncClient(timeout=30) as client:
+#         tasks = []
+#         for url in urls:
+#             tasks.append(get_url_data(url=url, client=client))
+#         results = await asyncio.gather(*tasks, return_exceptions=True)
+#     return results
+
+
+# class TimeoutError(Exception):
+#     pass
+
+
+# def matchingScore(sentence, content):
+#     if sentence in content:
+#         return 1
+#     sentence = removePunc(sentence)
+#     content = removePunc(content)
+#     if sentence in content:
+#         return 1
+#     else:
+#         n = 5
+#         ngrams = getQueries(sentence, n)
+#         if len(ngrams) == 0:
+#             return 0
+#         matched = [x for x in ngrams if " ".join(x) in content]
+#     return len(matched) / len(ngrams)
+
+
+# # def matchingScoreWithTimeout(sentence, content):
+# #     def timeout_handler():
+# #         raise TimeoutError("Function timed out")
+
+# #     timer = threading.Timer(10, timeout_handler)  # Set a timer for 2 seconds
+# #     timer.start()
+# #     try:
+# #         score = sentence_similarity(sentence, content)
+# #         # score = matchingScore(sentence, content)
+# #         timer.cancel()  # Cancel the timer if calculation completes before timeout
+# #         return score
+# #     except TimeoutError:
+# #         return 0
+
+
+# # async def matchingScoreAsync(sentences, content, content_idx, ScoreArray):
+# #     content = removePunc(content)
+# #     for j, sentence in enumerate(sentences):
+# #         sentence = removePunc(sentence)
+# #         if sentence in content:
+# #             ScoreArray[content_idx][j] = 1
+# #         else:
+# #             n = 5
+# #             ngrams = getQueries(sentence, n)
+# #             if len(ngrams) == 0:
+# #                 return 0
+# #             matched = [x for x in ngrams if " ".join(x) in content]
+# #             ScoreArray[content_idx][j] = len(matched) / len(ngrams)
+# #     print(
+# #         f"Analyzed {content_idx+1} of soups (SOUP SUCCEEDED)........................"
+# #     )
+# #     return ScoreArray
+
+
+# async def matchingScoreAsync(
+#     sentences, content, content_idx, ScoreArray, model, util
+# ):
 #     content = removePunc(content)
 #     for j, sentence in enumerate(sentences):
 #         sentence = removePunc(sentence)
-#         if sentence in content:
-#             ScoreArray[content_idx][j] = 1
-#         else:
-#             n = 5
-#             ngrams = getQueries(sentence, n)
-#             if len(ngrams) == 0:
-#                 return 0
-#             matched = [x for x in ngrams if " ".join(x) in content]
-#             ScoreArray[content_idx][j] = len(matched) / len(ngrams)
+#         similarity_score = sentence_similarity(sentence, content, model, util)
+#         ScoreArray[content_idx][j] = similarity_score
 #     print(
-#         f"Analyzed {content_idx+1} of soups (SOUP SUCCEEDED)........................"
+#         f"Analyzed {content_idx+1} of contents (CONTENT ANALYZED)........................"
 #     )
 #     return ScoreArray
 
-async def matchingScoreAsync(sentences, content, content_idx, ScoreArray, model, util):
-    content = removePunc(content)
-    for j, sentence in enumerate(sentences):
-        sentence = removePunc(sentence) 
-        similarity_score = sentence_similarity(sentence, content, model, util)
-        ScoreArray[content_idx][j] = similarity_score
-    print(f"Analyzed {content_idx+1} of contents (CONTENT ANALYZED)........................")
-    return ScoreArray
-
-
-async def parallel_analyze(soups, sentences, ScoreArray):
-    tasks = []
-    for i, soup in enumerate(soups):
-        if soup:
-            page_content = soup.text
-            tasks.append(
-                matchingScoreAsync(sentences, page_content, i, ScoreArray)
-            )
-        else:
-            print(
-                f"Analyzed {i+1} of soups (SOUP FAILED)........................"
-            )
-    ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
-    return ScoreArray
-
-
-async def parallel_analyze_2(soups, sentences, ScoreArray):
-    tasks = [[0] * len(ScoreArray[0]) for i in range(len(ScoreArray))]
-    for i, soup in enumerate(soups):
-        if soup:
-            page_content = soup.text
-            for j, sent in enumerate(sentences):
-                print(
-                    f"Analyzing {i+1} of {len(soups)} soups with {j+1} of {len(sentences)} sentences........................"
-                )
-                tasks[i][j] = sentence_similarity(sent, page_content)
-        else:
-            print(
-                f"Analyzed {i+1} of soups (SOUP FAILED)........................"
-            )
-    ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
-    return ScoreArray
+
+# async def parallel_analyze(soups, sentences, ScoreArray):
+#     tasks = []
+#     for i, soup in enumerate(soups):
+#         if soup:
+#             page_content = soup.text
+#             tasks.append(
+#                 matchingScoreAsync(sentences, page_content, i, ScoreArray)
+#             )
+#         else:
+#             print(
+#                 f"Analyzed {i+1} of soups (SOUP FAILED)........................"
+#             )
+#     ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
+#     return ScoreArray
+
+
+# async def parallel_analyze_2(soups, sentences, ScoreArray):
+#     tasks = [[0] * len(ScoreArray[0]) for i in range(len(ScoreArray))]
+#     for i, soup in enumerate(soups):
+#         if soup:
+#             page_content = soup.text
+#             for j, sent in enumerate(sentences):
+#                 print(
+#                     f"Analyzing {i+1} of {len(soups)} soups with {j+1} of {len(sentences)} sentences........................"
+#                 )
+#                 tasks[i][j] = sentence_similarity(sent, page_content)
+#         else:
+#             print(
+#                 f"Analyzed {i+1} of soups (SOUP FAILED)........................"
+#             )
+#     ScoreArray = await asyncio.gather(*tasks, return_exceptions=True)
+#     return ScoreArray