Update predictors.py

predictors.py

@@ -1,321 +1,565 @@
-import torch
-import numpy as np
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import nltk
-import torch.nn.functional as F
-import nltk
-from scipy.special import softmax
-import yaml
-from utils import *
-import joblib
-from optimum.bettertransformer import BetterTransformer
-import gc
-from cleantext import clean
-import gradio as gr
-from tqdm.auto import tqdm
-from transformers import pipeline
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-import nltk
+import time
 from nltk.tokenize import sent_tokenize
-from optimum.pipelines import pipeline
-
-with open("config.yaml", "r") as file:
-    params = yaml.safe_load(file)
-    
-nltk.download("punkt")
-nltk.download("stopwords")
-device_needed = "cuda" if torch.cuda.is_available() else "cpu"
-device = "cuda" if torch.cuda.is_available() else "cpu"
-print('DEVICE IS :' , device)
-
-text_bc_model_path = params["TEXT_BC_MODEL_PATH"]
-text_mc_model_path = params["TEXT_MC_MODEL_PATH"]
-text_quillbot_model_path = params["TEXT_QUILLBOT_MODEL_PATH"]
-quillbot_labels = params["QUILLBOT_LABELS"]
-mc_label_map = params["MC_OUTPUT_LABELS"]
-mc_token_size = int(params["MC_TOKEN_SIZE"])
-bc_token_size = int(params["BC_TOKEN_SIZE"])
-bias_checker_model_name = params['BIAS_CHECKER_MODEL_PATH']
-bias_corrector_model_name = params['BIAS_CORRECTOR_MODEL_PATH']
-# access_token = params['HF_TOKEN']
-
-text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
-text_bc_model = AutoModelForSequenceClassification.from_pretrained(text_bc_model_path).to(device)
-text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
-text_mc_model = AutoModelForSequenceClassification.from_pretrained(text_mc_model_path).to(device)
-quillbot_tokenizer = AutoTokenizer.from_pretrained(text_quillbot_model_path)
-quillbot_model = AutoModelForSequenceClassification.from_pretrained(text_quillbot_model_path).to(device)
-
-# proxy models for explainability
-mini_bc_model_name = "polygraf-ai/bc-model"
-bc_tokenizer_mini = AutoTokenizer.from_pretrained(mini_bc_model_name)
-bc_model_mini = AutoModelForSequenceClassification.from_pretrained(mini_bc_model_name).to(device_needed)
-mini_humanizer_model_name =  "polygraf-ai/humanizer-model"
-humanizer_tokenizer_mini = AutoTokenizer.from_pretrained(mini_humanizer_model_name)
-humanizer_model_mini = AutoModelForSequenceClassification.from_pretrained(mini_humanizer_model_name).to(device_needed)
-
-bc_model_mini = BetterTransformer.transform(bc_model_mini)
-humanizer_model_mini = BetterTransformer.transform(humanizer_model_mini)
-text_bc_model = BetterTransformer.transform(text_bc_model)
-text_mc_model = BetterTransformer.transform(text_mc_model)
-quillbot_model = BetterTransformer.transform(quillbot_model)
-
-bias_model_checker = AutoModelForSequenceClassification.from_pretrained(bias_checker_model_name)
-tokenizer = AutoTokenizer.from_pretrained(bias_checker_model_name)
-bias_model_checker = BetterTransformer.transform(bias_model_checker, keep_original_model=False)
-bias_checker = pipeline(
-    "text-classification",
-    model=bias_checker_model_name,
-    tokenizer=bias_checker_model_name,
-)
-gc.collect()
-bias_corrector = pipeline( "text2text-generation", model=bias_corrector_model_name, accelerator="ort")
-
-# model score calibration
-iso_reg = joblib.load("isotonic_regression_model.joblib")
-
-
-def split_text(text: str) -> list:
-    sentences = sent_tokenize(text)
-    return [[sentence] for sentence in sentences]
-
-def correct_text(text: str, bias_checker, bias_corrector, separator: str = " ") -> tuple:
-    sentence_batches = split_text(text)
-    corrected_text = []
-    corrections = []
-    for batch in tqdm(sentence_batches, total=len(sentence_batches), desc="correcting text.."):
-        raw_text = " ".join(batch)
-        results = bias_checker(raw_text)
-        if results[0]["label"] != "LABEL_1" or (results[0]["label"] == "LABEL_1" and results[0]["score"] < 0.9):
-            corrected_batch = bias_corrector(raw_text)
-            corrected_version = corrected_batch[0]["generated_text"]
-            corrected_text.append(corrected_version)
-            corrections.append((raw_text, corrected_version)) 
-        else:
-            corrected_text.append(raw_text)
-    corrected_text = separator.join(corrected_text)
-    return corrected_text, corrections
-
-def update(text: str):
-    text = clean(text, lower=False)
-    corrected_text, corrections = correct_text(text, bias_checker, bias_corrector)
-    corrections_display = "".join([f"{corr}" for orig, corr in corrections])
-    if corrections_display == "":
-        corrections_display = text
-    return corrections_display
-
-def update_main(text: str):
-    text = clean(text, lower=False)
-    corrected_text, corrections = correct_text(text, bias_checker, bias_corrector)
-    corrections_display = "\n\n".join([f"Original: {orig}\nCorrected: {corr}" for orig, corr in corrections])
-    return corrected_text, corrections_display
-
-def split_text(text: str) -> list:
-    sentences = sent_tokenize(text)
-    return [[sentence] for sentence in sentences]
-
-def get_token_length(tokenizer, sentence):
-    return len(tokenizer.tokenize(sentence))
-
-def split_text_allow_complete_sentences_nltk(text, type_det="bc"):
-    sentences = sent_tokenize(text)
-    chunks = []
-    current_chunk = []
-    current_length = 0
-    if type_det == "bc":
-        tokenizer = text_bc_tokenizer
-        max_tokens = bc_token_size
-    elif type_det == "mc":
-        tokenizer = text_mc_tokenizer
-        max_tokens = mc_token_size
-
-    elif type_det == "quillbot":
-        tokenizer = quillbot_tokenizer
-        max_tokens = 256
-
-    def add_sentence_to_chunk(sentence):
-        nonlocal current_chunk, current_length
-        sentence_length = get_token_length(tokenizer, sentence)
-        if current_length + sentence_length > max_tokens:
-            chunks.append((current_chunk, current_length))
-            current_chunk = []
-            current_length = 0
-        current_chunk.append(sentence)
-        current_length += sentence_length
-
-    for sentence in sentences:
-        add_sentence_to_chunk(sentence)
-    if current_chunk:
-        chunks.append((current_chunk, current_length))
-    adjusted_chunks = []
-    while chunks:
-        chunk = chunks.pop(0)
-        if len(chunks) > 0 and chunk[1] < max_tokens / 2:
-            next_chunk = chunks.pop(0)
-            combined_length = chunk[1] + next_chunk[1]
-            if combined_length <= max_tokens:
-                adjusted_chunks.append((chunk[0] + next_chunk[0], combined_length))
-            else:
-                adjusted_chunks.append(chunk)
-                chunks.insert(0, next_chunk)
-        else:
-            adjusted_chunks.append(chunk)
-    result_chunks = [" ".join(chunk[0]) for chunk in adjusted_chunks]
-    return result_chunks
-
-
-def predict_quillbot(text, bias_buster_selected):
-    if bias_buster_selected:
-        text = update(text)
-    with torch.no_grad():
-        quillbot_model.eval()
-        tokenized_text = quillbot_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            max_length=256,
-            return_tensors="pt",
-        ).to(device)
-        output = quillbot_model(**tokenized_text)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        q_score = {
-            "Humanized": output_norm[1].item(),
-            "Original": output_norm[0].item(),
-        }
-        return q_score
-
-
-def predict_for_explainanility(text, model_type=None):
-    if model_type == "quillbot":
-        cleaning = False
-        max_length = 256
-        model = humanizer_model_mini
-        tokenizer = humanizer_tokenizer_mini
-    elif model_type == "bc":
-        cleaning = True
-        max_length = bc_token_size
-        model = bc_model_mini
-        tokenizer = bc_tokenizer_mini
+from googleapiclient.discovery import build
+from collections import Counter
+import re, math
+from sentence_transformers import SentenceTransformer, util
+import asyncio
+import httpx
+from bs4 import BeautifulSoup
+import numpy as np
+import concurrent
+from multiprocessing import Pool
+from const import url_types
+from collections import defaultdict
+
+WORD = re.compile(r"\w+")
+model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+
+months = {
+    "January": "01",
+    "February": "02",
+    "March": "03",
+    "April": "04",
+    "May": "05",
+    "June": "06",
+    "July": "07",
+    "August": "08",
+    "September": "09",
+    "October": "10",
+    "November": "11",
+    "December": "12",
+}
+
+color_map = [
+    "#cf2323",
+    "#d65129",
+    "#d66329",
+    "#d67129",
+    "#eb9d59",
+    "#c2ad36",
+    "#d6ae29",
+    "#d6b929",
+    "#e1ed72",
+    "#c2db76",
+    "#a2db76",
+]
+
+
+def text_to_vector(text):
+    words = WORD.findall(text)
+    return Counter(words)
+
+
+def cosineSim(text1, text2):
+    vector1 = text_to_vector(text1)
+    vector2 = text_to_vector(text2)
+    # print vector1,vector2
+    cosine = get_cosine(vector1, vector2)
+    return cosine
+
+
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+    sum1 = sum([vec1[x] ** 2 for x in vec1.keys()])
+    sum2 = sum([vec2[x] ** 2 for x in vec2.keys()])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+    if denominator == 0:
+        return 0.0
     else:
-        raise ValueError("Invalid model type")
-    with torch.no_grad():
-        if cleaning:
-            text = [remove_special_characters(t) for t in text]
-        tokenized_text = tokenizer(
-            text,
-            return_tensors="pt",
-            padding="max_length",
-            truncation=True,
-            max_length=max_length,
-        ).to(device_needed)
-        outputs = model(**tokenized_text)
-        tensor_logits = outputs[0]
-        probas = F.softmax(tensor_logits).detach().cpu().numpy()
-    return probas
-
-
-def predict_bc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_bc_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            max_length=bc_token_size,
-            return_tensors="pt",
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        return output_norm
-
-
-def predict_mc(model, tokenizer, text):
-    with torch.no_grad():
-        model.eval()
-        tokens = text_mc_tokenizer(
-            text,
-            padding="max_length",
-            truncation=True,
-            return_tensors="pt",
-            max_length=mc_token_size,
-        ).to(device)
-        output = model(**tokens)
-        output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
-        return output_norm
-
-
-def predict_bc_scores(input):
-    bc_scores = []
-    samples_len_bc = len(
-        split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    )
-    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    for i in range(samples_len_bc):
-        cleaned_text_bc = remove_special_characters(segments_bc[i])
-        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
-        bc_scores.append(bc_score)
-    bc_scores_array = np.array(bc_scores)
-    average_bc_scores = np.mean(bc_scores_array, axis=0)
-    bc_score_list = average_bc_scores.tolist()
-    print(
-        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
-    )
-    # isotonic regression calibration
-    ai_score = iso_reg.predict([bc_score_list[1]])[0]
-    human_score = 1 - ai_score
-    bc_score = {"AI": ai_score, "HUMAN": human_score}
-    print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
-    print(f"Input Text: {cleaned_text_bc}")
-    return bc_score
-
-
-def predict_mc_scores(input):
-    # BC SCORE
-    bc_scores = []
-    samples_len_bc = len(
-        split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    )
-    segments_bc = split_text_allow_complete_sentences_nltk(input, type_det="bc")
-    for i in range(samples_len_bc):
-        cleaned_text_bc = remove_special_characters(segments_bc[i])
-        bc_score = predict_bc(text_bc_model, text_bc_tokenizer, cleaned_text_bc)
-        bc_scores.append(bc_score)
-    bc_scores_array = np.array(bc_scores)
-    average_bc_scores = np.mean(bc_scores_array, axis=0)
-    bc_score_list = average_bc_scores.tolist()
-    print(
-        f"Original BC scores: AI: {bc_score_list[1]}, HUMAN: {bc_score_list[0]}"
-    )
-    # isotonic regression calibration
-    ai_score = iso_reg.predict([bc_score_list[1]])[0]
-    human_score = 1 - ai_score
-    bc_score = {"AI": ai_score, "HUMAN": human_score}
-    print(f"Calibration BC scores: AI: {ai_score}, HUMAN: {human_score}")
-    mc_scores = []
-    segments_mc = split_text_allow_complete_sentences_nltk(
-        input, type_det="mc"
+        return float(numerator) / denominator
+
+
+def split_sentence_blocks(text, size):
+    if size == "Paragraph":
+        blocks = text.strip().split("\n")
+        return blocks
+    else:
+        sents = sent_tokenize(text.strip())
+        return sents
+
+
+def build_date(year=2024, month="March", day=1):
+    return f"{year}{months[month]}{day}"
+
+
+def split_ngrams(text, n):
+    words = text.split()
+    return [tuple(words[i : i + n]) for i in range(len(words) - n + 1)]
+
+
+def sentence_similarity(text1, text2):
+    embedding_1 = model.encode(text1, convert_to_tensor=True)
+    embedding_2 = model.encode(text2, convert_to_tensor=True)
+    o = util.pytorch_cos_sim(embedding_1, embedding_2)
+    return o.item()
+
+
+async def get_url_data(url, client):
+    try:
+        r = await client.get(url)
+        if r.status_code == 200:
+            soup = BeautifulSoup(r.content, "html.parser")
+            return soup
+    except Exception:
+        return None
+
+
+async def parallel_scrap(urls):
+    async with httpx.AsyncClient(timeout=30) as client:
+        tasks = []
+        for url in urls:
+            tasks.append(get_url_data(url=url, client=client))
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+    return results
+
+
+def merge_ngrams_into_sentence(ngrams):
+    if ngrams == None:
+        return ""
+    if len(ngrams) > 20:
+        ngrams = ngrams[:20]
+    merged_sentence = []
+    i = 0
+    for ngram in ngrams:
+        overlap = len(set(ngram) & set(merged_sentence[-len(ngram) :]))
+        if overlap == 0:
+            merged_sentence.extend(ngram)
+        elif overlap < len(ngram):
+            merged_sentence.extend(ngram[overlap:])
+    return " ".join(merged_sentence)
+
+
+def remove_ngrams_after(ngrams, target_ngram):
+    try:
+        index = ngrams.index(target_ngram)
+        return ngrams[: index + 1]
+    except ValueError:
+        return None
+
+
+def matching_score(sentence_content_tuple):
+    sentence, content, score = sentence_content_tuple
+    if sentence in content:
+        return 1, sentence
+    # if score > 0.9:
+    #     return score
+    else:
+        n = 5
+
+        # ngrams = split_ngrams(sentence, n)
+        # if len(ngrams) == 0:
+        #     return 0
+        # matched = [x for x in ngrams if " ".join(x) in content]
+        # return len(matched) / len(ngrams)
+
+        # list comprehension matching
+        # ngrams_sentence = split_ngrams(sentence, n)
+        # ngrams_content = [tuple(ngram) for ngram in split_ngrams(content, n)]
+        # if len(ngrams_sentence) == 0:
+        #     return 0, ""
+        # matched_ngrams = [
+        #     1 for ngram in ngrams_sentence if tuple(ngram) in ngrams_content
+        # ]
+        # matched_count = sum(matched_ngrams)
+
+        # set intersection matching
+        ngrams_sentence = set(split_ngrams(sentence, n))
+        ngrams_content = set(split_ngrams(content, n))
+        if len(ngrams_sentence) == 0:
+            return 0, ""
+        matched_ngrams = ngrams_sentence.intersection(ngrams_content)
+        matched_count = len(matched_ngrams)
+
+        # matched content
+        matched_content_ngrams = []
+        found = False
+        last_found = None
+        for ngram in ngrams_sentence:
+            for ngram_content in ngrams_content:
+                if tuple(ngram) == ngram_content:
+                    found = True
+                    last_found = ngram_content
+                if found:
+                    matched_content_ngrams.append(ngram_content)
+        matched_content_ngrams = remove_ngrams_after(
+            matched_content_ngrams, last_found
+        )
+        matched_content = merge_ngrams_into_sentence(matched_content_ngrams)
+
+        return matched_count / len(ngrams_sentence), matched_content
+
+
+def process_with_multiprocessing(input_data):
+    with Pool(processes=1) as pool:
+        scores = pool.map(matching_score, input_data)
+    return scores
+
+
+def map_sentence_url(sentences, score_array):
+    sentenceToMaxURL = [-1] * len(sentences)
+    for j in range(len(sentences)):
+        if j > 0:
+            maxScore = score_array[sentenceToMaxURL[j - 1]][j]
+            sentenceToMaxURL[j] = sentenceToMaxURL[j - 1]
+        else:
+            maxScore = -1
+        for i in range(len(score_array)):
+            margin = (
+                0.05
+                if (j > 0 and sentenceToMaxURL[j] == sentenceToMaxURL[j - 1])
+                else 0
+            )
+            if score_array[i][j] - maxScore > margin:
+                maxScore = score_array[i][j]
+                sentenceToMaxURL[j] = i
+    return sentenceToMaxURL
+
+
+def check_url_category(url):
+    for category, urls in url_types.items():
+        for u in urls:
+            if u in url:
+                return category
+    return "Internet Source"
+
+
+def google_search(
+    plag_option,
+    sentences,
+    url_count,
+    score_array,
+    url_list,
+    snippets,
+    sorted_date,
+    domains_to_skip,
+    api_key,
+    cse_id,
+    **kwargs,
+):
+    service = build("customsearch", "v1", developerKey=api_key)
+    num_pages = 1
+    for i, sentence in enumerate(sentences):
+        results = (
+            service.cse()
+            .list(q=sentence, cx=cse_id, sort=sorted_date, **kwargs)
+            .execute()
+        )
+        if "items" in results and len(results["items"]) > 0:
+            for count, link in enumerate(results["items"]):
+                if count >= num_pages:
+                    break
+                # skip user selected domains
+                if (domains_to_skip is not None) and any(
+                    ("." + domain) in link["link"] for domain in domains_to_skip
+                ):
+                    continue
+                # clean up snippet of '...'
+                snippet = link["snippet"]
+                ind = snippet.find("...")
+                if ind < 20 and ind > 9:
+                    snippet = snippet[ind + len("... ") :]
+                ind = snippet.find("...")
+                if ind > len(snippet) - 5:
+                    snippet = snippet[:ind]
+
+                # update cosine similarity between snippet and given text
+                url = link["link"]
+                if url not in url_list:
+                    url_list.append(url)
+                    score_array.append([0] * len(sentences))
+                    snippets.append([""] * len(sentences))
+                url_count[url] = url_count[url] + 1 if url in url_count else 1
+                snippets[url_list.index(url)][i] = snippet
+                if plag_option == "Standard":
+                    score_array[url_list.index(url)][i] = cosineSim(
+                        sentence, snippet
+                    )
+                else:
+                    score_array[url_list.index(url)][i] = sentence_similarity(
+                        sentence, snippet
+                    )
+    return url_count, score_array
+
+
+def plagiarism_check(
+    plag_option,
+    input,
+    year_from,
+    month_from,
+    day_from,
+    year_to,
+    month_to,
+    day_to,
+    domains_to_skip,
+    source_block_size,
+):
+    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    # api_key = "AIzaSyCLyCCpOPLZWuptuPAPSg8cUIZhdEMVf6g"
+    api_key = "AIzaSyCS1WQDMl1IMjaXtwSd_2rA195-Yc4psQE"
+    # api_key = "AIzaSyCB61O70B8AC3l5Kk3KMoLb6DN37B7nqIk"
+    # api_key = "AIzaSyCg1IbevcTAXAPYeYreps6wYWDbU0Kz8tg"
+    # api_key = "AIzaSyA5VVwY1eEoIoflejObrxFDI0DJvtbmgW8"
+    cse_id = "851813e81162b4ed4"
+
+    url_scores = []
+    sentence_scores = []
+    sentences = split_sentence_blocks(input, source_block_size)
+    url_count = {}
+    score_array = []
+    url_list = []
+    snippets = []
+    date_from = build_date(year_from, month_from, day_from)
+    date_to = build_date(year_to, month_to, day_to)
+    sort_date = f"date:r:{date_from}:{date_to}"
+    # get list of URLS to check
+    start_time = time.perf_counter()
+    url_count, score_array = google_search(
+        plag_option,
+        sentences,
+        url_count,
+        score_array,
+        url_list,
+        snippets,
+        sort_date,
+        domains_to_skip,
+        api_key,
+        cse_id,
     )
-    samples_len_mc = len(
-        split_text_allow_complete_sentences_nltk(input, type_det="mc")
+    print("GOOGLE SEARCH PROCESSING TIME: ", time.perf_counter() - start_time)
+    # Scrape URLs in list
+    start_time = time.perf_counter()
+    soups = asyncio.run(parallel_scrap(url_list))
+    print("SCRAPING PROCESSING TIME: ", time.perf_counter() - start_time)
+    input_data = []
+    for i, soup in enumerate(soups):
+        if soup:
+            page_content = soup.text
+            for j, sent in enumerate(sentences):
+                input_data.append((sent, page_content, score_array[i][j]))
+    start_time = time.perf_counter()
+    # scores = process_with_multiprocessing(input_data)
+    scores = []
+    for i in input_data:
+        scores.append(matching_score(i))
+    print("MATCHING SCORE PROCESSING TIME: ", time.perf_counter() - start_time)
+    matched_sentence_array = [
+        ["" for _ in range(len(score_array[0]))]
+        for _ in range(len(score_array))
+    ]
+
+    k = 0
+    # Update score array for each (soup, sentence)
+    for i, soup in enumerate(soups):
+        if soup:
+            for j, _ in enumerate(sentences):
+                score_array[i][j] = scores[k][0]
+                matched_sentence_array[i][j] = scores[k][1]
+                k += 1
+
+    sentenceToMaxURL = map_sentence_url(sentences, score_array)
+    index = np.unique(sentenceToMaxURL)
+
+    url_source = {}
+    for url in index:
+        s = [
+            score_array[url][sen]
+            for sen in range(len(sentences))
+            if sentenceToMaxURL[sen] == url
+        ]
+        url_source[url] = sum(s) / len(s)
+    index_descending = sorted(url_source, key=url_source.get, reverse=True)
+    urlMap = {}
+    for count, i in enumerate(index_descending):
+        urlMap[i] = count + 1
+
+    # build results
+    for i, sent in enumerate(sentences):
+        ind = sentenceToMaxURL[i]
+        if url_source[ind] > 0.1:
+            sentence_scores.append(
+                [
+                    sent,
+                    round(url_source[ind] * 100, 2),
+                    url_list[ind],
+                    urlMap[ind],
+                ]
+            )
+        else:
+            sentence_scores.append([sent, None, url_list[ind], -1])
+    print("SNIPPETS: ", snippets)
+    snippets = [[item for item in sublist if item] for sublist in snippets]
+    for ind in index_descending:
+        if url_source[ind] > 0.1:
+            matched_sentence_array = [
+                [item for item in sublist if item]
+                for sublist in matched_sentence_array
+            ]
+            matched_sentence = "...".join(
+                [sent for sent in matched_sentence_array[ind]]
+            )
+            if matched_sentence == "":
+                matched_sentence = "...".join([sent for sent in snippets[ind]])
+            url_scores.append(
+                [
+                    url_list[ind],
+                    round(url_source[ind] * 100, 2),
+                    urlMap[ind],
+                    matched_sentence,
+                ]
+            )
+
+    return sentence_scores, url_scores
+
+
+def html_highlight(
+    plag_option,
+    input,
+    year_from,
+    month_from,
+    day_from,
+    year_to,
+    month_to,
+    day_to,
+    domains_to_skip,
+    source_block_size,
+):
+    start_time = time.perf_counter()
+    sentence_scores, url_scores = plagiarism_check(
+        plag_option,
+        input,
+        year_from,
+        month_from,
+        day_from,
+        year_to,
+        month_to,
+        day_to,
+        domains_to_skip,
+        source_block_size,
     )
-    for i in range(samples_len_mc):
-        cleaned_text_mc = remove_special_characters(segments_mc[i])
-        mc_score = predict_mc(
-            text_mc_model, text_mc_tokenizer, cleaned_text_mc
-        )
-        mc_scores.append(mc_score)
-    mc_scores_array = np.array(mc_scores)
-    average_mc_scores = np.mean(mc_scores_array, axis=0)
-    mc_score_list = average_mc_scores.tolist()
-    mc_score = {}
-    for score, label in zip(mc_score_list, mc_label_map):
-        mc_score[label.upper()] = score
-
-    sum_prob = 1 - bc_score["HUMAN"]
-    for key, value in mc_score.items():
-        mc_score[key] = value * sum_prob
-    print("MC Score:", mc_score)
-    if sum_prob < 0.01:
-        mc_score = {}
-
-    return mc_score
+
+    html_content = """
+        <link href='https://fonts.googleapis.com/css?family=Roboto' rel='stylesheet'>
+        <div style='font-family: {font}; border: 2px solid black; padding: 10px; color: #FFFFFF;'>
+        <html>
+        <head>
+            <title>Toggle Details</title>
+            <style>
+                .score-container {
+                    display: flex;
+                    justify-content: space-around;
+                    align-items: left;
+                    padding: 20px;
+                }
+                .score-item {
+                    text-align: center;
+                    padding: 10px;
+                    background-color: #636362;
+                    border-radius: 5px;
+                    flex-grow: 1;
+                    margin: 0 5px;
+                }
+                .details {
+                    display: none;
+                    padding: 10px;
+                }
+                .url-link {
+                    font-size: 1.2em;
+                }
+                .url-link span {
+                    margin-right: 10px;
+                }
+                .toggle-button {
+                    color: #333;
+                    border: none;
+                    padding: 5px 10px;
+                    text-align: center;
+                    text-decoration: none;
+                    display: inline-block;
+                    cursor: pointer;
+                }
+            </style>
+        </head>
+    """
+
+    prev_idx = None
+    combined_sentence = ""
+    total_score = 0
+    total_count = 0
+    category_scores = defaultdict(set)
+    for sentence, score, url, idx in sentence_scores:
+        category = check_url_category(url)
+        if score is None:
+            total_score += 0
+        else:
+            total_score += score
+            category_scores[category].add(score)
+        total_count += 1
+
+        if idx != prev_idx and prev_idx is not None:
+            color = color_map[prev_idx - 1]
+            index_part = f"<span>[{prev_idx}]</span>"
+            formatted_sentence = f'<p style="background-color: {color}; padding: 2px;">{combined_sentence} {index_part}</p>'
+            html_content += formatted_sentence
+            combined_sentence = ""
+        combined_sentence += " " + sentence
+        prev_idx = idx
+
+    print(category_scores)
+    total_average_score = round(total_score / total_count, 2)
+    category_averages = {
+        category: round((sum(scores) / len(scores)), 2)
+        for category, scores in category_scores.items()
+    }
+
+    if combined_sentence:
+        color = color_map[prev_idx - 1]
+        index_part = ""
+        if prev_idx != -1:
+            index_part = f"<span>[{prev_idx}]</span>"
+        formatted_sentence = f'<p style="background-color: {color}; padding: 2px;">{combined_sentence} {index_part}</p>'
+        html_content += formatted_sentence
+
+    html_content += "<hr>"
+
+    html_content += f"""
+        <div class="score-container">
+        <div class="score-item">
+            <h3>Overall Similarity</h3>
+            <p>{total_average_score}%</p>
+        </div>
+    """
+    for category, score in category_averages.items():
+        html_content += f"""
+            <div class="score-item"><h3>{category}</h3><p>{score}%</p></div>
+        """
+    html_content += "</div>"
+
+    for url, score, idx, sentence in url_scores:
+        url_category = check_url_category(url)
+        color = color_map[idx - 1]
+        formatted_url = f"""
+            <p style="background-color: {color}; padding: 5px; font-size: 1.2em">[{idx}] <b>{url}</b></p><p><i>{url_category}</i></p>
+            <p> --- <b>Matching Score: </b>{score}%</p>
+            <p> --- <b>Original Source Content: </b>{sentence}</p>
+        """
+        # formatted_url = f"""
+        #     <div class="url-link">
+        #         <p style="background-color: {color}; padding: 5px; font-size: 1.2em">[{idx}] <b>{url}</b></p><p>{url_category}</p>
+        #         <a href="#" onclick="toggleDetails(event)" class="toggle-button">&gt;</a>
+        #     </div>
+        #     <div id="detailsContainer" class="details">
+        #         <p> --- <b>Matching Score: </b>{score}%</p>
+        #         <p> --- <b>Original Source Content: </b>{sentence}</p>
+        #     </div>
+        # """
+        html_content += formatted_url
+
+    html_content += "</html>"
+
+    print("PLAGIARISM PROCESSING TIME: ", time.perf_counter() - start_time)
+
+    return html_content