Merge branch 'main' into demo

app.py

@@ -18,6 +18,8 @@ from transformers import GPT2LMHeadModel, GPT2TokenizerFast
 import nltk, spacy, subprocess, torch
 import plotly.graph_objects as go
 import nltk
+from unidecode import unidecode
+
 
 nltk.download('punkt')
 tokenizer = AutoTokenizer.from_pretrained('google-bert/bert-base-uncased')
@@ -180,17 +182,30 @@ AI DETECTION SECTION
 """
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
-text_bc_model_path = "polygraf-ai/v3-bert-3-2m-trun-bc"
+text_bc_model_path = "polygraf-ai/v3-bert-3-2m-trun-bc-lighter-spec"
 text_bc_tokenizer = AutoTokenizer.from_pretrained(text_bc_model_path)
 text_bc_model = AutoModelForSequenceClassification.from_pretrained(text_bc_model_path).to(device)
 
-text_mc_model_path = "polygraf-ai/text-detect-mc-bert-base-uncased-v1-bert-429k"
+text_mc_model_path = "polygraf-ai/text-detect-mc-bert-base-uncased-v1-bert-429k-256"
 text_mc_tokenizer = AutoTokenizer.from_pretrained(text_mc_model_path)
 text_mc_model = AutoModelForSequenceClassification.from_pretrained(text_mc_model_path).to(device)
 
+def remove_accents(input_str):
+    # nfkd_form = unicodedata.normalize('NFKD', input_str)
+    # return "".join([char for char in nfkd_form if not unicodedata.combining(char)])
+    text_no_accents = unidecode(input_str)
+    return text_no_accents
+
 def remove_special_characters(text):
-    cleaned_text = re.sub(r'[^a-zA-Z0-9\s]', '', text)
-    return cleaned_text
+    text = remove_accents(text)
+    pattern = r'[^\w\s\d.,!?\'"()-;]+'
+    text = re.sub(pattern, '', text)
+    return text
+
+def remove_special_characters_2(text):
+    pattern = r'[^a-zA-Z0-9 ]+'
+    text = re.sub(pattern, '', text)
+    return text
 
 def update_character_count(text):
     return f"{len(text)} characters"
@@ -256,7 +271,7 @@ def predict_bc(model, tokenizer, text):
 
 def predict_mc(model, tokenizer, text):
     tokens = tokenizer(
-        text, padding='max_length', truncation=True, return_tensors="pt", max_length=512
+        text, padding='max_length', truncation=True, return_tensors="pt", max_length=256
     ).to(device)["input_ids"]
     output = model(tokens)
     output_norm = softmax(output.logits.detach().cpu().numpy(), 1)[0]
@@ -271,9 +286,10 @@ def ai_generated_test(ai_option, input):
     segments = split_text_allow_complete_sentences_nltk(input)
     
     for i in range(samples_len):
-        cleaned_text = remove_special_characters(segments[i])
-        bc_score = predict_bc(text_bc_model, text_bc_tokenizer,cleaned_text )
-        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text)
+        cleaned_text_bc = remove_special_characters(segments[i])
+        cleaned_text_mc = remove_special_characters_2(segments[i])
+        bc_score = predict_bc(text_bc_model, text_bc_tokenizer,cleaned_text_bc )
+        mc_score = predict_mc(text_mc_model, text_mc_tokenizer, cleaned_text_mc)
         bc_scores.append(bc_score)
         mc_scores.append(mc_score)
 
@@ -633,5 +649,5 @@ with gr.Blocks() as demo:
 
     date_from = ""
     date_to = ""
-
-demo.launch(share=True, server_name="0.0.0.0", server_port = 80, auth=("polygraf-admin", "test@aisd"))
+  
+demo.launch(share=True, server_name="0.0.0.0", auth=("polygraf-admin", "test@aisd"))

requirements.txt

@@ -21,4 +21,5 @@ textstat
 plotly
 tqdm
 pymupdf
-sentence-transformers
+sentence-transformers
+Unidecode