KavinduHansaka commited on
Commit
31c512a
·
verified ·
1 Parent(s): debc16e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +19 -15
app.py CHANGED
@@ -4,11 +4,12 @@ from detoxify import Detoxify
4
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
5
  import torch
6
 
7
- # Load models
8
  tox_model = Detoxify('multilingual')
9
  ai_tokenizer = AutoTokenizer.from_pretrained("openai-community/roberta-base-openai-detector")
10
  ai_model = AutoModelForSequenceClassification.from_pretrained("openai-community/roberta-base-openai-detector")
11
 
 
12
  TOXICITY_THRESHOLD = 0.7
13
  AI_THRESHOLD = 0.5
14
 
@@ -16,27 +17,30 @@ def detect_ai(text):
16
  with torch.no_grad():
17
  inputs = ai_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
18
  logits = ai_model(**inputs).logits
19
- prob = torch.sigmoid(logits).squeeze().item()
20
- return round(prob, 4)
21
 
22
  def classify_comments(comment_list):
23
  results = tox_model.predict(comment_list)
24
  df = pd.DataFrame(results, index=comment_list).round(4)
25
  df.columns = [col.replace("_", " ").title().replace(" ", "_") for col in df.columns]
26
  df.columns = [col.replace("_", " ") for col in df.columns]
27
- df["⚠️ Warning"] = df.apply(lambda row: "⚠️ High Risk" if any(score > TOXICITY_THRESHOLD for score in row) else "✅ Safe", axis=1)
 
 
 
28
  df["🧪 AI Probability"] = [detect_ai(c) for c in df.index]
29
- df["🧪 AI Detection"] = df["🧪 AI Probability"].apply(lambda x: "🤖 Likely AI" if x > AI_THRESHOLD else "🧍 Human")
 
 
30
  return df
31
 
32
  def run_classification(text_input, csv_file):
33
  comment_list = []
34
 
35
- # From text input
36
  if text_input.strip():
37
  comment_list += [c.strip() for c in text_input.strip().split('\n') if c.strip()]
38
 
39
- # From CSV
40
  if csv_file:
41
  df = pd.read_csv(csv_file.name)
42
  if 'comment' not in df.columns:
@@ -51,20 +55,20 @@ def run_classification(text_input, csv_file):
51
  csv_data.insert(0, "Comment", df.index)
52
  return df, ("toxicity_predictions.csv", csv_data.to_csv(index=False).encode())
53
 
54
- # UI layout
55
  with gr.Blocks(title="🌍 Toxic Comment & AI Detector") as app:
56
  gr.Markdown("## 🌍 Toxic Comment & AI Detector")
57
- gr.Markdown("Detects multilingual toxicity and whether the comment is AI-generated. Paste text or upload CSV.")
58
 
59
  with gr.Row():
60
- text_input = gr.Textbox(lines=8, label="💬 Enter Comments (one per line)")
61
- file_input = gr.File(label="📥 Upload CSV (with 'comment' column)")
62
 
63
- submit_btn = gr.Button("🔍 Analyze Comments")
64
- output_table = gr.Dataframe(label="📊 Results")
65
- download_btn = gr.File(label="📤 Download CSV")
66
 
67
- submit_btn.click(fn=run_classification, inputs=[text_input, file_input], outputs=[output_table, download_btn])
68
 
69
  if __name__ == "__main__":
70
  app.launch()
 
4
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
5
  import torch
6
 
7
+ # Load models once
8
  tox_model = Detoxify('multilingual')
9
  ai_tokenizer = AutoTokenizer.from_pretrained("openai-community/roberta-base-openai-detector")
10
  ai_model = AutoModelForSequenceClassification.from_pretrained("openai-community/roberta-base-openai-detector")
11
 
12
+ # Thresholds
13
  TOXICITY_THRESHOLD = 0.7
14
  AI_THRESHOLD = 0.5
15
 
 
17
  with torch.no_grad():
18
  inputs = ai_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
19
  logits = ai_model(**inputs).logits
20
+ probs = torch.softmax(logits, dim=1).squeeze().tolist()
21
+ return round(probs[1], 4) # Probability of AI-generated
22
 
23
  def classify_comments(comment_list):
24
  results = tox_model.predict(comment_list)
25
  df = pd.DataFrame(results, index=comment_list).round(4)
26
  df.columns = [col.replace("_", " ").title().replace(" ", "_") for col in df.columns]
27
  df.columns = [col.replace("_", " ") for col in df.columns]
28
+ df["⚠️ Warning"] = df.apply(
29
+ lambda row: "⚠️ High Risk" if any(score > TOXICITY_THRESHOLD for score in row) else "✅ Safe",
30
+ axis=1
31
+ )
32
  df["🧪 AI Probability"] = [detect_ai(c) for c in df.index]
33
+ df["🧪 AI Detection"] = df["🧪 AI Probability"].apply(
34
+ lambda x: "🤖 Likely AI" if x > AI_THRESHOLD else "🧍 Human"
35
+ )
36
  return df
37
 
38
  def run_classification(text_input, csv_file):
39
  comment_list = []
40
 
 
41
  if text_input.strip():
42
  comment_list += [c.strip() for c in text_input.strip().split('\n') if c.strip()]
43
 
 
44
  if csv_file:
45
  df = pd.read_csv(csv_file.name)
46
  if 'comment' not in df.columns:
 
55
  csv_data.insert(0, "Comment", df.index)
56
  return df, ("toxicity_predictions.csv", csv_data.to_csv(index=False).encode())
57
 
58
+ # Build the Gradio UI
59
  with gr.Blocks(title="🌍 Toxic Comment & AI Detector") as app:
60
  gr.Markdown("## 🌍 Toxic Comment & AI Detector")
61
+ gr.Markdown("Detects multilingual toxicity and whether a comment is AI-generated. Paste comments or upload a CSV.")
62
 
63
  with gr.Row():
64
+ text_input = gr.Textbox(lines=8, label="💬 Paste Comments (one per line)")
65
+ csv_input = gr.File(label="📥 Upload CSV (must have 'comment' column)")
66
 
67
+ submit_button = gr.Button("🔍 Analyze Comments")
68
+ output_table = gr.Dataframe(label="📊 Prediction Results")
69
+ download_button = gr.File(label="📤 Download CSV")
70
 
71
+ submit_button.click(fn=run_classification, inputs=[text_input, csv_input], outputs=[output_table, download_button])
72
 
73
  if __name__ == "__main__":
74
  app.launch()