Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -4,48 +4,55 @@ import re
|
|
4 |
|
5 |
sentiment_pipeline = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
|
6 |
|
|
|
|
|
7 |
moderation_guidelines = """
|
8 |
- Allow positive messages
|
9 |
- Block cuss words
|
10 |
- Allow negative comments about individuals but block negative comments against a community
|
11 |
-
- Block
|
12 |
"""
|
13 |
|
|
|
|
|
|
|
14 |
|
15 |
-
|
16 |
-
"damn", "hell", "shit", "fuck", "ass", "bastard", "bitch", "bollocks", "bugger",
|
17 |
-
"bullshit", "crap", "dammit", "douche", "dumbass", "faggot", "jackass", "jerk",
|
18 |
-
"motherfucker", "piss", "prick", "slut", "son of a bitch", "twat", "wanker"
|
19 |
-
}
|
20 |
-
|
21 |
-
community_terms = {"religion", "race", "ethnicity", "group", "community", "gender", "china"}
|
22 |
|
23 |
def extract_blocked_words(guidelines):
|
24 |
-
"""
|
25 |
match = re.search(r"block words:\s*(.*)", guidelines.lower())
|
26 |
-
return
|
27 |
|
28 |
def moderate_message(message, guidelines):
|
29 |
-
"""Moderates a message based on sentiment and dynamic
|
30 |
|
31 |
sentiment = sentiment_pipeline(message)[0]['label']
|
32 |
|
33 |
-
|
34 |
blocked_words = extract_blocked_words(guidelines)
|
35 |
-
|
36 |
-
|
37 |
allow_positive = "allow positive" in guidelines.lower()
|
38 |
block_cuss_words = "block cuss" in guidelines.lower()
|
39 |
allow_negative_personal = "allow negative comments about individuals" in guidelines.lower()
|
40 |
block_negative_community = "block negative comments against a community" in guidelines.lower()
|
|
|
|
|
|
|
41 |
|
42 |
-
|
43 |
-
words
|
44 |
-
if block_cuss_words and (words & default_cuss_words):
|
45 |
return "β Message Blocked: Contains inappropriate language."
|
46 |
|
|
|
47 |
if words & blocked_words:
|
48 |
return "π« Message Blocked: Contains restricted words."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
49 |
if sentiment == "POSITIVE" and allow_positive:
|
50 |
return f"β
Allowed (Positive): {message}"
|
51 |
|
@@ -59,14 +66,14 @@ def moderate_message(message, guidelines):
|
|
59 |
|
60 |
with gr.Blocks() as demo:
|
61 |
gr.Markdown("### π‘οΈ AI-Powered Moderation System")
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
with gr.Row():
|
66 |
msg_input = gr.Textbox(label="Enter Message")
|
67 |
msg_output = gr.Textbox(label="Moderation Result", interactive=False)
|
68 |
|
69 |
moderate_btn = gr.Button("Check Message")
|
|
|
70 |
moderate_btn.click(moderate_message, inputs=[msg_input, guidelines_input], outputs=[msg_output])
|
71 |
|
72 |
demo.launch()
|
|
|
4 |
|
5 |
sentiment_pipeline = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
|
6 |
|
7 |
+
ner_pipeline = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english", aggregation_strategy="simple")
|
8 |
+
|
9 |
moderation_guidelines = """
|
10 |
- Allow positive messages
|
11 |
- Block cuss words
|
12 |
- Allow negative comments about individuals but block negative comments against a community
|
13 |
+
- Block personal names
|
14 |
"""
|
15 |
|
16 |
+
default_cuss_words = {"damn", "hell", "shit", "fuck", "ass", "bastard", "bitch", "bollocks", "bugger",
|
17 |
+
"bullshit", "crap", "dammit", "douche", "dumbass", "faggot", "jackass", "jerk",
|
18 |
+
"motherfucker", "piss", "prick", "slut", "son of a bitch", "twat", "wanker"}
|
19 |
|
20 |
+
community_terms = {"religion", "race", "ethnicity", "group", "community", "gender"}
|
|
|
|
|
|
|
|
|
|
|
|
|
21 |
|
22 |
def extract_blocked_words(guidelines):
|
23 |
+
"""Extracts blocked words from moderation guidelines."""
|
24 |
match = re.search(r"block words:\s*(.*)", guidelines.lower())
|
25 |
+
return {word.strip() for word in match.group(1).split(",") if word.strip()} if match else set()
|
26 |
|
27 |
def moderate_message(message, guidelines):
|
28 |
+
"""Moderates a message based on sentiment and dynamic moderation rules."""
|
29 |
|
30 |
sentiment = sentiment_pipeline(message)[0]['label']
|
31 |
|
|
|
32 |
blocked_words = extract_blocked_words(guidelines)
|
|
|
|
|
33 |
allow_positive = "allow positive" in guidelines.lower()
|
34 |
block_cuss_words = "block cuss" in guidelines.lower()
|
35 |
allow_negative_personal = "allow negative comments about individuals" in guidelines.lower()
|
36 |
block_negative_community = "block negative comments against a community" in guidelines.lower()
|
37 |
+
block_personal_names = "block personal names" in guidelines.lower()
|
38 |
+
|
39 |
+
words = set(re.findall(r'\w+', message.lower()))
|
40 |
|
41 |
+
# 1. Block Cuss Words
|
42 |
+
if block_cuss_words and words & default_cuss_words:
|
|
|
43 |
return "β Message Blocked: Contains inappropriate language."
|
44 |
|
45 |
+
# 2. Block Dynamically Defined Words
|
46 |
if words & blocked_words:
|
47 |
return "π« Message Blocked: Contains restricted words."
|
48 |
+
|
49 |
+
# 3. Block Personal Names Dynamically
|
50 |
+
if block_personal_names:
|
51 |
+
entities = ner_pipeline(message)
|
52 |
+
for entity in entities:
|
53 |
+
if entity['entity_group'] == 'PER':
|
54 |
+
return "π« Message Blocked: Contains personal names."
|
55 |
+
|
56 |
if sentiment == "POSITIVE" and allow_positive:
|
57 |
return f"β
Allowed (Positive): {message}"
|
58 |
|
|
|
66 |
|
67 |
with gr.Blocks() as demo:
|
68 |
gr.Markdown("### π‘οΈ AI-Powered Moderation System")
|
69 |
+
guidelines_input = gr.Textbox(value=moderation_guidelines, label="Moderation Guidelines (Admins Can Update)", lines=6)
|
70 |
+
|
|
|
71 |
with gr.Row():
|
72 |
msg_input = gr.Textbox(label="Enter Message")
|
73 |
msg_output = gr.Textbox(label="Moderation Result", interactive=False)
|
74 |
|
75 |
moderate_btn = gr.Button("Check Message")
|
76 |
+
|
77 |
moderate_btn.click(moderate_message, inputs=[msg_input, guidelines_input], outputs=[msg_output])
|
78 |
|
79 |
demo.launch()
|