Stricter moderation

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

app.py

@@ -15,6 +15,8 @@ from video_config import MODEL_FRAME_RATES, calculate_frames
 import asyncio
 from openai import OpenAI
 import base64
+from google.cloud import vision
+from google.oauth2 import service_account
 
 dotenv.load_dotenv()
 
@@ -360,9 +362,10 @@ def poll_generation_status(generation_id):
 
 async def moderate_prompt(prompt: str) -> dict:
     """
-    Check if a text prompt contains NSFW content
+    Check if a text prompt contains NSFW content with strict rules against inappropriate content
     """
     try:
+        # First check with OpenAI moderation
         response = openai_client.moderations.create(input=prompt)
         result = response.results[0]
         
@@ -378,56 +381,153 @@ async def moderate_prompt(prompt: str) -> dict:
                 "reason": f"Content flagged for: {', '.join(flagged_categories)}"
             }
         
+        # Additional checks for keywords related to minors or inappropriate content
+        keywords = [
+            "child", "kid", "minor", "teen", "young", "baby", "infant", "underage",
+            "naked", "nude", "nsfw", "porn", "xxx", "sex", "explicit",
+            "inappropriate", "adult content"
+        ]
+        
+        lower_prompt = prompt.lower()
+        found_keywords = [word for word in keywords if word in lower_prompt]
+        
+        if found_keywords:
+            return {
+                "isNSFW": True,
+                "reason": f"Content contains inappropriate keywords: {', '.join(found_keywords)}"
+            }
+        
         return {"isNSFW": False, "reason": None}
     except Exception as e:
         print(f"Error during prompt moderation: {e}")
-        return {"isNSFW": False, "reason": None}
+        # If there's an error, reject the prompt to be safe
+        return {
+            "isNSFW": True,
+            "reason": "Failed to verify prompt safety - please try again"
+        }
 
 async def moderate_image(image_path: str) -> dict:
     """
-    Check if an image contains NSFW content using OpenAI's vision capabilities
+    Check if an image contains NSFW content using both Google Cloud Vision API's SafeSearch detection
+    and OpenAI's vision model for double verification
     """
     try:
-        # Convert image to base64
+        # Convert image to base64 for OpenAI
         with open(image_path, "rb") as image_file:
             base64_image = base64.b64encode(image_file.read()).decode('utf-8')
         
-        response = openai_client.chat.completions.create(
+        # 1. Google Cloud Vision API Check using proper client library
+        try:
+            # Get service account info from environment
+            service_account_info = json.loads(os.getenv('SERVICE_ACCOUNT_JSON'))
+            
+            # Initialize Vision client with credentials
+            credentials = service_account.Credentials.from_service_account_info(service_account_info)
+            vision_client = vision.ImageAnnotatorClient(credentials=credentials)
+            
+            # Load image content
+            with open(image_path, "rb") as image_file:
+                content = image_file.read()
+            
+            # Create image object
+            image = vision.Image(content=content)
+            
+            # Perform safe search detection
+            response = vision_client.safe_search_detection(image=image)
+            safe_search = response.safe_search_annotation
+            
+            # Map likelihood values
+            likelihood_values = {
+                vision.Likelihood.VERY_LIKELY: 4,
+                vision.Likelihood.LIKELY: 3,
+                vision.Likelihood.POSSIBLE: 2,
+                vision.Likelihood.UNLIKELY: 1,
+                vision.Likelihood.VERY_UNLIKELY: 0,
+                vision.Likelihood.UNKNOWN: 0
+            }
+            
+            # Get likelihood scores
+            adult_score = likelihood_values[safe_search.adult]
+            racy_score = likelihood_values[safe_search.racy]
+            violence_score = likelihood_values[safe_search.violence]
+            medical_score = likelihood_values[safe_search.medical]
+            
+            # Determine if content is NSFW according to Vision API
+            vision_reasons = []
+            if adult_score >= 3:  # LIKELY or VERY_LIKELY
+                vision_reasons.append("adult content")
+            if racy_score >= 3:  # LIKELY or VERY_LIKELY
+                vision_reasons.append("suggestive content")
+            if violence_score >= 3:  # LIKELY or VERY_LIKELY
+                vision_reasons.append("violent content")
+
+                
+            # Print Vision API results
+            print("Google Cloud Vision API Results:")
+            print(f"Adult: {vision.Likelihood(safe_search.adult).name}")
+            print(f"Racy: {vision.Likelihood(safe_search.racy).name}")
+            print(f"Violence: {vision.Likelihood(safe_search.violence).name}")
+            print(f"Medical: {vision.Likelihood(safe_search.medical).name}")
+            
+        except Exception as vision_error:
+            print(f"Error with Vision API: {vision_error}")
+            vision_reasons = []  # Continue with OpenAI check if Vision API fails
+        
+        # 2. OpenAI Vision Check
+        openai_response = openai_client.chat.completions.create(
             model="gpt-4o",
             messages=[
                 {
                     "role": "system",
-                    "content": "You are a content moderation system. Your task is to determine if an image contains NSFW content. Respond with only 'NSFW' if the image contains inappropriate sexual content such as nudity, pornography (especially child pornography), or other explicit material. Otherwise, respond with 'SAFE'."
+                    "content": "You are a content review system. Your task is to analyze the image and determine if it contains any inappropriate content that would make it unsuitable for general audiences. Especially don't allow any content involving minors (children). Please respond with 'SAFE' if the image is appropriate, or 'NSFW: <reason>' if you detect any concerning content."
                 },
                 {
                     "role": "user",
                     "content": [
-                        {"type": "text", "text": "Is this image appropriate or does it contain NSFW content?"},
+                        {"type": "text", "text": "Is this image appropriate for general audiences?"},
                         {"type": "image_url", "image_url": {
                             "url": f"data:image/jpeg;base64,{base64_image}"
                         }}
                     ]
                 }
             ],
-            max_tokens=10
+            max_tokens=150
         )
         
-        result = response.choices[0].message.content.strip()
+        openai_result = openai_response.choices[0].message.content.strip()
+        print("\nOpenAI Vision Check Result:")
+        print(openai_result)
+        
+        # Combine results from both services
+        is_nsfw_vision = bool(vision_reasons)
+        is_nsfw_openai = openai_result.startswith("NSFW:")
         
-        if "NSFW" in result:
+        # If either service flags the content, reject it
+        if is_nsfw_vision or is_nsfw_openai:
+            reasons = []
+            if vision_reasons:
+                reasons.append(f"Google Vision: {', '.join(vision_reasons)}")
+            if is_nsfw_openai:
+                reasons.append(f"OpenAI: {openai_result[5:].strip()}")
+                
             return {
                 "isNSFW": True,
-                "reason": "Image contains potentially inappropriate content"
+                "reason": f"Image contains inappropriate content: {'; '.join(reasons)}"
             }
         
         return {"isNSFW": False, "reason": None}
+        
     except Exception as e:
         print(f"Error during image moderation: {e}")
-        return {"isNSFW": False, "reason": None}
+        # If there's an error, reject the image to be safe
+        return {
+            "isNSFW": True,
+            "reason": "Failed to verify image safety - please try again"
+        }
 
 async def moderate_combined(prompt: str, image_path: str) -> dict:
     """
-    Check if both a prompt and image contain NSFW content together
+    Check if both a prompt and image contain NSFW content together with strict rules
     """
     try:
         # Convert image to base64
@@ -439,14 +539,24 @@ async def moderate_combined(prompt: str, image_path: str) -> dict:
             messages=[
                 {
                     "role": "system",
-                    "content": "You are a content moderation system. Your task is to determine if an image contains NSFW content. The image + text prompt combined will be generated into a video. Please assess if the generated video would be inappropriate in terms of sexual content only (pornography or child pornography). Respond with 'NSFW: reason' if inappropriate, or 'SAFE' if appropriate."
+                    "content": """You are a strict content moderation system. Your task is to determine if an image and prompt combination would result in inappropriate content.
+
+STRICT RULES:
+1. NO content involving minors in any way
+2. NO nudity or sexually suggestive content
+3. NO extreme violence or gore
+4. NO hate speech or discriminatory content
+5. NO illegal activities
+
+Respond with 'NSFW: <reason>' if ANY of these rules are violated, or 'SAFE' if appropriate.
+Be extremely cautious - if there's any doubt, mark it as NSFW."""
                 },
                 {
                     "role": "user",
                     "content": [
                         {
                             "type": "text",
-                            "text": f'Please moderate this image and prompt combination for an image-to-video generation:\n\nPrompt: "{prompt}"'
+                            "text": f'Please moderate this image and prompt combination for an image-to-video generation:\n\nPrompt: "{prompt}"\n\nEnsure NO inappropriate content, especially involving minors.'
                         },
                         {
                             "type": "image_url",
@@ -472,7 +582,11 @@ async def moderate_combined(prompt: str, image_path: str) -> dict:
         }
     except Exception as e:
         print(f"Error during combined moderation: {e}")
-        return {"isNSFW": False, "reason": None}
+        # If there's an error, reject to be safe
+        return {
+            "isNSFW": True,
+            "reason": "Failed to verify content safety - please try again"
+        }
 
 async def generate_video(input_image, subject, duration, selected_index, progress=gr.Progress()):
     try:
@@ -564,7 +678,7 @@ async def generate_video(input_image, subject, duration, selected_index, progres
         raise e
 
 def update_selection(evt: gr.SelectData):
-  selected_lora = loras[evt.index]
+  selected_lora = loras[evt.index]  
   sentence = f"Selected LoRA: {selected_lora['title']}"
   return selected_lora['id'], sentence
 
@@ -954,12 +1068,11 @@ with gr.Blocks(css=css, theme=gr.themes.Soft(primary_hue="indigo", neutral_hue="
     # Add a custom handler to check if inputs are valid
     def check_inputs(subject, image_input, selected_index):
         if not selected_index:
-            return gr.Error("You must select a LoRA before proceeding.")
+            raise gr.Error("You must select a LoRA before proceeding.")
         if not subject.strip():
-            return gr.Error("Please describe your subject.")
+            raise gr.Error("Please describe your subject.")
         if image_input is None:
-            return gr.Error("Please upload an image or select an example image.")
-        return None
+            raise gr.Error("Please upload an image or select an example image.")
     
     # Use gr.on for the button click with validation
     button.click(
@@ -981,4 +1094,4 @@ with gr.Blocks(css=css, theme=gr.themes.Soft(primary_hue="indigo", neutral_hue="
 
 if __name__ == "__main__":
     demo.queue(default_concurrency_limit=20)
-    demo.launch(ssr_mode=False)
+    demo.launch(ssr_mode=False, share=True)