Merge branch 'main' into staging

.gitignore

@@ -1,3 +1,5 @@
 __pycache__/
 .env
-nohup.out
+nohup.out
+*.out
+*.log

ai_generate.py

@@ -3,6 +3,12 @@ from openai import OpenAI
 import os
 from transformers import pipeline
 from groq import Groq
+import base64
+import vertexai
+from vertexai.generative_models import GenerativeModel, Part, FinishReason
+import vertexai.preview.generative_models as generative_models
+import google.generativeai as genai
+import anthropic
 from langchain_community.document_loaders import PyMuPDFLoader
 from langchain_community.document_loaders import TextLoader
 from langchain_community.embeddings.sentence_transformer import (
@@ -19,9 +25,20 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
+os.environ["GRPC_VERBOSITY"] = "ERROR"
+os.environ["GLOG_minloglevel"] = "2"
+
 groq_client = Groq(
     api_key=os.environ.get("GROQ_API_KEY"),
 )
+openai_client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+# give access to all APIs for GCP instance
+# gcloud auth application-default login
+genai.configure(api_key=os.environ.get("GENAI_API_KEY"))
+vertexai.init(project="proprietary-info-detection", location="us-central1")
+gemini_client = GenerativeModel("gemini-1.5-pro-001")
+claude_client = anthropic.Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+
 
 
 def create_db_with_langchain(path):
@@ -67,7 +84,6 @@ def generate_groq_base(text, model):
         ],
         temperature=1,
         max_tokens=1024,
-        top_p=1,
         stream=True,
         stop=None,
     )
@@ -88,35 +104,57 @@ def generate_groq(text, model, path):
 def generate_openai(text, model, openai_client):
     message = [{"role": "user", "content": text}]
     response = openai_client.chat.completions.create(
-        model=model, messages=message, temperature=0.2, max_tokens=800, frequency_penalty=0.0
+        model=model,
+        messages=message,
+        temperature=1,
+        max_tokens=1024,
     )
     return response.choices[0].message.content
 
 
-def generate(text, model, path, api):
-    if model == "Llama 3":
+def generate_gemini(text, model, gemini_client):
+    safety_settings = {
+        generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+        generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+        generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+        generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    }
+    generation_config = {
+        "max_output_tokens": 1024,
+        "temperature": 1.0,
+        "top_p": 1.0,
+    }
+    response = gemini_client.generate_content(
+        [text],
+        generation_config=generation_config,
+        safety_settings=safety_settings,
+        stream=False,
+    )
+    return response.text
+
+
+def generate_claude(text, model, claude_client):
+    response = claude_client.messages.create(
+        model=model,
+        max_tokens=1024,
+        temperature=1.0,
+        system="You are helpful assistant.",
+        messages=[{"role": "user", "content": [{"type": "text", "text": text}]}],
+    )
+    return response.content[0].text.strip()
+
+
+def generate(text, model, path, api=None):
+
+    if model == "LLaMA 3":
         return generate_groq(text, "llama3-70b-8192", path)
-    elif model == "Groq":
-        return generate_groq(text, "llama3-groq-70b-8192-tool-use-preview", path)
-    elif model == "Mistral":
-        return generate_groq(text, "mixtral-8x7b-32768", path)
-    elif model == "Gemma":
-        return generate_groq(text, "gemma2-9b-it", path)
-    elif model == "OpenAI GPT 3.5":
-        try:
-            openai_client = OpenAI(api_key=api)
-            return generate_openai(text, "gpt-3.5-turbo", openai_client)
-        except:
-            return "Please add a valid API key"
-    elif model == "OpenAI GPT 4":
-        try:
-            openai_client = OpenAI(api_key=api)
-            return generate_openai(text, "gpt-4-turbo", openai_client)
-        except:
-            return "Please add a valid API key"
+    elif model == "OpenAI GPT 4o Mini":
+        return generate_openai(text, "gpt-4o-mini", openai_client)
     elif model == "OpenAI GPT 4o":
-        try:
-            openai_client = OpenAI(api_key=api)
-            return generate_openai(text, "gpt-4o", openai_client)
-        except:
-            return "Please add a valid API key"
+        return generate_openai(text, "gpt-4o", openai_client)
+    elif model == "OpenAI GPT 4":
+        return generate_openai(text, "gpt-4-turbo", openai_client)
+    elif model == "Gemini 1.5 Pro":
+        return generate_gemini(text, "", gemini_client)
+    elif model == "Claude Sonnet 3.5":
+        return generate_claude(text, "claude-3-5-sonnet-20240620", claude_client)

app.py

@@ -1,9 +1,11 @@
+"""
+nohup python3 app.py &
+"""
+
 import openai
 import gradio as gr
 from typing import Dict, List
 import re
-from humanize import paraphrase_text
-from ai_generate import generate
 import requests
 import language_tool_python
 import torch
@@ -13,12 +15,13 @@ from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipe
 from scipy.special import softmax
 from collections import defaultdict
 import nltk
+from datetime import date
+
 from utils import remove_special_characters
 from plagiarism import google_search, months, domain_list, build_date
-from datetime import date
+from humanize import paraphrase_text, device
+from ai_generate import generate
 
-# Check if CUDA is available
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(f"Using device: {device}")
 
 models = {
@@ -65,7 +68,7 @@ def format_and_correct(text: str) -> str:
     Please correct the formatting, grammar, and spelling errors in the following text without changing its content significantly. Ensure proper paragraph breaks and maintain the original content:
     {text}
     """
-    corrected_text = generate(prompt, "Groq", None)
+    corrected_text = generate(prompt, "Llama 3", None)
     return clean_text(corrected_text)
 
 
@@ -287,7 +290,7 @@ def generate_article(
     conclusion_type: str,
     ai_model: str,
     content_string: str,
-    api_key: str = None,
+    # api_key: str = None,
     pdf_file_input=None,
     generated_article: str = None,
     user_comments: str = None,
@@ -317,7 +320,8 @@ def generate_article(
         prompt = generate_prompt(settings)
 
     print(prompt)
-    if ai_model in ["OpenAI GPT 3.5", "OpenAI GPT 4"]:
+    # TODO: Why do we need this ??
+    if ai_model in ["OpenAI GPT 3.5"]:
         response = openai.ChatCompletion.create(
             model="gpt-4" if ai_model == "OpenAI GPT 4" else "gpt-3.5-turbo",
             messages=[
@@ -334,7 +338,11 @@ def generate_article(
         )
         article = response.choices[0].message.content.strip()
     else:
-        article = generate(prompt, ai_model, pdf_file_input, api_key)
+        article = generate(
+            prompt,
+            ai_model,
+            pdf_file_input,  # api_key
+        )
 
     return clean_text(article)
 
@@ -407,7 +415,7 @@ def generate_and_format(
     num_examples,
     conclusion_type,
     ai_model,
-    api_key,
+    # api_key,
     google_search_check,
     year_from,
     month_from,
@@ -416,7 +424,8 @@ def generate_and_format(
     month_to,
     day_to,
     domains_to_include,
-    search_keywords,
+    include_sites,
+    exclude_sites,
     pdf_file_input,
     generated_article: str = None,
     user_comments: str = None,
@@ -426,10 +435,14 @@ def generate_and_format(
     sorted_date = f"date:r:{date_from}:{date_to}"
     content_string = ""
     final_query = topic
-    if search_keywords != "":
-        quoted_keywords = [f'"{keyword.strip()}"' for keyword in search_keywords.split(",")]
-        final_query = final_query + " " + " ".join(quoted_keywords)
-    print(final_query)
+    if include_sites:
+        site_queries = [f"site:{site.strip()}" for site in include_sites.split(",")]
+        final_query += " " + " OR ".join(site_queries)
+    if exclude_sites:
+        exclude_queries = [f"-site:{site.strip()}" for site in exclude_sites.split(",")]
+        final_query += " " + " ".join(exclude_queries)
+    print(f"Final Query: {final_query}")
+
     if google_search_check:
         url_content = google_search(final_query, sorted_date, domains_to_include)
         content_string = "\n".join(
@@ -452,7 +465,7 @@ def generate_and_format(
         conclusion_type,
         ai_model,
         content_string,
-        api_key,
+        # api_key,
         pdf_file_input,
         generated_article,
         user_comments,
@@ -501,6 +514,11 @@ def create_interface():
                                 "Research paper",
                                 "News article",
                                 "White paper",
+                                "LinkedIn post",
+                                "X (Twitter) post",
+                                "Instagram Video Content",
+                                "TikTok Video Content",
+                                "Facebook post",
                             ],
                             value="Article",
                             label="Format",
@@ -602,6 +620,25 @@ def create_interface():
                     with gr.Row():
                         google_search_check = gr.Checkbox(label="Enable Google Search For Recent Sources", value=True)
                     with gr.Group(visible=True) as search_options:
+                        with gr.Row():
+                            include_sites = gr.Textbox(
+                                label="Include Specific Websites",
+                                placeholder="Enter comma-separated keywords",
+                                elem_classes="input-highlight-yellow",
+                            )
+                        with gr.Row():
+                            exclude_sites = gr.Textbox(
+                                label="Exclude Specific Websites",
+                                placeholder="Enter comma-separated keywords",
+                                elem_classes="input-highlight-yellow",
+                            )
+                        with gr.Row():
+                            domains_to_include = gr.Dropdown(
+                                domain_list,
+                                value=domain_list,
+                                multiselect=True,
+                                label="Domains To Include",
+                            )
                         with gr.Row():
                             month_from = gr.Dropdown(
                                 choices=months,
@@ -622,32 +659,26 @@ def create_interface():
                             day_to = gr.Textbox(label="To Day", value=d1[0])
                             year_to = gr.Textbox(label="To Year", value=d1[2])
 
-                        with gr.Row():
-                            domains_to_include = gr.Dropdown(
-                                domain_list,
-                                value=domain_list,
-                                multiselect=True,
-                                label="Domains To Include",
-                            )
-                        with gr.Row():
-                            search_keywords = gr.Textbox(
-                                label="Keywords",
-                                placeholder="Enter comma-separated keywords",
-                                elem_classes="input-highlight-yellow",
-                            )
                     gr.Markdown("# Add Optional PDF File with Information", elem_classes="text-center text-3xl mb-6")
                     pdf_file_input = gr.File(label="Upload PDF")
 
                 with gr.Group():
                     gr.Markdown("## AI Model Configuration", elem_classes="text-xl mb-4")
                     ai_generator = gr.Dropdown(
-                        choices=["Llama 3", "Groq", "Mistral", "Gemma", "OpenAI GPT 3.5", "OpenAI GPT 4"],
-                        value="Llama 3",
+                        choices=[
+                            "OpenAI GPT 4",
+                            "OpenAI GPT 4o",
+                            "OpenAI GPT 4o Mini",
+                            "Claude Sonnet 3.5",
+                            "Gemini 1.5 Pro",
+                            "LLaMA 3",
+                        ],
+                        value="OpenAI GPT 4o Mini",
                         label="AI Model",
                         elem_classes="input-highlight-pink",
                     )
-                    input_api = gr.Textbox(label="API Key", visible=False)
-                    ai_generator.change(update_visibility_api, ai_generator, input_api)
+                    # input_api = gr.Textbox(label="API Key", visible=False)
+                    # ai_generator.change(update_visibility_api, ai_generator, input_api)
 
                 generate_btn = gr.Button("Generate Article", variant="primary")
 
@@ -738,7 +769,7 @@ def create_interface():
                 input_num_examples,
                 input_conclusion,
                 ai_generator,
-                input_api,
+                # input_api,
                 google_search_check,
                 year_from,
                 month_from,
@@ -747,7 +778,8 @@ def create_interface():
                 month_to,
                 day_to,
                 domains_to_include,
-                search_keywords,
+                include_sites,
+                exclude_sites,
                 pdf_file_input,
             ],
             outputs=[output_article],
@@ -770,7 +802,7 @@ def create_interface():
                 input_num_examples,
                 input_conclusion,
                 ai_generator,
-                input_api,
+                # input_api,
                 google_search_check,
                 year_from,
                 month_from,
@@ -781,7 +813,8 @@ def create_interface():
                 domains_to_include,
                 pdf_file_input,
                 output_article,
-                search_keywords,
+                include_sites,
+                exclude_sites,
                 ai_comments,
             ],
             outputs=[output_article],

plagiarism.py

@@ -114,6 +114,7 @@ def google_search(
         api_key,
         cse_id,
     )
+    print("URLS: ", url_list)
     print("GOOGLE SEARCH PROCESSING TIME: ", time.perf_counter() - start_time)
     # Scrape URLs in list
     start_time = time.perf_counter()

requirements.txt

@@ -19,4 +19,6 @@ sentence-transformers
 langchain-community
 pymupdf
 chromadb
-language-tool-python
+language-tool-python
+anthropic
+google-generativeai