add rag for all models, optimized references-related functions

ai_generate.py

@@ -21,6 +21,9 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnablePassthrough
 from langchain.chains import RetrievalQA
 from langchain_groq import ChatGroq
+from langchain_openai import ChatOpenAI
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_anthropic import ChatAnthropic
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -40,6 +43,30 @@ gemini_client = GenerativeModel("gemini-1.5-pro-001")
 claude_client = anthropic.Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
 
 
+rag_llms = {
+    "LLaMA 3": ChatGroq(
+        temperature=0,
+        model_name="llama3-70b-8192",
+    ),
+    "OpenAI GPT 4o Mini": ChatOpenAI(
+        temperature=0,
+        model_name="gpt-4o-mini",
+    ),
+    "OpenAI GPT 4o": ChatOpenAI(
+        temperature=0,
+        model_name="gpt-4o",
+    ),
+    "OpenAI GPT 4": ChatOpenAI(
+        temperature=0,
+        model_name="gpt-4-turbo",
+    ),
+    "Gemini 1.5 Pro": ChatGoogleGenerativeAI(temperature=0, model="gemini-1.5-pro"),
+    "Claude Sonnet 3.5": ChatAnthropic(
+        temperature=0,
+        model_name="claude-3-5-sonnet-20240620",
+    ),
+}
+
 
 def create_db_with_langchain(path):
     loader = PyMuPDFLoader(path)
@@ -56,11 +83,9 @@ def create_db_with_langchain(path):
     return db
 
 
-def generate_groq_rag(text, model, path):
-    llm = ChatGroq(
-        temperature=0,
-        model_name=model,
-    )
+def generate_rag(text, model, path):
+    print(f"Generating text using RAG for {model}...")
+    llm = rag_llms[model]
     db = create_db_with_langchain(path)
     retriever = db.as_retriever(search_type="mmr", search_kwargs={"k": 4, "fetch_k": 20})
     prompt = hub.pull("rlm/rag-prompt")
@@ -72,7 +97,7 @@ def generate_groq_rag(text, model, path):
     return rag_chain.invoke(text).content
 
 
-def generate_groq_base(text, model):
+def generate_groq(text, model):
     completion = groq_client.chat.completions.create(
         model=model,
         messages=[
@@ -94,13 +119,6 @@ def generate_groq_base(text, model):
     return response
 
 
-def generate_groq(text, model, path):
-    if path:
-        return generate_groq_rag(text, model, path)
-    else:
-        return generate_groq_base(text, model)
-
-
 def generate_openai(text, model, openai_client):
     message = [{"role": "user", "content": text}]
     response = openai_client.chat.completions.create(
@@ -145,16 +163,23 @@ def generate_claude(text, model, claude_client):
 
 
 def generate(text, model, path, api=None):
-
-    if model == "LLaMA 3":
-        return generate_groq(text, "llama3-70b-8192", path)
-    elif model == "OpenAI GPT 4o Mini":
-        return generate_openai(text, "gpt-4o-mini", openai_client)
-    elif model == "OpenAI GPT 4o":
-        return generate_openai(text, "gpt-4o", openai_client)
-    elif model == "OpenAI GPT 4":
-        return generate_openai(text, "gpt-4-turbo", openai_client)
-    elif model == "Gemini 1.5 Pro":
-        return generate_gemini(text, "", gemini_client)
-    elif model == "Claude Sonnet 3.5":
-        return generate_claude(text, "claude-3-5-sonnet-20240620", claude_client)
+    if path:
+        result = generate_rag(text, model, path)
+        if "references" not in result.lower():
+            result += "\n\n" + "References:"
+        result += "\n\n" + f"{path}"
+        return result
+    else:
+        print(f"Generating text for {model}...")
+        if model == "LLaMA 3":
+            return generate_groq(text, "llama3-70b-8192")
+        elif model == "OpenAI GPT 4o Mini":
+            return generate_openai(text, "gpt-4o-mini", openai_client)
+        elif model == "OpenAI GPT 4o":
+            return generate_openai(text, "gpt-4o", openai_client)
+        elif model == "OpenAI GPT 4":
+            return generate_openai(text, "gpt-4-turbo", openai_client)
+        elif model == "Gemini 1.5 Pro":
+            return generate_gemini(text, "", gemini_client)
+        elif model == "Claude Sonnet 3.5":
+            return generate_claude(text, "claude-3-5-sonnet-20240620", claude_client)

app.py

@@ -63,23 +63,41 @@ def clean_text(text: str) -> str:
     return "\n".join(cleaned_paragraphs)
 
 
-def format_and_correct(text: str) -> str:
-    prompt = f"""
-    Please correct the formatting, grammar, and spelling errors in the following text without changing its content significantly. Ensure proper paragraph breaks and maintain the original content:
-    {text}
-    """
-    corrected_text = generate(prompt, "Llama 3", None)
-    return clean_text(corrected_text)
+def split_text_from_refs(text: str, sep="\n"):
+    lines = text.split("\n")
+    references = []
+    article_text = []
+    index_pattern = re.compile(r"\[(\d+)\]")
+    in_references = False
 
+    for line in lines:
+        if line.strip().lower() == "references" or line.strip().lower() == "references:":
+            in_references = True
+            continue
+        if line.strip().lower().startswith("references:"):
+            in_references = True
+        if in_references:
+            matches = index_pattern.split(line)
+            for match in matches:
+                if match.strip() and not match.isdigit() and not match.strip().lower().startswith("references:"):
+                    references.append(match.strip())
+        else:
+            article_text.append(line)
 
-def format_and_correct_para(text: str) -> str:
-    paragraphs = text.split("\n")
-    corrected_paragraphs = []
-    for paragraph in paragraphs:
-        corrected = format_and_correct(paragraph)
-        corrected_paragraphs.append(corrected)
-    corrected_text = "\n\n".join(corrected_paragraphs)
-    return corrected_text
+    formatted_refs = []
+    for i, ref in enumerate(references, 1):
+        ref = remove_bracketed_numbers(ref)
+        formatted_refs.append(f"[{i}] {ref}{sep}")
+
+    return "\n\n".join(article_text), f"{sep}{sep}References:{sep}" + f"{sep}".join(formatted_refs)
+
+
+def ends_with_references(text):
+    # Define a regular expression pattern for variations of "References:"
+    pattern = re.compile(r"\b[Rr]eferences:\s*$", re.IGNORECASE | re.MULTILINE)
+
+    # Check if the text ends with any form of "References:"
+    return bool(pattern.search(text.strip()))
 
 
 def format_and_correct_language_check(text: str) -> str:
@@ -109,7 +127,7 @@ def ai_generated_test(text, model="BC Original"):
     return predict(models[model], tokenizers[model], text)
 
 
-def process_text(text, model="BC Original"):
+def detection_polygraf(text, model="BC Original"):
     # sentences = split_into_sentences(text)
     sentences = nltk.sent_tokenize(text)
     num_sentences = len(sentences)
@@ -146,12 +164,11 @@ def process_text(text, model="BC Original"):
                 overall_scores.append(avg_score)
             i = i + 1
         combined_sentences = " ".join(colored_sentences)
-        print(combined_sentences)
         colored_paragraphs.append(combined_sentences)
 
     overall_score = sum(overall_scores) / len(overall_scores)
     overall_score = {"HUMAN": 1 - overall_score, "AI": overall_score}
-    return overall_score, format_references("<br><br>".join(colored_paragraphs))
+    return overall_score, "<br><br>".join(colored_paragraphs)
 
 
 ai_check_options = [
@@ -202,12 +219,14 @@ class GPT2PPL:
 def ai_generated_test_gptzero(text):
     gptzero_model = GPT2PPL()
     result = gptzero_model(text)
-    print(result)
     return result, None
 
 
 def highlighter_polygraf(text, model="Polygraf AI (Base Model)"):
-    return process_text(text=text, model=model)
+    body, references = split_text_from_refs(text, "<br>")
+    score, text = detection_polygraf(text=body, model=model)
+    text = text + "<br>" + references
+    return score, text
 
 
 def ai_check(text: str, option: str):
@@ -224,7 +243,6 @@ def ai_check(text: str, option: str):
 def generate_prompt(settings: Dict[str, str]) -> str:
     prompt = f"""
     I am a {settings['role']}
-
     Write a {settings['article_length']} words (around) {settings['format']} on {settings['topic']}.
     
     Style and Tone:
@@ -240,10 +258,11 @@ def generate_prompt(settings: Dict[str, str]) -> str:
     {', '.join(settings['keywords'])}
     
     Additional requirements:
+    - Don't start with "Here is a...", start with the requested text directly
     - Include {settings['num_examples']} relevant examples or case studies
     - Incorporate data or statistics from {', '.join(settings['references'])}
     - End with a {settings['conclusion_type']} conclusion
-    - Add a "References" section at the end with at least 3 credible sources, formatted as [1], [2], etc.
+    - Add a "References" section in the format "References:\n" at the end with at least 3 credible sources, formatted as [1], [2], etc. with each source on their own line
     - Do not make any headline, title bold.
     {settings['sources']}
     
@@ -256,12 +275,11 @@ def generate_prompt(settings: Dict[str, str]) -> str:
 def regenerate_prompt(settings: Dict[str, str]) -> str:
     prompt = f"""
     I am a {settings['role']}
-
     "{settings['generated_article']}"
-
     Edit the given text based on user comments.
     
     Comments:
+    - Don't start with "Here is a...", start with the requested text directly
     - {settings['user_comments']}
     - The original content should not be changed. Make minor modifications based on user comments above.
     - Keep the references the same as the given text in the same format.
@@ -319,30 +337,12 @@ def generate_article(
     else:
         prompt = generate_prompt(settings)
 
-    print(prompt)
-    # TODO: Why do we need this ??
-    if ai_model in ["OpenAI GPT 3.5"]:
-        response = openai.ChatCompletion.create(
-            model="gpt-4" if ai_model == "OpenAI GPT 4" else "gpt-3.5-turbo",
-            messages=[
-                {
-                    "role": "system",
-                    "content": "You are a professional content writer with expertise in various fields.",
-                },
-                {"role": "user", "content": prompt},
-            ],
-            max_tokens=3000,
-            n=1,
-            stop=None,
-            temperature=0.7,
-        )
-        article = response.choices[0].message.content.strip()
-    else:
-        article = generate(
-            prompt,
-            ai_model,
-            pdf_file_input,  # api_key
-        )
+    print("Generated Prompt...\n", prompt)
+    article = generate(
+        prompt,
+        ai_model,
+        pdf_file_input,  # api_key
+    )
 
     return clean_text(article)
 
@@ -355,14 +355,16 @@ def humanize(
     top_k: int = 50,
     length_penalty: float = 1,
 ) -> str:
+    body, references = split_text_from_refs(text)
     result = paraphrase_text(
-        text=text,
+        text=body,
         model_name=model,
         temperature=temperature,
         repetition_penalty=repetition_penalty,
         top_k=top_k,
         length_penalty=length_penalty,
     )
+    result = result + "\n\n" + references
     return format_and_correct_language_check(result)
 
 
@@ -377,18 +379,20 @@ def format_references(text: str) -> str:
     lines = text.split("\n")
     references = []
     article_text = []
+    index_pattern = re.compile(r"\[(\d+)\]")
     in_references = False
 
     for line in lines:
-        if (
-            line.strip().lower() == "references"
-            or line.strip().lower() == "references:"
-            or line.strip().lower().startswith("references:")
-        ):
+        if line.strip().lower() == "references" or line.strip().lower() == "references:":
             in_references = True
             continue
+        if line.strip().lower().startswith("references:"):
+            in_references = True
         if in_references:
-            references.append(line.strip())
+            matches = index_pattern.split(line)
+            for match in matches:
+                if match.strip() and not match.isdigit() and not match.strip().lower().startswith("references:"):
+                    references.append(match.strip())
         else:
             article_text.append(line)
 
@@ -430,25 +434,26 @@ def generate_and_format(
     generated_article: str = None,
     user_comments: str = None,
 ):
-    date_from = build_date(year_from, month_from, day_from)
-    date_to = build_date(year_to, month_to, day_to)
-    sorted_date = f"date:r:{date_from}:{date_to}"
     content_string = ""
-    final_query = topic
-    if include_sites:
-        site_queries = [f"site:{site.strip()}" for site in include_sites.split(",")]
-        final_query += " " + " OR ".join(site_queries)
-    if exclude_sites:
-        exclude_queries = [f"-site:{site.strip()}" for site in exclude_sites.split(",")]
-        final_query += " " + " ".join(exclude_queries)
-    print(f"Final Query: {final_query}")
-
     if google_search_check:
+        date_from = build_date(year_from, month_from, day_from)
+        date_to = build_date(year_to, month_to, day_to)
+        sorted_date = f"date:r:{date_from}:{date_to}"
+        final_query = topic
+        if include_sites:
+            site_queries = [f"site:{site.strip()}" for site in include_sites.split(",")]
+            final_query += " " + " OR ".join(site_queries)
+        if exclude_sites:
+            exclude_queries = [f"-site:{site.strip()}" for site in exclude_sites.split(",")]
+            final_query += " " + " ".join(exclude_queries)
+        print(f"Google Search Query: {final_query}")
         url_content = google_search(final_query, sorted_date, domains_to_include)
         content_string = "\n".join(
             f"{url.strip()}: \n{content.strip()[:2000]}" for url, content in url_content.items()
         )
-        content_string = "Use the trusted information here from the URLs I've found for you:\n" + content_string
+        content_string = (
+            "Use the trusted information here from the URLs and add them as References:\n" + content_string
+        )
     article = generate_article(
         input_role,
         topic,
@@ -470,6 +475,10 @@ def generate_and_format(
         generated_article,
         user_comments,
     )
+    if ends_with_references(article) and url_content is not None:
+        for url in url_content.keys():
+            article += f"\n{url}"
+
     return format_references(article)
 
 

plagiarism.py

@@ -15,7 +15,8 @@ def clean_html(text):
     result += article.title + "\n"
     paragraphs = justext.justext(text, justext.get_stoplist("English"))
     for paragraph in paragraphs:
-        result += paragraph.text
+        if not paragraph.is_boilerplate:
+            result += paragraph.text
     return result
 
 
@@ -128,6 +129,7 @@ def google_search(
             break
         if soup:
             text = clean_html(soup.text)
-            result_content[url] = text
-            count += 1
+            if len(text) > 500:
+                result_content[url] = text
+                count += 1
     return result_content

requirements.txt

@@ -21,4 +21,8 @@ pymupdf
 chromadb
 language-tool-python
 anthropic
-google-generativeai
+google-generativeai
+langchain-google-genai
+langchain-anthropic
+langchain-openai
+vertexai