changed split logic to resolve short generated text, more search website and some logging

ai_generate.py

@@ -17,7 +17,7 @@ from langchain_community.embeddings.sentence_transformer import (
 )
 from langchain.schema import StrOutputParser
 from langchain_community.vectorstores import Chroma
-from langchain_text_splitters import CharacterTextSplitter
+from langchain_text_splitters import CharacterTextSplitter, RecursiveCharacterTextSplitter
 from langchain import hub
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnablePassthrough, RunnableMap
@@ -44,10 +44,17 @@ vertexai.init(project="proprietary-info-detection", location="us-central1")
 gemini_client = GenerativeModel("gemini-1.5-pro-001")
 claude_client = anthropic.Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
 
+# LLM params
 # For GPT-4 1 word is about 1.3 tokens.
 temperature = 1.0
 max_tokens = 2048
 
+# RAG params
+CHUNK_SIZE = 1024
+CHUNK_OVERLAP = CHUNK_SIZE // 8
+K = 10
+FETCH_K = 20
+
 llm_model_translation = {
     "LLaMA 3": "llama3-70b-8192",
     "OpenAI GPT 4o Mini": "gpt-4o-mini",
@@ -82,7 +89,7 @@ def load_llm(model: str, api_key: str, temperature: float = 1.0, max_length: int
 
 def create_db_with_langchain(path: list[str], url_content: dict):
     all_docs = []
-    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)
     embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
     if path:
         for file in path:
@@ -99,6 +106,10 @@ def create_db_with_langchain(path: list[str], url_content: dict):
             docs = text_splitter.split_documents([doc])
             all_docs.extend(docs)
 
+    # print docs
+    for idx, doc in enumerate(all_docs):
+        print(f"Doc: {idx} | Length = {len(doc.page_content)}")
+
     assert len(all_docs) > 0, "No PDFs or scrapped data provided"
     db = Chroma.from_documents(all_docs, embedding_function)
     return db
@@ -120,7 +131,7 @@ def generate_rag(
         print("Failed to load LLM. Aborting operation.")
         return None
     db = create_db_with_langchain(path, url_content)
-    retriever = db.as_retriever(search_type="mmr", search_kwargs={"k": 4, "fetch_k": 20})
+    retriever = db.as_retriever(search_type="mmr", search_kwargs={"k": K, "fetch_k": FETCH_K})
     rag_prompt = hub.pull("rlm/rag-prompt")
 
     def format_docs(docs):

app.py

@@ -435,7 +435,7 @@ def generate_and_format(
 ):
     content_string = ""
     url_content = None
-    ai_model = "Claude Sonnet 3.5"
+    ai_model = "OpenAI GPT 4o"
     if google_search_check:
         date_from = build_date(year_from, month_from, day_from)
         date_to = build_date(year_to, month_to, day_to)
@@ -637,7 +637,7 @@ def create_interface():
                     google_default = False
                     with gr.Row():
                         google_search_check = gr.Checkbox(
-                            label="Enable Google Search For Recent Sources", value=google_default
+                            label="Enable Internet Search For Recent Sources", value=google_default
                         )
                     with gr.Group(visible=google_default) as search_options:
                         with gr.Row():
@@ -682,7 +682,7 @@ def create_interface():
                     gr.Markdown("# Add Optional PDF Files with Information", elem_classes="text-center text-3xl mb-6")
                     pdf_file_input = gr.File(label="Upload PDF(s)", file_count="multiple", file_types=[".pdf"])
 
-                # HIDE AI MODEL SELECTION
+                # NOTE: HIDE AI MODEL SELECTION
                 # with gr.Group():
                 #     gr.Markdown("## AI Model Configuration", elem_classes="text-xl mb-4")
                 #     ai_generator = gr.Dropdown(
@@ -710,18 +710,15 @@ def create_interface():
                                 "Base Model",
                                 "Large Model",
                                 "XL Model",
-                                # "XL Law Model",
-                                # "XL Marketing Model",
-                                # "XL Child Style Model",
                             ],
-                            value="Large Model",
+                            value="XL Model",
                             label="Humanizer Model Version",
                         )
                     with gr.Row():
                         temperature_slider = gr.Slider(
-                            minimum=0.5, maximum=2.0, step=0.1, value=1.3, label="Temperature"
+                            minimum=0.5, maximum=2.0, step=0.1, value=1.1, label="Temperature"
                         )
-                        top_k_slider = gr.Slider(minimum=0, maximum=300, step=25, value=50, label="Top k")
+                        top_k_slider = gr.Slider(minimum=0, maximum=300, step=25, value=40, label="Top k")
                     with gr.Row():
                         repetition_penalty_slider = gr.Slider(
                             minimum=1.0, maximum=2.0, step=0.1, value=1, label="Repetition Penalty"

google_search.py

@@ -29,21 +29,38 @@ def clean_html(text):
 
 
 def build_results_beautifulsoup(url_list):
-    # Scrape URLs in list
+    print("Starting to scrape URLs...")
     start_time = time.perf_counter()
+
+    # scrape URLs in list
     soups = asyncio.run(parallel_scrap(url_list))
-    print("Scraping processing time: ", time.perf_counter() - start_time)
+
+    scraping_time = time.perf_counter() - start_time
+    print(f"Scraping processing time: {scraping_time:.2f} seconds")
+
     result_content = {}
-    num_pages = 3
+    num_pages = 10
     count = 0
+
+    print("Starting to process each URL...")
     for url, soup in zip(url_list, soups):
         if count >= num_pages:
+            print(f"Reached the limit of {num_pages} pages. Stopping processing.")
             break
+
         if soup:
+            print(f"Processing URL: {url}")
             text = clean_html(soup.text)
             if len(text) > 500:
+                print(f"Adding content from URL: {url}, content length: {len(text)}")
                 result_content[url] = text
                 count += 1
+            else:
+                print(f"Skipped URL: {url}, content too short (length: {len(text)})")
+        else:
+            print(f"Skipped URL: {url}, no soup content available.")
+
+    print("Finished processing URLs.")
     return result_content