Update main.py

main.py

@@ -1,100 +1,13 @@
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.responses import PlainTextResponse
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
-from twilio.twiml.messaging_response import MessagingResponse
-import os
-import google.generativeai as genai
-
-secret = os.environ["key"]
-
-genai.configure(api_key=secret)
-model = genai.GenerativeModel('gemini-pro')
-
-GOOD_BOY_URL = (
-    "https://images.unsplash.com/photo-1518717758536-85ae29035b6d?ixlib=rb-1.2.1"
-    "&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=1350&q=80"
-)
-
-prompt = """Certainly! Below is a chat conversation template for your AI company, Nuron, specialized in developing AI and ML solutions and chatbot systems. The conversation includes typical customer inquiries and responses that can be used during business hours (9 AM to 5 PM, Monday to Friday).
-
----
-
-**Customer**: Hi, I'm interested in learning more about your AI and ML solutions. Can you tell me more?
-
-**Nuron Support**: Hello! Thank you for reaching out to Nuron. We specialize in developing cutting-edge AI and ML solutions tailored to meet your business needs. Our services include predictive analytics, natural language processing, computer vision, and custom chatbot systems. How can we assist you today?
-
----
-
-**Customer**: Who is behind Nuron?
-
-**Nuron Support**: Nuron was founded by Arafath, our CEO, who has extensive experience in AI and ML technologies. Under his leadership, we have delivered numerous successful projects across various industries.
-
----
-
-**Customer**: What are your working hours?
-
-**Nuron Support**: Our working hours are from 9 AM to 5 PM, Monday to Friday. During these hours, our team is available to assist you with any inquiries or support you might need.
-
----
-
-**Customer**: Can you develop a custom chatbot for my business?
-
-**Nuron Support**: Absolutely! We specialize in creating custom chatbot systems designed to enhance customer engagement and streamline business processes. Please provide us with some details about your requirements, and we'll be happy to discuss how we can help.
-
----
-
-**Customer**: How can I get in touch with Arafath?
-
-**Nuron Support**: You can reach out to Arafath by emailing our support team at [email protected], and we will ensure your message is forwarded to him. Alternatively, you can schedule a meeting through our website.
-
----
-
-**Customer**: What industries do you serve?
-
-**Nuron Support**: We have experience working with a diverse range of industries including healthcare, finance, retail, and manufacturing. Our AI and ML solutions are adaptable and can be customized to fit the unique needs of any sector.
-
----
-
-**Customer**: How do I get started with a project at Nuron?
-
-**Nuron Support**: To get started, simply contact us through our website or email us at [email protected] with a brief description of your project. We'll arrange a consultation to understand your requirements and provide you with a tailored proposal.
-
----
-
-**Customer**: What is the process for developing an AI solution with Nuron?
-
-**Nuron Support**: Our process typically involves the following steps:
-1. Initial consultation to understand your needs.
-2. Proposal and project planning.
-3. Data collection and analysis.
-4. Model development and training.
-5. Implementation and integration.
-6. Testing and validation.
-7. Deployment and support.
-
-We ensure to keep you updated at each stage and incorporate your feedback to deliver the best solution.
-
----
-
-**Customer**: Can you provide support after the project is completed?
-
-**Nuron Support**: Yes, we offer comprehensive post-deployment support and maintenance services to ensure that your AI and ML solutions continue to perform optimally. Our team is always here to assist you with any updates or issues you may encounter.
-
----
-
-**Customer**: Thank you for the information!
-
-**Nuron Support**: You're welcome! If you have any more questions or need further assistance, feel free to contact us. Have a great day!
-
----
-
-use this details to give answer for my questio.only give system response only(not include customer message)
-question : """
-
-
+import requests
+from AWSClaude import AWSClaude
+import json
+import concurrent.futures
+import time
 
 app = FastAPI()
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -103,20 +16,120 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-@app.post("/whatsapp")
-async def reply_whatsapp(request: Request):
-    form_data = await request.form()
-    num_media = int(form_data.get("NumMedia", 0))
-    from_number = form_data.get("From")
-    message_body = form_data.get("Body")
-
-    gen_response = model.generate_content(str(prompt)+message_body)
-
-    response = MessagingResponse()
-
-    msg = response.message(gen_response.text)
-    #msg.media(GOOD_BOY_URL)
 
-    return PlainTextResponse(str(response), media_type="application/xml")
 
-# Run the application (Make sure you have the necessary setup to run FastAPI)
+@app.post("/get_n_depth_results")
+async def get_n_depth_results(url,input_query):
+    all_content = {}
+
+    def add_pdf_content(selected_pdf):
+        for pdf_url in selected_pdf:
+            print(pdf_url)
+            response = requests.get(pdf_url)
+
+            # Save the content of the response as a PDF file
+            pdf_path = "temp.pdf"
+            with open(pdf_path, "wb") as file:
+                file.write(response.content)
+
+            print(f"PDF file saved as {pdf_path}")
+
+            url = "http://localhost:5000/ask"
+            # url = "https://us-central1-neuralgap-1.cloudfunctions.net/scraperPDFDocxTables_v2"
+
+            data = {"processTables": "True"}
+
+            headers = {"Origin": "http://localhost:8080"}
+
+            with open(pdf_path, "rb") as file:
+                file_contents = file.read()
+
+                files = {
+                    "pdf": (
+                        pdf_path,
+                        file_contents,
+                        "application/pdf",
+                    )
+                }
+
+                response = requests.post(url, files=files, data=data, headers=headers)
+                all_content[pdf_url] = response.json()
+
+    def scrapper(input_url):
+        params = {'url': input_url}
+        headers = {'accept': 'application/json'}
+        url = 'https://chromium-qpxamiokfa-uc.a.run.app/get_scraped_data'
+        response = requests.get(url, headers=headers, params=params)
+        all_url = response.json()["URL"]
+        all_content[input_url] = response.json()["Content"]
+        return all_url
+
+    pdf_urls = []
+
+    def separate_pdf_and_nonPDF_links(urls):
+        # Separate URLs into two lists
+        pdf_links = [url for url in urls if url and url.endswith('.pdf')]
+        if pdf_links:
+            pdf_urls.append(pdf_links)
+        return [url for url in urls if not (url and url.endswith('.pdf'))]  # other links for rescraping
+
+    def call_llm_service(scraped_data, input_url, input_query, pdf):
+        query = f"""
+        Here are my scraped links:
+
+        {scraped_data}
+
+        correct hostname: {input_url} use this host name for all other tasks
+        
+        I need the always full (www.hotname.com/../) {pdf} URLs for the most relevant links related to "{input_query}". use the correct hostname from this provided content, give raw hyperlink with json format only don't give extra text details. only give json output
+        example json format is only links don't include keys (i need the always full (www.hotname.com/../))
+        """
+        llm = "ClaudeHaiku"
+        env = ""
+        user_id = "KAusXF7jp0Q40urdZWtDLXEhrmA"
+        thread_id = "hKxvoVgi7vRJCHhvMzH5"
+        stream_id = "stream1"
+        app_type = "sentinel"
+        other_request_params = {"messages": [
+            {"role": "user", "content": query},
+        ]}
+        return AWSClaude(llm, env, user_id, thread_id, stream_id, app_type, other_request_params).invoke()
+
+    input_url = f'["{url}"]'
+    input_query = input_query
+
+    for step in range(1, 3):
+        print(f"=================={step} step of scraping to get selected URLs from LLM=================================")
+        next_urls = []
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(scrapper, input_url) for input_url in (json.loads(input_url)[:2])]
+            for future in concurrent.futures.as_completed(futures):
+                next_urls.append(separate_pdf_and_nonPDF_links(future.result()))
+        selected_links_from_llm = call_llm_service(next_urls, input_url, input_query, "")
+        input_url = selected_links_from_llm
+        print(json.loads(input_url)[:2])
+
+    if not pdf_urls:
+        print(pdf_urls)
+        #return all_content.keys()
+        return all_content
+    else:
+        selected_pdf = json.loads(call_llm_service(pdf_urls, input_url, input_query, "only end with .pdf extension"))
+        print(pdf_urls)
+        print("selected pdf")
+        print(selected_pdf)
+        #return all_content.keys()
+        return all_content
+
+# # Start time
+# start_time = time.time()
+
+# print(main("https://www.keells.com/", "Please analyse reports"))
+
+# # End time
+# end_time = time.time()
+
+# # Calculate the time taken
+# time_taken = end_time - start_time
+
+# print(f"Time taken: {time_taken} seconds")