Duplicate from rohan13/makerlab-bot

Co-authored-by: Rohan Marwaha <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+env/
+.idea

README.md

@@ -0,0 +1,13 @@
+---
+title: Makerlab Bot
+emoji: 📉
+colorFrom: blue
+colorTo: blue
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: rohan13/makerlab-bot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,42 @@
+from flask import Flask, render_template
+from flask_executor import Executor
+from flask_socketio import SocketIO, emit
+from flask_cors import cross_origin, CORS
+from main import run
+from gevent import monkey
+
+monkey.patch_all(ssl=False)
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'secret!'
+socketio = SocketIO(app, cors_allowed_origins="*", async_mode='gevent', logger=True)
+cors = CORS(app)
+
+executor = Executor(app)
+
+executor.init_app(app)
+app.config['EXECUTOR_MAX_WORKERS'] = 10
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@socketio.on('message')
+def handle_message(data):
+    question = data['question']
+    print("question: " + question)
+
+    if executor.futures:
+        emit('response', {'response': 'Server is busy, please try again later'})
+        return
+
+    try:
+        future = executor.submit(run, question)
+        response = future.result()
+        emit('response', {'response': response})
+    except Exception as e:
+        traceback.print_exc()
+        emit('response', {'response': 'Server is busy. Please try again later.'})
+
+if __name__ == '__main__':
+    socketio.run(app, host="0.0.0.0", port=7860)

main.py

@@ -0,0 +1,27 @@
+from utils import create_index, get_agent_chain, get_prompt_and_tools, get_search_index
+from utils import get_custom_agent, get_prompt_and_tools_for_custom_agent
+question_starters = ['who', 'why', 'what', 'how', 'where', 'when', 'which', 'whom', 'whose']
+
+def run(question):
+
+    index = get_search_index()
+
+    # prompt, tools = get_prompt_and_tools()
+
+    # agent_chain = get_agent_chain(prompt, tools)
+
+    prompt, tools = get_prompt_and_tools_for_custom_agent()
+
+    agent_chain = get_custom_agent(prompt, tools)
+
+    result = None
+
+    try:
+        result = agent_chain.run(question)
+        print(result)
+    except ValueError as ve:
+        if "Could not parse LLM output:" in ve.args[0] and question.lower().startswith(tuple(question_starters)) and not question.lower().endswith('?'):
+            question = question + '?'
+            result = agent_chain.run(question)
+
+    return result

open_ai.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a87eb7dbbfd3245fc8025fb2467723bdcba8cdf308127050de9f8bbdeb21bc
+size 2838573

open_ai.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c399ec43dab5c11fabcc598d507b88db77a59b019c1b2533f2792199c6a1fcc
+size 3171039

requirements.txt

@@ -0,0 +1,11 @@
+faiss-cpu==1.7.3
+langchain==0.0.131
+beautifulsoup4==4.12.0
+PyPDF2==3.0.1
+openai==0.27.4
+flask==2.2.3
+flask-socketio==5.3.3
+flask-cors==3.0.10
+flask-executor==1.0.0
+gevent==22.10.2
+gevent-websocket==0.10.1

static/chatbot.js

@@ -0,0 +1,159 @@
+$(document).ready(function() {
+    // Initialize variables
+    var $chatContainer = $('.chat-container');
+    var $chatHeader = $('.chat-header');
+    var $chatBody = $('.chat-body');
+    var $chatInput = $('.chat-input');
+    var $input = $('.chat-input input');
+    var $submit = $('.chat_submit');
+    var session_id = '';
+    $chatBody.children().each(function() {
+        $(this).addClass('chat-message');
+    });
+    const buttonLabels = ["What is Makerlab?", "What is 3D printing?",
+        "Who are the founders of Makerlab?", "What are the 3D printing prices at Makerlab?",
+        "How can I host a birthday at Makerlab?", "Can I book an appointment at Makerlab?",
+        "Tell me about softwares used to create 3D printing designs", "Hi, I am bob. Tell me when Makerlab was founded.",
+        "Can I get my custom designs 3D printed at Makerlab?", "Can I host a private event at Makerlab?",
+        "Does Makerlab host any workshop?", "When is Makerlab open?", "How can I contact the Makerlab Team?"];
+
+
+    // Initialize SocketIO connection
+    var socket = io.connect('https://' + document.domain + ':' + location.port);
+    const container = document.getElementById("button-container");
+
+
+    for (let i = 0; i < buttonLabels.length; i++) {
+
+        const button = document.createElement("button");
+        button.innerHTML = buttonLabels[i];
+        button.setAttribute("class", "queries");
+        button.setAttribute("id", `button-${i}`);
+        button.style.margin = "5px";
+        container.appendChild(button);
+    }
+    scrollButtons();
+
+
+    // Function to send message to Flask-SocketIO app
+    function sendMessage(message) {
+        console.log("message: " + message )
+        socket.emit('message', {'question': message});
+    }
+
+    // Function to display message
+    function displayMessage(message, isUser, hasHtml) {
+        var $message = $('<div>').addClass('chat-message round');
+        if (hasHtml) {
+            $messageText = $('<p>').html(message);
+        } else {
+            $messageText = $('<p>').html(message.replace(/(https?:\/\/[^\s,]+)/g, '<a href="$1" target="_blank">$1</a>').replace(/(SOURCES:)/, '<br>$1'));
+        }
+        // var $messageText = $('<p>').html(message.replace(/(https?:\/\/[^\s,]+)/g, '<a href="$1">$1</a>'));
+
+        $message.append($messageText);
+        if (isUser) {
+            $message.addClass('user');
+        } else {
+            $message.addClass('bot')
+        }
+        if ($chatBody) {
+            $chatBody.append($message);
+            if ($chatBody[0]) {
+                $chatBody.animate({scrollTop: $chatBody[0].scrollHeight}, 300);
+            }
+        } else {
+            $('.chat-container').append($message);
+            $('.chat-container').animate({scrollTop: 0}, 300);
+        }
+    }
+
+
+    socket.on('response', function(data) {
+        console.log("Received response: " + data.response)
+        var response = data.response;
+        displayMessage(response, false);
+    });
+
+
+    // Send message on submit
+    $submit.click(function(event) {
+        event.preventDefault();
+        var message = $input.val().trim();
+        console.log("Submit clicked: " + message)
+        if (message !== '') {
+            displayMessage(message, true);
+            sendMessage(message);
+            $input.val('');
+        }
+    });
+
+    // Send message on enter key press
+    $input.keydown(function(event) {
+        if (event.keyCode === 13) {
+            event.preventDefault();
+            $submit.click();
+        }
+    });
+
+    // Initial message
+    displayMessage('Learn about <a href="https://makerlab.illinois.edu/" target="_blank">Makerlab</a>', false, true);
+
+    // Function to minimize the widget
+    function minimizeWidget() {
+        $chatContainer.addClass('minimized');
+        $chatHeader.hide();
+        $chatBody.hide()
+        $chatInput.hide();
+        $chatContainer.append('<div class="chat-bot-icon"><i class="fa fa-android"></i></div>');
+    }
+
+    // Function to maximize the widget
+    function maximizeWidget() {
+        $chatContainer.removeClass('minimized');
+        $chatBody.show()
+        $chatHeader.show();
+        $chatInput.show();
+        $('.chat-bot-icon').remove();
+    }
+
+    // Minimize the widget on click of close button
+    $chatHeader.find('.chat-close').click(function() {
+        minimizeWidget();
+    });
+
+    // Maximize the widget on click of chat-bot-icon
+    $chatContainer.on('click', '.chat-bot-icon', function() {
+        maximizeWidget();
+    });
+
+    // Add event listener to each button
+    $('.queries').click(function() {
+        // Set the value of the input field to the text content of the clicked button
+        $('input[type="text"]').val($(this).text());
+    });
+
+    function scrollButtons() {
+        var container = document.getElementById("button-container");
+        var buttons = container.querySelectorAll(".queries");
+        var current = 0;
+
+        var scrollInterval = setInterval(function() {
+            buttons[current].scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" });
+            current = (current + 1) % buttons.length;
+        }, 1000);
+
+        container.addEventListener("mouseenter", function() {
+            clearInterval(scrollInterval);
+        });
+
+        container.addEventListener("mouseleave", function() {
+            scrollInterval = setInterval(function() {
+                buttons[current].scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" });
+                current = (current + 1) % buttons.length;
+            }, 1000);
+        });
+    }
+
+
+});

static/style.css

@@ -0,0 +1,296 @@
+.chat-container {
+    position: fixed;
+    bottom: 30px;
+    right: 30px;
+    z-index: 999;
+    background-color: #fff;
+    border-radius: 10px;
+    box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.2);
+    max-width: 400px;
+    min-width: 400px;
+}
+
+.round {
+    border-radius: 10px;
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 30px;
+
+}
+
+.chat-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 10px;
+    background-color: darkblue;
+    color: #fff;
+    border-top-left-radius: 10px;
+    border-top-right-radius: 10px;
+}
+
+.chat-header h4 {
+    margin: 0;
+}
+
+.chat-close {
+    cursor: pointer;
+}
+
+.chat-body {
+    height: 400px;
+    overflow-y: scroll;
+    padding: 10px;
+    word-wrap: break-word;
+    display:flex;
+    flex-direction: column;
+}
+
+.chat-message {
+    margin: 10px;
+}
+
+.chat-message p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+    color: #fff;
+}
+
+.chat-message.user {
+    display: flex;
+    align-self: flex-end;
+    justify-content: flex-end;
+    text-align: right;
+    align-items: center;
+    background-color: rgba(0, 0, 139, 0.75);
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+    border-bottom-left-radius: 10px;
+    word-wrap: break-word;
+}
+
+
+.chat-message.bot {
+    display: flex;
+    align-self: flex-start;
+    justify-content: flex-start;
+    text-align: left;
+    align-items: center;
+    background-color: rgba(0, 0, 139, 0.75);
+    border-top-left-radius: 0px;
+    border-bottom-right-radius: 10px;
+    border-bottom-left-radius: 0px;
+    word-wrap: break-word;
+}
+
+.chat-message.bot p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+    overflow-wrap: anywhere;
+}
+
+.chat-message.user:after {
+    content: "";
+    position: relative;
+    top: 0;
+    right: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-left: 16px solid #00008BBF;
+    border-top-right-radius: 10px;
+}
+
+.chat-message.bot:before {
+    content: "";
+    position: relative;
+    top: 0;
+    left: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-right: 15px solid #00008BBF;
+    border-top-left-radius: 10px;
+}
+
+
+.chat-input {
+    display: flex;
+    margin-top: 10px;
+}
+
+.chat-input input {
+    flex-grow: 1;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    margin-right: 10px;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}
+
+.chat-input button {
+    background-color: #00008BBF;
+    color: #fff;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    cursor: pointer;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}
+
+/* CSS for chat-container when minimized */
+.chat-container.minimized {
+    min-width: 70px;
+    height: 70px;
+    border-radius: 50%;
+    position: fixed;
+    bottom: 10px;
+    right: 10px;
+    z-index: 9999;
+    background-color: #fff;
+    box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.3);
+    cursor: pointer;
+}
+
+/* CSS for chat-bot-icon */
+.chat-bot-icon {
+    font-size: 30px;
+    color: #00008BBF;
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+}
+
+/* CSS for chat-header when not minimized */
+.chat-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    background-color: #6c7ae0;
+    color: #fff;
+    padding: 10px;
+    border-top-left-radius: 5px;
+    border-top-right-radius: 5px;
+}
+
+/* CSS for chat-container when not minimized */
+.chat-container:not(.minimized) {
+    border-radius: 5px;
+    position: fixed;
+    bottom: 10px;
+    right: 10px;
+    z-index: 9999;
+    background-color: #fff;
+    box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+/* CSS for chat-bot-icon when chat-container is minimized */
+.chat-container.minimized .chat-bot-icon {
+    display: block;
+}
+
+/* CSS for chat-bot-icon when chat-container is not minimized */
+.chat-container:not(.minimized) .chat-bot-icon {
+    display: none;
+}
+
+.queries {
+
+    padding: 8px 12px;
+    font-size: 16px;
+    font-weight: bold;
+    text-align: center;
+    text-decoration: none;
+    border: 0.5px solid #a5a0a0;
+    border-radius: 20px;
+    color: #000;
+    background-color: #343a404a;
+    cursor: pointer;
+    margin: 5px;
+}
+
+.queries:hover {
+    background-color: #343a40ad;
+}
+
+.queries:active {
+    background-color: #0053a4;
+}
+
+#button-container {
+    display: flex;
+    position: relative;
+    left: 2%;
+    top: 40%;
+    flex-direction: column;
+    justify-content: inherit;
+    align-items: center;
+    width: auto;
+    overflow-y: scroll;
+    max-height: 350px;
+    padding-top: 110%;
+    margin-top: 2%;
+
+}
+
+#button-container button {
+    margin-bottom: 10px;
+}
+
+.query-heading {
+    display: flex;
+    position: relative;
+    width: auto%;
+    background-color: #fff;
+    padding: 10px;
+    z-index: 1;
+    justify-content: inherit;
+    width: 100%;
+    border-bottom: 1px solid #2f4f4f5e;
+}
+
+.sample-query {
+    display: flex;
+    position: absolute;
+    left: 30%;
+    top: 10%;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: center;
+    width: auto;
+    padding: 10px;
+    border: 1px solid #2f4f4f5e;
+    justify-content: center;
+    border-radius: 10px;
+    max-width: 30%;
+}
+
+::-webkit-scrollbar {
+    width: 8px;
+}
+
+::-webkit-scrollbar-track {
+    background-color: #f4f4f4;
+}
+
+::-webkit-scrollbar-thumb {
+    background-color: #a3bfe9a6;
+    border-radius: 20px;
+}
+
+
+

templates/index.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>MakerlabX3DPrinting QA</title>
+    <meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+    <link rel="stylesheet" href="static/style.css">
+</head>
+<body>
+<div class = "sample-query">
+<div class="query-heading"><h4>Sample Queries</h4></div>
+<div id="button-container">
+</div></div>
+<div class="chat-container">
+    <div class="chat-header">
+        <h4>Makerlab Q&A Bot</h4>
+        <i class="fa fa-close chat-close"></i>
+    </div>
+    <div class="chat-bot-icon">
+        <i class="fa fa-android"></i> <!-- Replace with your bot icon -->
+    </div>
+    <div class="chat-body chat-messages round"></div>
+    <div class="chat-input">
+        <input type="text" placeholder="Type your message">
+        <button class="chat_submit">Send</button>
+    </div>
+</div>
+<!--<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.js" integrity="sha512-sY2t8W1xNQ2yB+1RFXJv+wwhdN7CHX9Z+fhM7JH/3B3q1x7VJBOwKe+zb7VW0EC8XG5M5rjBQd7+47F5fQlhKQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>-->
+<script src="https://cdn.socket.io/4.4.1/socket.io.min.js"></script>
+<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+<script src="{{ url_for('static', filename='chatbot.js') }}"></script>
+</body>
+</html>

utils.py

@@ -0,0 +1,335 @@
+import os
+import pickle
+import re
+import time
+from typing import List, Union
+from urllib.parse import urlparse, urljoin
+
+import faiss
+import requests
+from PyPDF2 import PdfReader
+from bs4 import BeautifulSoup
+from langchain import OpenAI, LLMChain
+from langchain.agents import ConversationalAgent
+from langchain.agents import Tool, AgentExecutor, LLMSingleActionAgent, AgentOutputParser
+from langchain.prompts import BaseChatPromptTemplate
+from langchain.chains import ConversationalRetrievalChain
+from langchain.docstore.document import Document
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.schema import AgentAction, AgentFinish, HumanMessage
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores.faiss import FAISS
+
+book_url = 'https://g.co/kgs/2VFC7u'
+book_file = "Book.pdf"
+url = 'https://makerlab.illinois.edu/'
+
+pickle_file = "open_ai.pkl"
+index_file = "open_ai.index"
+
+gpt_3_5 = OpenAI(model_name='gpt-3.5-turbo',temperature=0)
+
+embeddings = OpenAIEmbeddings()
+
+chat_history = []
+
+memory = ConversationBufferWindowMemory(memory_key="chat_history")
+
+gpt_3_5_index = None
+
+class CustomOutputParser(AgentOutputParser):
+
+    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
+        # Check if agent replied without using tools
+        if "AI:" in llm_output:
+            return AgentFinish(return_values={"output": llm_output.split("AI:")[-1].strip()},
+                               log=llm_output)
+        # Check if agent should finish
+        if "Final Answer:" in llm_output:
+            return AgentFinish(
+                # Return values is generally always a dictionary with a single `output` key
+                # It is not recommended to try anything else at the moment :)
+                return_values={"output": llm_output.split("Final Answer:")[-1].strip()},
+                log=llm_output,
+            )
+        # Parse out the action and action input
+        regex = r"Action: (.*?)[\n]*Action Input:[\s]*(.*)"
+        match = re.search(regex, llm_output, re.DOTALL)
+        if not match:
+            raise ValueError(f"Could not parse LLM output: `{llm_output}`")
+        action = match.group(1).strip()
+        action_input = match.group(2)
+        # Return the action and action input
+        return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output)
+
+# Set up a prompt template
+class CustomPromptTemplate(BaseChatPromptTemplate):
+    # The template to use
+    template: str
+    # The list of tools available
+    tools: List[Tool]
+
+    def format_messages(self, **kwargs) -> str:
+        # Get the intermediate steps (AgentAction, Observation tuples)
+        # Format them in a particular way
+        intermediate_steps = kwargs.pop("intermediate_steps")
+        thoughts = ""
+        for action, observation in intermediate_steps:
+            thoughts += action.log
+            thoughts += f"\nObservation: {observation}\nThought: "
+        # Set the agent_scratchpad variable to that value
+        kwargs["agent_scratchpad"] = thoughts
+        # Create a tools variable from the list of tools provided
+        kwargs["tools"] = "\n".join([f"{tool.name}: {tool.description}" for tool in self.tools])
+        # Create a list of tool names for the tools provided
+        kwargs["tool_names"] = ", ".join([tool.name for tool in self.tools])
+        formatted = self.template.format(**kwargs)
+        return [HumanMessage(content=formatted)]
+
+def get_search_index():
+    global gpt_3_5_index
+    if os.path.isfile(pickle_file) and os.path.isfile(index_file) and os.path.getsize(pickle_file) > 0:
+        # Load index from pickle file
+        with open(pickle_file, "rb") as f:
+            search_index = pickle.load(f)
+    else:
+        search_index = create_index()
+
+    gpt_3_5_index = search_index
+
+
+def create_index():
+    source_chunks = create_chunk_documents()
+    search_index = search_index_from_docs(source_chunks)
+    faiss.write_index(search_index.index, index_file)
+    # Save index to pickle file
+    with open(pickle_file, "wb") as f:
+        pickle.dump(search_index, f)
+    return search_index
+
+
+def create_chunk_documents():
+    sources = fetch_data_for_embeddings(url, book_file, book_url)
+    # print("sources" + str(len(sources)))
+
+    splitter = CharacterTextSplitter(separator=" ", chunk_size=800, chunk_overlap=0)
+
+    source_chunks = splitter.split_documents(sources)
+
+    for chunk in source_chunks:
+        print("Size of chunk: " + str(len(chunk.page_content) + len(chunk.metadata)))
+        if chunk.page_content is None or chunk.page_content == '':
+            print("removing chunk: "+ chunk.page_content)
+            source_chunks.remove(chunk)
+        elif len(chunk.page_content) >=1000:
+            print("splitting document")
+            source_chunks.extend(splitter.split_documents([chunk]))
+    # print("Chunks: " + str(len(source_chunks)) + "and type " + str(type(source_chunks)))
+    return source_chunks
+
+
+def fetch_data_for_embeddings(url, book_file, book_url):
+    sources = get_website_data(url)
+    sources.extend(get_document_data(book_file, book_url))
+    return sources
+
+def get_website_data(index_url):
+    # Get all page paths from index
+    paths = get_paths(index_url)
+
+    # Filter out invalid links and join them with the base URL
+    links = get_links(index_url, paths)
+
+    return get_content_from_links(links, index_url)
+
+
+def get_content_from_links(links, index_url):
+    content_list = []
+    for link in set(links):
+        if link.startswith(index_url):
+            page_data = requests.get(link).content
+            soup = BeautifulSoup(page_data, "html.parser")
+
+            # Get page content
+            content = soup.get_text(separator="\n")
+            # print(link)
+
+            # Get page metadata
+            metadata = {"source": link}
+
+            content_list.append(Document(page_content=content, metadata=metadata))
+    time.sleep(1)
+    # print("content list" + str(len(content_list)))
+    return content_list
+
+
+def get_paths(index_url):
+    index_data = requests.get(index_url).content
+    soup = BeautifulSoup(index_data, "html.parser")
+    paths = set([a.get('href') for a in soup.find_all('a', href=True)])
+    return paths
+
+
+def get_links(index_url, paths):
+    links = []
+    for path in paths:
+        url = urljoin(index_url, path)
+        parsed_url = urlparse(url)
+        if parsed_url.scheme in ["http", "https"] and "squarespace" not in parsed_url.netloc:
+            links.append(url)
+    return links
+
+
+def get_document_data(book_file, book_url):
+    document_list = []
+    with open(book_file, 'rb') as f:
+        pdf_reader = PdfReader(f)
+        for i in range(len(pdf_reader.pages)):
+            page_text = pdf_reader.pages[i].extract_text()
+            metadata = {"source": book_url}
+            document_list.append(Document(page_content=page_text, metadata=metadata))
+
+    # print("document list" + str(len(document_list)))
+    return document_list
+
+def search_index_from_docs(source_chunks):
+    # Create index from chunk documents
+    # print("Size of chunk" + str(len(source_chunks)))
+    search_index = FAISS.from_texts([doc.page_content for doc in source_chunks], embeddings, metadatas=[doc.metadata for doc in source_chunks])
+    return search_index
+
+
+def get_qa_chain(gpt_3_5_index):
+    global gpt_3_5
+    print("index: " + str(gpt_3_5_index))
+    return ConversationalRetrievalChain.from_llm(gpt_3_5, chain_type="stuff", get_chat_history=get_chat_history,
+                                                 retriever=gpt_3_5_index.as_retriever(), return_source_documents=True, verbose=True)
+
+def get_chat_history(inputs) -> str:
+    res = []
+    for human, ai in inputs:
+        res.append(f"Human:{human}\nAI:{ai}")
+    return "\n".join(res)
+
+
+def generate_answer(question) -> str:
+    global chat_history, gpt_3_5_index
+    gpt_3_5_chain = get_qa_chain(gpt_3_5_index)
+    result = gpt_3_5_chain(
+        {"question": question, "chat_history": chat_history,"vectordbkwargs": {"search_distance": 0.8}})
+    print("REsult: " + str(result))
+    chat_history = [(question, result["answer"])]
+    sources = []
+
+    for document in result['source_documents']:
+        source = document.metadata['source']
+        sources.append(source)
+
+    source = ',\n'.join(set(sources))
+    return result['answer'] + '\nSOURCES: ' + source
+
+
+def get_agent_chain(prompt, tools):
+    global gpt_3_5
+    # output_parser = CustomOutputParser()
+    llm_chain = LLMChain(llm=gpt_3_5, prompt=prompt)
+    agent = ConversationalAgent(llm_chain=llm_chain, tools=tools, verbose=True)
+    agent_chain = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=memory,
+                                                     intermediate_steps=True)
+    return agent_chain
+
+
+def get_prompt_and_tools():
+    tools = get_tools()
+
+    prefix = """Have a conversation with a human, answering the following questions as best you can. 
+    Always try to use Vectorstore first. 
+    Your name is Makerlab Bot because you are a personal assistant of Makerlab. You have access to the following tools:"""
+    suffix = """Begin! If you use any tool, ALWAYS return a "SOURCES" part in your answer"
+    
+    {chat_history}
+    Question: {input}
+    {agent_scratchpad}
+    SOURCES:"""
+    prompt = ConversationalAgent.create_prompt(
+        tools,
+        prefix=prefix,
+        suffix=suffix,
+        input_variables=["input", "chat_history", "agent_scratchpad"]
+    )
+    # print("Template: " + prompt.template)
+    return prompt, tools
+
+
+def get_tools():
+    tools = [
+        Tool(
+            name="Vectorstore",
+            func=generate_answer,
+            description="useful for when you need to answer questions about the Makerlab or 3D Printing.",
+            return_direct=True
+        )]
+    return tools
+
+def get_custom_agent(prompt, tools):
+
+    llm_chain = LLMChain(llm=gpt_3_5, prompt=prompt)
+
+    output_parser = CustomOutputParser()
+    tool_names = [tool.name for tool in tools]
+    agent = LLMSingleActionAgent(
+        llm_chain=llm_chain,
+        output_parser=output_parser,
+        stop=["\nObservation:"],
+        allowed_tools=tool_names
+    )
+    agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=memory,
+                                                        intermediate_steps=True)
+    return agent_executor
+
+def get_prompt_and_tools_for_custom_agent():
+    template = """
+    Have a conversation with a human, answering the following questions as best you can. 
+    Always try to use Vectorstore first. 
+    Your name is Makerlab Bot because you are a personal assistant of Makerlab. You have access to the following tools:
+    
+    {tools}
+
+    To answer for the new input, use the following format:
+    
+    New Input: the input question you must answer
+    Thought: Do I need to use a tool? Yes
+    Action: the action to take, should be one of [{tool_names}]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question. SOURCES: the sources referred to find the final answer
+
+    
+    When you have a response to say to the Human and DO NOT need to use a tool:
+    1. DO NOT return "SOURCES" if you did not use any tool.
+    2. You MUST use this format:
+    ```
+    Thought: Do I need to use a tool? No
+    AI: [your response here]
+    ```
+
+    Begin! Remember to speak as a personal assistant when giving your final answer.
+    ALWAYS return a "SOURCES" part in your answer, if you used any tool. 
+    
+    Previous conversation history:
+    {chat_history}
+    New input: {input}
+    {agent_scratchpad}
+    SOURCES:"""
+    tools = get_tools()
+    prompt = CustomPromptTemplate(
+        template=template,
+        tools=tools,
+        # This omits the `agent_scratchpad`, `tools`, and `tool_names` variables because those are generated dynamically
+        # This includes the `intermediate_steps` variable because that is needed
+        input_variables=["input", "intermediate_steps", "chat_history"]
+    )
+    return prompt, tools