Upload 27 files

.env

@@ -0,0 +1,9 @@
+FASTAPI_CONFIG=production
+SECRET='zu=*nck@&r26rsa$2qv8a7g-p$slw79ym81mylj6s**w(da6&#'
+OPENAI_API_KEY=sk-proj-GYY7hx5SM02oDz5lhrK0T3BlbkFJegeuu9bXdNONd3Ds1rqY
+
+#DATABASE_USER=
+#DATABASE_PASSWORD=
+#DATABASE_HOST=
+#DATABASE_PORT=
+#DATABASE_NAME=

.env.example

@@ -0,0 +1,3 @@
+FASTAPI_CONFIG=production
+SECRET='zu=*nck@&r26rsa$2qv8a7g-p$slw79ym81mylj6s**w(da6&#'
+OPENAI_API_KEY=

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.10
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+RUN chmod -R 777 /code
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,3 @@
+from project import create_app
+
+app = create_app()

project/__init__.py

@@ -0,0 +1,27 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from project.config import settings
+
+
+def create_app() -> FastAPI:
+    app = FastAPI()
+
+    from project.bot import bot_router
+    app.include_router(bot_router, tags=['bot'])
+
+    from project.ws import ws_router
+    app.include_router(ws_router, tags=['ws'])
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.ORIGINS,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.mount('/static', StaticFiles(directory="static"), name="static")
+
+    return app

project/asgi.py

@@ -0,0 +1,4 @@
+from project import create_app
+
+app = create_app()
+

project/bot/__init__.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+
+bot_router = APIRouter(
+    prefix=''
+)
+
+from project.bot import views

project/bot/chatbot.py

@@ -0,0 +1,156 @@
+import asyncio
+import base64
+import os
+import tempfile
+
+import numpy as np
+
+from project.config import settings
+import pandas as pd
+
+
+class ChatBot:
+    chat_history = []
+
+    def __init__(self, memory=None):
+        self.chat_history.append({
+            "role": 'assistant',
+            'content': "Hi! What would you like to order from the food?"
+        })
+
+    @staticmethod
+    def _transform_bytes_to_file(data_bytes) -> str:
+        audio_bytes = base64.b64decode(data_bytes)
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp3')
+        try:
+            temp_file.write(audio_bytes)
+            filepath = temp_file.name
+        finally:
+            temp_file.close()
+        return filepath
+
+    @staticmethod
+    async def _transcript_audio(temp_filepath: str) -> str:
+        with open(temp_filepath, 'rb') as file:
+            transcript = await settings.OPENAI_CLIENT.audio.transcriptions.create(
+                model='whisper-1',
+                file=file,
+                # language='nl'
+            )
+        text = transcript.text
+        return text
+
+    @staticmethod
+    async def _convert_to_embeddings(query: str):
+        response = await settings.OPENAI_CLIENT.embeddings.create(
+            input=query,
+            model='text-embedding-3-large'
+        )
+        embeddings = response.data[0].embedding
+        return embeddings
+
+    @staticmethod
+    async def _convert_response_to_voice(ai_response: str) -> str:
+        audio = await settings.OPENAI_CLIENT.audio.speech.create(
+            model="tts-1",
+            voice="alloy",
+            input=ai_response
+        )
+        encoded_audio = base64.b64encode(audio.content).decode('utf-8')
+        return encoded_audio
+
+    @staticmethod
+    async def _get_context_data(query: list[float]) -> str:
+        query = np.array([query]).astype('float32')
+        _, distances, indices = settings.FAISS_INDEX.range_search(query.astype('float32'), settings.SEARCH_RADIUS)
+        indices_distances_df = pd.DataFrame({'index': indices, 'distance': distances})
+        filtered_data_df = settings.products_dataset.iloc[indices]
+        filtered_data_df['distance'] = indices_distances_df['distance'].values
+        sorted_data_df: pd.DataFrame = filtered_data_df.sort_values(by='distance').reset_index(drop=True)
+        sorted_data_df = sorted_data_df.drop('distance', axis=1)
+        data = sorted_data_df.head(1).to_dict(orient='records')
+        context_str = ''
+        for row in data:
+            context_str += f'{row["Search"]}\n\n'
+        return context_str
+
+    async def _rag(self, query: str, query_type: str, context: str = None):
+        if context:
+            self.chat_history.append({'role': 'assistant', 'content': context})
+            prompt = settings.PRODUCT_PROMPT
+        else:
+            if 'search' in query_type.lower():
+                prompt = settings.EMPTY_PRODUCT_PROMPT
+            elif 'purchase' in query_type.lower():
+                prompt = settings.ADD_TO_CART_PROMPT
+            elif 'product_list' in query_type.lower():
+                prompt = settings.PRODUCT_LIST_PROMPT
+            else:
+                prompt = settings.EMPTY_PRODUCT_PROMPT
+        self.chat_history.append({
+            'role': 'user',
+            'content': query
+        })
+        messages = [
+            {
+                'role': 'system',
+                'content': f"{prompt}"
+            },
+        ]
+        messages += self.chat_history
+        completion = await settings.OPENAI_CLIENT.chat.completions.create(
+            messages=messages,
+            temperature=0,
+            n=1,
+            model="gpt-3.5-turbo",
+        )
+        response = completion.choices[0].message.content
+        self.chat_history.append({'role': 'assistant', 'content': response})
+        return response
+
+    async def _get_query_type(self, query: str) -> str:
+        assistant_message = self.chat_history[-1]['content']
+        messages = [
+            {
+                "role": 'system',
+                'content': settings.ANALYZER_PROMPT
+            },
+            {
+                "role": 'user',
+                "content": f"Assistant message: {assistant_message}\n"
+                           f"User response: {query}"
+            }
+        ]
+        completion = await settings.OPENAI_CLIENT.chat.completions.create(
+            messages=messages,
+            temperature=0,
+            n=1,
+            model="gpt-3.5-turbo",
+        )
+        response = completion.choices[0].message.content
+        return response
+
+    async def ask(self, data: dict):
+        audio = data['audio']
+        temp_filepath = self._transform_bytes_to_file(audio)
+        transcript = await self._transcript_audio(temp_filepath)
+        query_type = await self._get_query_type(transcript)
+
+        context = None
+        if query_type == 'search':
+            transformed_query = await self._convert_to_embeddings(transcript)
+            context = await self._get_context_data(transformed_query)
+        ai_response = await self._rag(transcript, query_type, context)
+        voice_ai_response = await self._convert_response_to_voice(ai_response)
+
+        data = {
+            'user_query': transcript,
+            'ai_response': ai_response,
+            'voice_response': voice_ai_response
+        }
+
+        try:
+            os.remove(temp_filepath)
+        except FileNotFoundError:
+            pass
+        return data

project/bot/search_tools/cleaned_products.csv

@@ -0,0 +1,34 @@
+Search
+Naam van de producten: Maak je eigen Poke . Productomschrijving: nan. Prijs: € 14.95
+"Naam van de producten: So Salmon Bowl. Productomschrijving: Gemarineerde zalm, komkommer, avocado, wortel, edamame, sesam zaadjes, gebakken ui en Japanse mayo. Prijs: € 14.95"
+"Naam van de producten: Rock 'n chicken. Productomschrijving: Krokante kip, mais, komkommer, edamame bonen, rode ui, gebakken ui, chilisaus dressing, sesamzaadjes.. Prijs: € 14.95"
+"Naam van de producten: So Salmon Bowl. Productomschrijving: Gemarineerde zalm, komkommer, avocado, wortel, edamame, sesam zaadjes, gebakken ui en Japanse mayo. Prijs: € 14.95"
+"Naam van de producten: Truly Tuna. Productomschrijving: Raw Tuna, avacado, wortel, komkommer, mais, gebakken ui, nori flakes, wasabi malunaise. Prijs: € 14.95"
+"Naam van de producten: Rock 'n chicken. Productomschrijving: Krokante kip, mais, komkommer, edamame bonen, rode ui, gebakken ui, chilisaus dressing, sesamzaadjes.. Prijs: € 14.95"
+"Naam van de producten: I'm Veggie Baby. Productomschrijving: Knapperige tofu, Komkommer, ananas, lente ui, mango, sriracha saus, kokos vlokken en sesamzaadjes.. Prijs: € 14.5"
+"Naam van de producten: Teriyaki Chicks. Productomschrijving: Krokante kip, komkommer, mais, edamame, wortel, sesamzaadjes, gebakken ui, teriyaki dressing. Prijs: € 14.95"
+"Naam van de producten: Spicy Tuna. Productomschrijving: Spicy gemarineerde tonijn, Edamame, mango, wakame, mais, masago en gebakken ui.. Prijs: € 14.95"
+"Naam van de producten: Tempura Trouble. Productomschrijving: Ebi tempura garnalen, Avocado, edamame, wortel, komkommer, furikake en gebakken uitjes. Sriracha malunaise.. Prijs: € 14.95"
+Naam van de producten: Maak je eigen Poke . Productomschrijving: nan. Prijs: € 14.95
+Naam van de producten: Edamame. Productomschrijving: Japanse sojaboontjes. Deze zijn op smaak gebracht met een beetje zeezout. Prijs: € 5.95
+"Naam van de producten: Chuka Wakame. Productomschrijving: Zeewier salade op smaak met sesamolie, sojasaus en sesamzaadjes. Prijs: € 5.95"
+Naam van de producten: Ebi Fry - 4 Stuks. Productomschrijving: Gefrituurde garnaal.. Prijs: € 5.95
+Naam van de producten: Karaage - 5 Stuks. Productomschrijving: Gefrituurde kipstukjes in Japanese style. Prijs: € 2.98
+"Naam van de producten: Malumpia. Productomschrijving: Malu´s treasure, 8 mini loempia's.. Prijs: € 5.95"
+Naam van de producten: Gyoza. Productomschrijving: Japanse Dumpling. 5 stuks.. Prijs: € 5.95
+Naam van de producten: Never Enough. Productomschrijving: Malu's zeldzame kipstukken geserveerd met een overheerlijke chilisaus. 5 stuks.. Prijs: € 5.95
+"Naam van de producten: Pink Lady. Productomschrijving: Aardbei, banaan, soja-vanillemelk. Prijs: € 4.5"
+"Naam van de producten: Sweet Baby. Productomschrijving: Framboos, soja-vanillemelk, banaan. Prijs: € 4.5"
+"Naam van de producten: Power Shake. Productomschrijving: Framboos, aarbei, soja-vanillemelk. Prijs: € 4.5"
+"Naam van de producten: Aloha Shake. Productomschrijving: Avocado, banaan, soja-vanillemelk. Prijs: € 4.5"
+"Naam van de producten: Choco Loco. Productomschrijving: Banaan, avocado, soja-chocolademelk. Prijs: € 4.5"
+"Naam van de producten: Coca Cola original taste 330ml. Productomschrijving: Coca-Cola Original Taste, een heerlijke en verfrissende bruisende frisdrank, voor het eerst ontwikkeld in 1886, met een geweldige smaak. Made for sharing. Prijs: € 2.5"
+Naam van de producten: Coca-Cola Light 330ml. Productomschrijving: Coca-Cola Light is een heerlijke calorie- en suikervrije bruisende frisdrank met een geweldige smaak en slechts 1 kcal per 330ml. Iconische en verfrissende cola. Prijs: € 2.5
+"Naam van de producten: Fanta Orange 330ml. Productomschrijving: Fanta Orange: de iconische, heerlijke bruisende frisdrank gemaakt met vruchtensap. Perfect voor snacken en ontwikkeld om kleur aan je leven toe te voegen. Prijs: € 2.5"
+Naam van de producten: Hawaii Tropical Drink. Productomschrijving: nan. Prijs: € 2.5
+"Naam van de producten: Sprite zero sugar 330ml. Productomschrijving: Laat je ware ik sprankelen met Sprite: een zuivere, fris smakende, bruisende limoen- en citroenfrisdrank. 100% natuurlijke aroma's, zonder suiker, zonder calorieën.. Prijs: € 2.5"
+"Naam van de producten: Fuze Tea black tea sparkling lemon 330ml. Productomschrijving: FUZETEA Black Tea Sparkling Lemon is een unieke, caloriearme* bruisende theedrank gemaakt met een fusie van zwarte thee en zonnig citroen.. Prijs: € 2.5"
+"Naam van de producten: Fuze Tea green tea 330ml. Productomschrijving: FUZETEA Green Tea Citrus is een unieke, verfrissende caloriearme* theedrank gemaakt met een fusie van groene thee en citrus.. Prijs: € 2.5"
+Naam van de producten: Red Dragon. Productomschrijving: Framboos - Banaan en Appelsap. Prijs: € 4.5
+Naam van de producten: Tropical Island. Productomschrijving: Passievrucht - Ananas - Appelsap. Prijs: € 4.5
+Naam van de producten: Spicy Dutch. Productomschrijving: Wortel - Appelsap - Gember. Prijs: € 4.5

project/bot/templates/voice.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Real Estate</title>
+    <!-- Fonts -->
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=Montserrat:wght@300;400;500&display=swap" rel="stylesheet">
+    <!-- Bootstrap css -->
+    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet"
+          integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+    <link rel="stylesheet" href="../../../static/css/style.css">
+</head>
+
+<div id="message"></div>
+
+<div class="overlay" id="loadingModal">
+    <div class="report-loader">
+        <div class="report-inner report-one"></div>
+        <div class="report-inner report-two"></div>
+        <div class="report-inner report-three"></div>
+    </div>
+</div>
+<div class="container-fluid px-0" style="height: 100vh">
+    <div class="row mx-0 d-flex align-items-center" style="height: 100vh">
+        <div class="col-4 ms-2">
+            <div class="rounded-5 border shadow" style="height: 95vh ;background-color: #f1f1f1">
+                <div id="chatHistory"  class="my-4" style="height: 90vh; overflow-y: auto" >
+
+                </div>
+            </div>
+        </div>
+        <div class="col-6 align-items-center justify-content-around d-flex">
+            <button class="btn btn-lg btn-success" onclick="startCall()">Start call</button>
+            <button class="btn btn-lg btn-primary" onclick="stopAndSend()">Send a record</button>
+            <button class="btn btn-lg btn-danger" onclick="endCall()">End call</button>
+        </div>
+    </div>
+</div>
+</html>
+<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"
+        integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4"
+        crossorigin="anonymous"></script>
+<script src="https://code.jquery.com/jquery-3.6.3.min.js"
+        integrity="sha256-pvPw+upLPUjgMXY0G+8O0xUf+/Im1MZjXxxgOcBQBXU=" crossorigin="anonymous"></script>
+<link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css">
+<script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
+<script src="https://kit.fontawesome.com/d4ffd37f75.js" crossorigin="anonymous"></script>
+<script type="text/javascript" src="../../../static/js/voice.js"></script>

project/bot/utils.py

@@ -0,0 +1,108 @@
+import base64
+import datetime
+import os
+import tempfile
+from io import BytesIO
+
+from PIL import Image
+import aiofiles
+from fastapi import UploadFile
+from fastapi.params import File
+
+from project.bot.models import Report
+from project.config import settings
+
+
+async def generate_ai_report(history: list[str], language: str) -> Report:
+    history = list(map(lambda x: {'role': 'user', 'content': x}, history))
+    messages = [
+        {
+            'role': 'system',
+            'content': f"""Summarize the key points from the user's messages, organizing the summary into a structured 
+        format. Conclude with a brief report that encapsulates the essence of the 
+        discussion. Make your answer in {language}"""
+        },
+        *history
+    ]
+    chat_completion = await settings.OPENAI_CLIENT.chat.completions.create(
+        model="gpt-4-0125-preview",
+        temperature=0.4,
+        n=1,
+        messages=messages
+    )
+    response = chat_completion.choices[0].message.content
+    report = Report()
+    report.content = response
+    return report
+
+
+async def encode_file_to_base64(filepath):
+    async with aiofiles.open(filepath, 'rb') as file:
+        content = await file.read()
+        return base64.b64encode(content).decode('utf-8')
+
+
+async def transcript_audio_from_base64(data: str):
+    current_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    file_name = f"voice_record_{current_time}.mp3"
+    file_path = str(settings.BASE_DIR / 'project' / 'records' / file_name)
+    data_bytes = base64.b64decode(data)
+    async with aiofiles.open(file_path, 'wb') as f:
+        await f.write(data_bytes)
+    with open(file_path, 'rb') as f:
+        transcript = await settings.OPENAI_CLIENT.audio.transcriptions.create(
+            model='whisper-1',
+            file=f,
+        )
+    text = transcript.text
+    return text, file_path
+
+
+async def generate_image_description(image: str, file_format: str) -> str:
+    messages = [
+        {
+            'role': 'system',
+            'content': settings.REPORT_PROMPT
+        },
+        {
+            'role': 'user',
+            'content': [
+                {
+                    'type': 'text',
+                    'text': settings.IMAGE_PROMPT
+                },
+                {
+                    'type': 'image_url',
+                    'image_url': {
+                        'url': f"data:image/{file_format};base64,{image}",
+                        'detail': 'low'
+                    }
+                }
+            ]
+        }
+    ]
+    chat_completion = await settings.OPENAI_CLIENT.chat.completions.create(
+        model="gpt-4-vision-preview",
+        temperature=0.5,
+        n=1,
+        messages=messages
+    )
+    response = chat_completion.choices[0].message.content
+    return response
+
+
+def compress_and_save_image(image_content: bytes, width=768) -> str:
+    img = Image.open(BytesIO(image_content))
+    orig_width, orig_height = img.size
+    new_height = int((orig_height * width) / orig_width)
+    resized_img = img.resize((width, new_height), Image.LANCZOS)
+    current_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    file_format = img.format.lower()
+    file_path = str(settings.BASE_DIR / 'project' / 'images' / f'image_{current_time}.{file_format}')
+    if file_format in ['jpeg', 'jpg']:
+        resized_img.save(file_path, 'JPEG', optimize=True, quality=70)
+    elif file_format == 'png':
+        resized_img.save(file_path, 'PNG', optimize=True, compress_level=7)
+    else:
+        raise ValueError(f"{file_format.upper()} format is not supported.")
+    return file_path

project/bot/views.py

@@ -0,0 +1,11 @@
+from fastapi.templating import Jinja2Templates
+from fastapi.requests import Request
+
+from project.bot import bot_router
+
+template = Jinja2Templates(directory='project/bot/templates')
+
+
+@bot_router.get('/', name='voice')
+async def voice(request: Request):
+    return template.TemplateResponse("voice.html", {'request': request})

project/config.py

@@ -0,0 +1,66 @@
+import pathlib
+from functools import lru_cache
+
+import pandas as pd
+from openai import AsyncOpenAI
+from environs import Env
+import faiss
+
+env = Env()
+env.read_env()
+
+
+class BaseConfig:
+    BASE_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent
+    OPENAI_CLIENT = AsyncOpenAI(api_key=env('OPENAI_API_KEY'))
+    FAISS_INDEX = faiss.read_index(str(BASE_DIR / 'project' / 'bot' / 'search_tools' / 'products_faiss.index'))
+    SEARCH_RADIUS = 1
+    PRODUCT_PROMPT = "Je bent de virtuele assistent van de Malu Haarlemse winkel. Uw taak is om het product " \
+                           "zeer kort en bondig aan de gebruiker te presenteren. De gebruiker zal vragen om voedsel " \
+                           "te bestellen in de winkel, en u, met behulp van uw vorige antwoord als kennis van " \
+                           "productinformatie, moet kort vertellen over het product, de prijs en aanbieden om het toe "
+    EMPTY_PRODUCT_PROMPT = "Je bent de virtuele assistent van de Malu Haarlemse winkel. Uw taak is om de gebruiker heel kort en bondig te vragen om informatie over het voedsel in meer detail te verstrekken."
+    ANALYZER_PROMPT = """
+Je bent een model voor classificatie. Analyseer de boodschap van de assistent en het antwoord van de gebruiker. Je taak is om hun dialoog te classificeren naar een van de antwoordcategorieën "search", "product_list", "purchase".
+search - wanneer de gebruiker voedselkenmerken, naam, prijs opsomt. De gebruiker zal vragen om een specifiek product te vinden
+product_list - wanneer de gebruiker vraagt om alle producten die aan de winkelwagen zijn toegevoegd, weer te geven. Of alle producten die hij van plan is te bestellen.
+purchase - wanneer de assistent voorstelt om een product aan de winkelwagen toe te voegen en de gebruiker akkoord gaat.
+Dus je antwoord moet slechts één woord bevatten - de categorie waartoe de dialoog van de assistent en de gebruiker behoort (search, product_list of purchase).
+"""
+    ADD_TO_CART_PROMPT = "Je bent de virtuele assistent van de Malu Haarlemse winkel. Uw taak is om heel kort en bondig te reageren op het verzoek van de gebruiker om voedsel aan het winkelwagentje toe te voegen. Stel bijvoorbeeld dat het gevraagde product aan de winkelwagen is toegevoegd en bied aan om iets anders te kopen"
+    PRODUCT_LIST_PROMPT = "Je bent de virtuele assistent van de Malu Haarlemse winkel. Uw taak, op verzoek van de gebruiker, is om een lijst van alle producten toegevoegd aan de winkelwagen en vertellen de totale kosten. Analyseer de geschiedenis van de dialoog om te begrijpen welke producten de gebruiker aan de winkelwagen heeft toegevoegd"
+
+
+class DevelopmentConfig(BaseConfig):
+    pass
+
+
+class ProductionConfig(BaseConfig):
+    ORIGINS = ["http://127.0.0.1:8000",
+               "http://localhost:8000",
+               'https://alcolm.com/',
+               'https://help.alcolm.com/'
+               ]
+
+    def __init__(self):
+        self.products_dataset = pd.read_csv(
+            str(self.BASE_DIR / 'project' / 'bot' / 'search_tools' / 'cleaned_products.csv'))
+
+
+class TestConfig(BaseConfig):
+    pass
+
+
+@lru_cache()
+def get_settings() -> DevelopmentConfig | ProductionConfig | TestConfig:
+    config_cls_dict = {
+        'development': DevelopmentConfig,
+        'production': ProductionConfig,
+        'testing': TestConfig
+    }
+    config_name = env('FASTAPI_CONFIG', default='production')
+    config_cls = config_cls_dict[config_name]
+    return config_cls()
+
+
+settings = get_settings()

project/ws/__init__.py

@@ -0,0 +1,5 @@
+from fastapi import APIRouter
+
+ws_router = APIRouter()
+
+from . import views

project/ws/views.py

@@ -0,0 +1,16 @@
+from fastapi import WebSocket, WebSocketDisconnect
+from . import ws_router
+from ..bot.chatbot import ChatBot
+
+
+@ws_router.websocket("/ws/{client_id}")
+async def websocket_endpoint(websocket: WebSocket, client_id: str):
+    await websocket.accept()
+    chatbot = ChatBot()
+    try:
+        while True:
+            data = await websocket.receive_json()
+            answer = await chatbot.ask(data)
+            await websocket.send_json(answer)
+    except WebSocketDisconnect:
+        pass

requirements.txt

@@ -0,0 +1,37 @@
+annotated-types==0.6.0
+anyio==4.3.0
+certifi==2024.2.2
+click==8.1.7
+distro==1.9.0
+environs==11.0.0
+exceptiongroup==1.2.1
+faiss-gpu==1.7.2
+fastapi==0.110.3
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+idna==3.7
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+marshmallow==3.21.2
+numpy==1.26.4
+openai==1.25.1
+packaging==24.0
+pandas==2.2.2
+pydantic==2.7.1
+pydantic_core==2.18.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+PyYAML==6.0.1
+six==1.16.0
+sniffio==1.3.1
+starlette==0.37.2
+tqdm==4.66.4
+typing_extensions==4.11.0
+tzdata==2024.1
+uvicorn==0.29.0
+uvloop==0.19.0
+watchfiles==0.21.0
+websockets==12.0

static/css/style.css

@@ -0,0 +1,543 @@
+code {
+    font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+    monospace;
+}
+
+::-webkit-scrollbar {
+    width: 10px;
+}
+
+::-webkit-scrollbar-track {
+    border-radius: 10px;
+    background: transparent;
+}
+
+::-webkit-scrollbar-thumb {
+    background: #14274E;
+    border-radius: 10px
+}
+
+::-webkit-scrollbar-thumb:hover {
+    background: #1d3970;
+}
+
+
+:root {
+    margin: 0px;
+}
+
+.App {
+    display: none;
+    position: fixed;
+    bottom: 20px;
+    left: 20px;
+    z-index: 81;
+
+    width: 540px;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.3);
+    border-radius: 10px;
+}
+
+.App .head-container {
+    background-color: #14274E;
+    padding: 19px 50px 19px 50px;
+    display: flex;
+    align-items: center;
+    border-radius: 10px 10px 0px 0px;
+}
+
+.App .head-container .head-log-wrapper {
+    display: flex;
+}
+
+.App .head-container .head-log-wrapper .head-icon {
+    margin-right: 15.72px;
+}
+
+.App .head-container .head-log-wrapper .head-content {
+    position: relative;
+    margin-top: 5px;
+}
+
+.App .head-container .head-log-wrapper .head-content p {
+    margin: 0px;
+    position: absolute;
+    bottom: -1px;
+    color: white;
+    font-weight: 500;
+    letter-spacing: 0.755px;
+    font-size: 16px;
+    margin-left: 18px;
+}
+
+.App .head-container .head-close-wrapper {
+    margin-left: auto;
+    display: flex;
+    cursor: pointer;
+}
+
+.App .head-container .head-close-wrapper p {
+    margin: 0px;
+    color: #EF9E00;
+    font-size: 18px;
+    font-style: normal;
+    font-weight: 500;
+}
+
+.App .head-container .head-close-wrapper span {
+    display: flex;
+    align-items: center;
+}
+
+.App .chatui-container {
+    background-image: url('/static/bgs/bg.png');
+    background-size: cover;
+    background-repeat: no-repeat;
+
+    padding-top: 20px;
+    padding-left: 35px;
+    padding-right: 35px;
+    overflow-y: scroll;
+    /* Define the animation for scrolling down */
+    scroll-behavior: smooth;
+}
+
+.App .chatui-container .user-msg {
+    max-width: 360px;
+    margin-left: auto;
+    border: 2px solid #EF9E00;
+    padding: 10px 10px 10px 10px;
+    border-radius: 15px 0px 15px 15px;
+    background: #FCF4E5;
+    word-wrap: break-word;
+    box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.25);
+    margin-bottom: 40px;
+}
+
+.App .chatui-container .bot-msg {
+    display: flex;
+}
+
+.App .chatui-container .bot-avatar-box span {
+    background-color: #14274E;
+    width: 55px;
+    height: 55px;
+    border-radius: 55px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin-bottom: 10px;
+    box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.25);
+}
+
+.App .chatui-container .bot-msg .bot-content-box {
+    max-width: 360px;
+    /* width: 100%; */
+    margin-left: 20px;
+    padding: 10px 15px 10px 15px;
+    border-radius: 0px 15px 15px 15px;
+    border: 2px solid var(--surface-surface-invert, #14274E);
+    background: #E7E8EC;
+    box-shadow: 0px 4px 4px 0px rgba(0, 0, 0, 0.25);
+    margin-bottom: 40px;
+}
+
+.App .chatui-container .bot-msg .bot-content-box p {
+    margin: 0 0 0em;
+<!-- /* min-height: 48px; */
+-->
+}
+
+.App .search-container {
+    padding: 15px 15px 15px 15px;
+    display: flex;
+    background-color: #ffffff;
+    align-items: center;
+    border-radius: 0px 0px 10px 10px;
+    border-bottom: 1px solid rgba(57, 72, 103, 0.1);
+    border-left: 1px solid rgba(57, 72, 103, 0.1);
+    border-right: 1px solid rgba(57, 72, 103, 0.1);
+}
+
+.App .search-container .search-input {
+    width: 100%;
+    padding: 15px 20px 15px 20px;
+    font-size: 16px;
+    border: 2px solid #000308;
+    border-radius: 10px 0px 0px 10px;
+    border-right: none;
+    outline: none;
+}
+
+.App .search-container .submit-btn {
+    cursor: pointer;
+}
+
+.App .search-container .submit-btn span {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background-color: #14274E;
+    padding: 11px;
+    border-radius: 0px 10px 10px 0px;
+}
+
+
+.sticky-chat-icon {
+    position: fixed;
+    bottom: 20px;
+    left: 20px;
+    width: 80px;
+    height: 80px;
+    text-align: center;
+    cursor: pointer;
+    z-index: 80;
+}
+
+
+.chat-wrapper {
+    height: 80px;
+    width: 80px;
+    background-color: #14274e;
+    border-radius: 50% 50% 50% 0;
+    border: 2px solid #ef9e00;
+    display: flex; /* Включить flex-контейнер */
+    justify-content: center; /* Выравнивание по горизонтали по центру */
+    align-items: center;
+}
+
+.bot-avatar {
+    fill: white;
+    width: 40px;
+    height: 40px;
+}
+
+.sticky-chat-icon:hover .bot-avatar {
+    fill: #ef9e00; /* Цвет при наведении (оранжевый) */
+    width: 45px; /* Размер при наведении (45x45 пикселей) */
+    height: 45px;
+}
+
+
+.ball {
+    background-color: #394867;
+    width: 8px;
+    height: 8px;
+    margin: 2px;
+    border-radius: 100%;
+    display: inline-block;
+    animation: bounce 0.5s ease-in-out infinite;
+}
+
+.ball1 {
+    animation-delay: 0.1s;
+    animation-direction: alternate; /* Для движения вверх и вниз */
+}
+
+.ball2 {
+    animation-delay: 0.2s;
+    animation-direction: alternate;
+}
+
+.ball3 {
+    animation-delay: 0.3s;
+    animation-direction: alternate;
+}
+
+@keyframes bounce {
+    0%, 100% {
+        transform: translateY(0);
+    }
+    50% {
+        transform: translateY(-7px); /* Для дерганного движения */
+    }
+}
+
+/* new loading state */
+.loading-text {
+    min-height: 48px;
+}
+
+.loading-text p {
+    opacity: 0;
+}
+
+@keyframes fade-in-out {
+    0% {
+        opacity: 0;
+    }
+    50% {
+        opacity: 1;
+    }
+    100% {
+        opacity: 0;
+    }
+}
+
+@keyframes fade-in-out-2 {
+    0% {
+        opacity: 0;
+    }
+    55% {
+        opacity: 1;
+    }
+    100% {
+        opacity: 0;
+    }
+}
+
+/* end of new loading state */
+
+
+.gpt-send-button {
+    background-color: #14274E;
+    width: 100%;
+    display: flex;
+    justify-content: center;
+    border-radius: 10px;
+    border: 2px solid #14274E;
+    align-items: center;
+    cursor: pointer;
+}
+
+.gpt-send-button span {
+    color: #FCFDFF;
+    text-align: center;
+    font-size: 20px;
+    padding: 10px 8px;
+    font-style: normal;
+    font-weight: 700;
+    line-height: normal;
+    letter-spacing: -0.4px;
+}
+
+.send-button-blocked {
+    background-color: #14274ed8;
+    border: 2px solid #14274ed8 !important;
+    pointer-events: none;
+    cursor: default;
+}
+
+.gpt-form {
+    /* max-width: var(--reading-width, 48em); */
+    margin-right: auto;
+    margin-left: auto;
+    margin-bottom: 40px;
+}
+
+.gpt-input {
+    height: 46px;
+    width: 100%;
+    margin: 10px 0;
+    display: flex;
+    box-sizing: border-box;
+    padding: 2px 20px;
+    flex-direction: column;
+    justify-content: center;
+    align-items: flex-start;
+    flex: 1 0 0;
+    border-radius: 10px;
+    border: 2px solid #14274E;
+    background-color: #ffffff;
+    color: rgb(42, 43, 42);
+    font-size: 14px;
+    line-height: 1.2em;
+}
+
+.gpt-input::placeholder {
+    color: rgba(42, 43, 42, 0.808);
+}
+
+.gpt-input.invalid {
+    border: 2px solid rgb(219, 0, 0);
+}
+
+
+.error-message {
+    color: rgb(219, 0, 0);
+    font-size: 14px;
+    margin-top: 5px;
+    display: none;
+}
+
+.gpt-textarea {
+    height: auto;
+    resize: vertical;
+    padding-top: 20px;
+    font-size: 14px;
+}
+
+.gpt-textarea::placeholder {
+    color: rgba(42, 43, 42, 0.808);
+}
+
+.custom_bot_initial_question {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+    row-gap: 10px;
+    max-width: 335px;
+    margin-left: 20px;
+    margin-top: -2.7rem;
+    margin-bottom: 40px;
+}
+
+.custom_bot_initial_question p {
+    padding: 5px 12px;
+    font-size: 12px;
+    background: #fff;
+    border: 2px solid #14274E;
+    border-radius: 50px;
+    margin: 0;
+    transition: 0.3s ease;
+    cursor: pointer;
+    font-style: italic;
+}
+
+.custom_bot_initial_question p:hover {
+    background: #14274E;
+    color: #fff;
+}
+
+@media (max-width: 767px) {
+    .App {
+        width: 100%;
+        height: 100vh;
+        left: 0;
+        bottom: 0;
+        display: none;
+        position: fixed;
+        z-index: 81;
+        box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.3);
+        border-radius: 10px;
+        /* Дополнительные стили для .app на маленьких экранах */
+    }
+
+    .chat-section {
+        height: 100vh;
+    }
+
+    .App .chatui-container {
+        padding-left: 15px;
+        padding-right: 15px;
+    }
+
+    .App .chatui-container .user-msg {
+        padding: 15px 20px;
+        margin-bottom: 20px;
+    }
+
+    .App .chatui-container .bot-msg .bot-content-box {
+        padding: 5px 20px 5px 20px;
+        margin-bottom: 20px;
+    }
+
+    .custom_bot_initial_question {
+        margin-top: 0;
+    }
+
+    #chatui-container {
+        height: calc(100vh - 95px - 93px);
+    }
+}
+
+
+.overlay {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background: rgba(255, 255, 255, 0.7);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    visibility: hidden;
+    z-index: 999;
+}
+
+.overlay h1 {
+    margin: 0;
+    font-size: 2em;
+}
+
+
+.report-loader {
+    position: absolute;
+    top: calc(50% - 45px);
+    left: calc(50% - 45px);
+    width: 90px;
+    height: 90px;
+    border-radius: 50%;
+    perspective: 800px;
+}
+
+.report-inner {
+    position: absolute;
+    box-sizing: border-box;
+    width: 100%;
+    height: 100%;
+    border-radius: 50%;
+}
+
+.report-inner.report-one {
+    left: 0;
+    top: 0;
+    animation: rotate-one 1s linear infinite;
+    border-bottom: 3px solid #d52121;
+}
+
+.report-inner.report-two {
+    right: 0%;
+    top: 0%;
+    animation: rotate-two 1s linear infinite;
+    border-right: 3px solid #d52121;
+}
+
+.report-inner.report-three {
+    right: 0%;
+    bottom: 0%;
+    animation: rotate-three 1s linear infinite;
+    border-top: 3px solid #d52121;
+}
+
+@keyframes rotate-one {
+    0% {
+        transform: rotateX(35deg) rotateY(-45deg) rotateZ(0deg);
+    }
+    100% {
+        transform: rotateX(35deg) rotateY(-45deg) rotateZ(360deg);
+    }
+}
+
+@keyframes rotate-two {
+    0% {
+        transform: rotateX(50deg) rotateY(10deg) rotateZ(0deg);
+    }
+    100% {
+        transform: rotateX(50deg) rotateY(10deg) rotateZ(360deg);
+    }
+}
+
+@keyframes rotate-three {
+    0% {
+        transform: rotateX(35deg) rotateY(55deg) rotateZ(0deg);
+    }
+    100% {
+        transform: rotateX(35deg) rotateY(55deg) rotateZ(360deg);
+    }
+}
+
+#message {
+    display: none;
+    position: absolute;
+    top: 0;
+    right: 0;
+    background-color: #b3ff00;
+    padding: 10px;
+    z-index: 100;
+  }
+
+.message h5{
+    margin-bottom: 0;
+}

static/js/voice.js

@@ -0,0 +1,146 @@
+const initialMessage = ''
+
+const loadingModal = document.getElementById('loadingModal');
+const messageHistory = document.getElementById('chatHistory')
+let mediaRecorder;
+let audioChunks = [];
+let socket
+
+function makeLoading() {
+    loadingModal.style.visibility = 'visible';
+}
+
+function stopLoading() {
+    loadingModal.style.visibility = 'hidden';
+}
+
+function startCall() {
+    const uuid = generateUUID()
+    messageHistory.innerHTML = ``
+    socket = new WebSocket(`ws://127.0.0.1:8000/ws/${uuid}`);
+    socket.onopen = () => {
+        startRecording()
+    };
+
+    socket.onclose = (event) => console.log('WebSocket disconnected', event);
+    socket.onerror = (error) => {
+        alert('Something was wrong. Try again later.')
+        window.location.reload()
+    };
+    socket.onmessage = (event) => playResponse(event.data);
+}
+
+const startRecording = () => {
+    showMessage('You can speak!')
+    navigator.mediaDevices.getUserMedia({audio: true})
+        .then(stream => {
+            mediaRecorder = new MediaRecorder(stream);
+            try {
+                mediaRecorder.ondataavailable = (event) => {
+                    audioChunks.push(event.data)
+
+                };
+            } catch (e) {
+                alert('It is not possible to send an empty message')
+            }
+            mediaRecorder.start();
+        })
+        .catch(error => {
+            console.error('Error accessing the microphone', error);
+        });
+};
+
+const stopAndSend = () => {
+    makeLoading()
+    mediaRecorder.ondataavailable = (event) => {
+        audioChunks.push(event.data)
+        console.log(audioChunks)
+
+        const audioBlob = new Blob(audioChunks, {type: 'audio/wav'});
+        const reader = new FileReader();
+        reader.readAsDataURL(audioBlob);
+        reader.onloadend = () => {
+            let base64String = reader.result;
+            base64String = base64String.split(',')[1];
+            const dataWS = {
+                'audio': base64String,
+            }
+            console.log(dataWS)
+            socket.send(JSON.stringify(dataWS));
+            audioChunks = [];
+        }
+    }
+    try {
+        mediaRecorder.stop();
+    } catch (e) {
+        return
+    }
+}
+
+
+const playResponse = (data, initMessage = false) => {
+    data = JSON.parse(data)
+    if (!initMessage) {
+        const aiResponse = data['ai_response']
+        if (aiResponse.startsWith('https://blue-estate-agency.com/') || aiResponse.startsWith('http://blue-estate-agency.com/')) {
+            window.open(aiResponse, '_blank');
+        }
+    }
+    const audioSrc = `data:audio/mp3;base64,${data['voice_response']}`;
+    const audio = new Audio(audioSrc);
+    stopLoading()
+    if (!initMessage) {
+        createMessage('User', data['user_query'])
+    }
+    audio.play();
+
+    audio.onended = () => {
+        startRecording()
+        if (!initMessage) {
+            createMessage('Assistant', data['ai_response'])
+        }
+    };
+}
+
+function endCall() {
+    if (mediaRecorder && mediaRecorder.state !== 'inactive') {
+        mediaRecorder.stop();
+        audioChunks = []
+    }
+    if (socket && socket.readyState === WebSocket.OPEN) {
+        socket.close();
+    }
+    showMessage('The call is ended')
+}
+
+function generateUUID() {
+    const arr = new Uint8Array(16);
+    window.crypto.getRandomValues(arr);
+
+    arr[6] = (arr[6] & 0x0f) | 0x40;
+    arr[8] = (arr[8] & 0x3f) | 0x80;
+
+    return ([...arr].map((b, i) =>
+        (i === 4 || i === 6 || i === 8 || i === 10 ? "-" : "") + b.toString(16).padStart(2, "0")
+    ).join(""));
+}
+
+function showMessage(message_text) {
+    const message = document.getElementById('message');
+    message.innerText = message_text
+    message.style.display = 'block';
+
+    setTimeout(function () {
+        message.style.display = 'none';
+    }, 2000);
+}
+
+function createMessage(type, message) {
+    const newMessage = document.createElement('div')
+    newMessage.className = 'message rounded-4 bg-white mb-4 mx-4 py-2 px-3 border'
+    newMessage.innerHTML = `
+    <h5>${type}</h5>
+    ${message}
+    `
+    messageHistory.appendChild(newMessage)
+}