initial commit

.github/workflows/dockerhub.yaml

@@ -0,0 +1,39 @@
+name: Publish ConversAI Docker image
+
+on:
+  push:
+    branches: [ dev ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: techconsp/tcp_frwjeiqhpyl9ty53cyfg2jtpbhwwzl_co
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -0,0 +1,148 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Machine Learning and Speech Libraries
+# TensorFlow
+*.ckpt*
+*.pbtxt
+*.tfevents*
+# PyTorch
+*.pt
+# Keras
+*.h5
+# Scikit-learn
+*.pkl
+# Speech Recognition
+*.wav
+*.mp3
+.idea/
+logs
+images
+resources

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY . /app
+
+RUN chmod -R 777 /app
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+        build-essential \
+        git \
+        cmake \
+        poppler-utils \
+        ffmpeg \
+        libsm6 \
+        libxext6 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir nltk && \
+    mkdir -p /app/nltk_data && \
+    chmod -R 777 /app/nltk_data && \
+    python -m nltk.downloader -d /app/nltk_data all
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 8000
+
+CMD ["python", "app.py"]

README.md

@@ -1,8 +1,8 @@
 ---
 title: J7RSYILECL
 emoji: 🚀
-colorFrom: indigo
-colorTo: blue
+colorFrom: purple
+colorTo: purple
 sdk: docker
 pinned: false
 ---

app.py

@@ -0,0 +1,55 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import uvicorn
+from fastapi import FastAPI, Depends
+from fastapi.middleware.cors import CORSMiddleware
+from src import logging as logger
+from src.api.conversai_api import conversai_api_router
+from src.api.speech_api import speech_translator_router
+from src.api.user_management_api import user_management_api_router
+from src.utils.utils import load_ini_config
+from src.api.analytics_api import analytic_endpoints_router
+from src.api.jwt_bearer import access_check_bearer
+from src.api.jewel_mirror import goldprice_apirouter
+
+
+logger.info("---------------------------> Starting FastAPI Server <---------------------------")
+
+config = load_ini_config("config.ini")
+app = FastAPI(docs_url=config.get('fastapi_config', 'docs_url'), redoc_url=config.get('fastapi_config', 'redoc_url'),
+              openapi_url=config.get('fastapi_config', 'openapi_url')
+              )
+
+PROTECTED = [Depends(access_check_bearer)]
+
+logger.info("---------------------------> FastAPI Server Started <---------------------------")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allow access from all sources
+    allow_credentials=True,
+    allow_methods=["*"],  # Allow all HTTP methods
+    allow_headers=["*"],  # Allow all request headers
+)
+app.include_router(user_management_api_router, prefix="/conversai")
+logger.info("---------------------------> User Management API Started <---------------------------")
+
+app.include_router(conversai_api_router, prefix="/conversai")
+logger.info("---------------------------> ConversAI API Started <---------------------------")
+
+app.include_router(speech_translator_router, prefix="/conversai")
+logger.info("---------------------------> Speech Translator API Started <---------------------------")
+
+app.include_router(analytic_endpoints_router, prefix="/conversai",dependencies=PROTECTED)
+logger.info("---------------------------> Analytics API Started <---------------------------")
+
+#for goldpricing
+app.include_router(goldprice_apirouter,prefix="/conversai",dependencies=PROTECTED)
+logger.info("---------------------------> Goldpricing API Started <---------------------------")
+
+
+
+if __name__ == '__main__':
+    uvicorn.run(app, port=int(config.get('fastapi_config', 'port')), host=config.get('fastapi_config', 'host'),
+                timeout_keep_alive=300, timeout_graceful_shutdown=600)

config.ini

@@ -0,0 +1,39 @@
+[default]
+chatbot_name = ConversAI
+chatbot_prefix = convai
+
+[data_analyzer]
+groq_llm_name = llama-3.1-8b-instant
+additional_query = .In case, you are to plot a chart, make sure the x-axis labels are 90 degree rotated.
+verbose = False
+
+; [easy_ocr]
+; model_path = resources/easyocr_model
+; language = en
+; gpu = True
+
+[speech_to_text]
+model_id = openai/whisper-large-v3
+max_new_tokens = 128
+chunks_length_s = 30
+batch_size = 16
+
+[supabase_chatbot_management]
+user_config_table = ConversAI_UserConfig
+chat_bot_table = ConversAI_ChatbotInfo
+
+[all_mini_l6_v2_vector_embedding]
+device = cuda
+normalize_embeddings = True
+
+
+[fastapi_config]
+host = 0.0.0.0
+port = 8000
+docs_url = /docs
+redoc_url = /redoc
+openapi_url = /openapi.json
+
+
+[oauth]
+redirect_to : https://convers-ai-test.vercel.app/home/

examples/easy_ocr_example.py

@@ -0,0 +1,12 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import cv2
+
+from src.services.ocr.easy_ocr.easy_ocr_ import EasyOCR_
+
+if __name__ == '__main__':
+    image = cv2.imread("images/img.png")
+    ocr = EasyOCR_()
+    print(ocr.read_text(image))

examples/url_text_extraction_example.py

@@ -0,0 +1,10 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.website_url.text_extraction_urls import extract_text_from_url
+
+if __name__ == '__main__':
+    website = "https://huggingface.co/BAAI/bge-m3"
+    extracted_text = extract_text_from_url(website)
+    print(extracted_text)

logging_config.yaml

@@ -0,0 +1,27 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+  standard:
+    format: '[%(asctime)s: %(levelname)s: %(module)s: %(message)s]'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: standard
+    stream: ext://sys.stdout
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: INFO
+    formatter: standard
+    filename: logs/application.log
+    when: midnight
+    interval: 1
+    backupCount: 30
+loggers:
+  __main__:
+    level: DEBUG
+    handlers: [console, file]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [console, file]

requirements.txt

@@ -0,0 +1,99 @@
+accelerate==0.33.0
+annotated-types==0.7.0
+anyio==4.4.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.111.1
+fastapi-cli==0.0.4
+filelock==3.15.4
+fsspec==2024.6.1
+gTTS==2.5.2
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.24.5
+idna==3.7
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.6.20
+nvidia-nvtx-cu12==12.1.105
+packaging==24.1
+psutil==6.0.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+PyYAML==6.0.1
+regex==2024.7.24
+requests==2.32.3
+rich==13.7.1
+safetensors==0.4.3
+scikit-build==0.18.0
+shellingham==1.5.4
+sniffio==1.3.1
+starlette==0.37.2
+sympy==1.13.1
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.0
+tqdm==4.66.4
+transformers @ git+https://github.com/huggingface/transformers.git@85a1269e19af022e04bc2aad82572cd5a9e8cdd9
+triton==3.0.0
+typer==0.12.3
+typing_extensions==4.12.2
+urllib3==2.2.2
+uvicorn==0.30.4
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==12.0
+bs4
+huggingface-hub
+fastembed
+nest_asyncio
+beautifulsoup4
+flashrank
+flashrank[listwise]
+PyMuPDF
+langchain
+langchain-community
+langchain-cohere
+langchain-huggingface
+langchain-qdrant
+langchain-groq
+lxml
+python-dotenv
+pillow
+pandas
+sentence-transformers
+supabase
+unstructured
+urllib3
+langsmith
+pandasai
+easyocr
+youtube-transcript-api
+pdf2image
+PyPDF2
+PyJWT
+

src/__init__.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+
+import logging.config
+import yaml
+import os
+
+if os.path.exists("logs"):
+    pass
+else:
+    os.makedirs("logs")
+
+log_config_path = os.path.join(os.getcwd(), "logging_config.yaml")
+with open(log_config_path, 'r') as file:
+    config = yaml.safe_load(file.read())
+
+logging.config.dictConfig(config)

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/api/analytics_api.py

@@ -0,0 +1,235 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+from dateutil.parser import isoparse
+from fastapi.routing import APIRouter
+from src.pipeline.conversai_analytic_pipeline import ConversAIAnalyticPipeline
+from fastapi import Request
+from src.utils.error_handling import create_success_response, raise_http_exception, \
+    success_response_user_management
+from src.models.apis_models import FeedbackRequest, DailyActiveEndUserRequest, AverageSessionInteractionRequest, \
+    TokenUsageRequest, UserSatisfactionRateRequest
+from src import logging as logger
+
+analytic_endpoints_router = APIRouter(tags=["Analytics Endpoints"])
+
+conversai_analytic_pipeline = ConversAIAnalyticPipeline()
+
+
+@analytic_endpoints_router.post("/daily_chat_count")
+async def daily_chat_count(
+        request: DailyActiveEndUserRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_chat_count API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        dates = [
+            isoparse(i["timestamp"]).date()
+            for i in response
+            if start_date <= isoparse(i["timestamp"]).date() <= end_date
+        ]
+
+        date_count = Counter(dates)
+
+        data = [{"date": date.isoformat(), "count": count} for date, count in date_count.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_chat_count API Response Success  for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> daily_chat_count API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/daily_active_end_user")
+async def daily_active_end_user(
+        request: DailyActiveEndUserRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_active_end_user API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        ip_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                ip_by_date[date].add(ip_address)
+
+        data = [{"date": date.isoformat(), "terminal": len(ips)} for date, ips in ip_by_date.items() if len(ips) > 1]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_active_end_user API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> daily_active_end_user API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/average_session_interaction")
+async def average_session_interaction(
+        request: AverageSessionInteractionRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> average_session_interaction API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        total_messages_by_date = defaultdict(int)
+        unique_ips_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                total_messages_by_date[date] += 1
+                unique_ips_by_date[date].add(ip_address)
+
+        data = []
+        for date in sorted(total_messages_by_date.keys()):
+            total_messages = total_messages_by_date[date]
+            unique_ips = len(unique_ips_by_date[date])
+            average_interactions = total_messages / unique_ips if unique_ips > 0 else 0
+            data.append({"date": date.isoformat(), "interactions": average_interactions})
+
+        response = create_success_response(code=200, data=dict(data=data))
+        logger.info(f">>> average_session_interaction API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> average_session_interaction API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/token_usages")
+async def token_usages(request: TokenUsageRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> token_usages API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        token_usage_by_date = defaultdict(int)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                response_token_count = i.get("ResponseTokenCount")
+                if response_token_count is not None:
+                    token_usage_by_date[date] += response_token_count
+
+        data = [{"date": date.isoformat(), "total_tokens": total_tokens} for date, total_tokens in
+                token_usage_by_date.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> token_usages API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> token_usages API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/add_feedback")
+async def add_feedback(req: Request, request: FeedbackRequest):
+    feedback, user_id, vectorstore = request.feedback, request.user_id, request.vectorstore
+    try:
+        logger.info(f">>> add_feedback API Triggered by {request.vectorstore} <<<")
+
+        client_ip = req.client.host
+        city = conversai_analytic_pipeline.get_ip_info(client_ip)
+
+        conversai_analytic_pipeline.add_feedback_(feedback, user_id, city, client_ip, vectorstore)
+
+        response = success_response_user_management(code=200, message="Add Feedback Sucess")
+        logger.info(f">>> add_feedback API Response Success for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> add_feedback API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/user_satisfaction_rate")
+async def user_satisfaction_rate(
+        request: UserSatisfactionRateRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> user_satisfaction_rate API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        feedback_counts = defaultdict(lambda: {"like": 0, "dislike": 0})
+        response = conversai_analytic_pipeline.feedback_table_(vectorstore)
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                feedback = i.get("feedback")
+                if feedback == "like":
+                    feedback_counts[date]["like"] += 1
+                elif feedback == "dislike":
+                    feedback_counts[date]["dislike"] += 1
+
+        data = []
+        for date in sorted(feedback_counts.keys()):
+            like_count = feedback_counts[date]["like"]
+            dislike_count = feedback_counts[date]["dislike"]
+            total_feedback = like_count + dislike_count
+            satisfaction_rate = (like_count / total_feedback * 100) if total_feedback > 0 else 0
+            data.append({"date": date.isoformat(), "rate": satisfaction_rate})
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> user_satisfaction_rate API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.info(f">>> user_satisfaction_rate API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/conversai_api.py

@@ -0,0 +1,645 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+import io
+import json
+import os
+import string
+import tempfile
+import requests
+import pandas as pd
+from src import logging as logger
+from supabase import create_client
+from urllib.parse import urlparse
+
+from src.api.jwt_bearer import access_check_bearer
+from src.models.apis_models import *
+from fastapi.requests import Request
+from fastapi.routing import APIRouter
+from fastapi import UploadFile, File, HTTPException, Form, Depends
+from src.pipeline.conversai_pipeline import ConversAIPipeline
+from src.api.user_management_api import user_management
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.services.supabase.user_management.token_limit import token_limit_check
+from src.utils.error_handling import create_error_response, create_success_response, raise_http_exception
+from src.api.user_management_api import user_management as user_management_pipeline
+from src.utils.utils import get_ip_info, encode_to_base64, clean_text, decode_base64
+
+conversai_api_router = APIRouter(tags=["ConversAI"])
+
+supabase_client = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY"))
+supabase_client_ = supabase_client
+conversai_pipeline = ConversAIPipeline()
+
+
+@conversai_api_router.post("/add_text" )
+async def add_text(request: AddTextRequest):
+    logger.info(f">>>AddText API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, text = request.vectorstore, request.text
+        track_usage(vectorstore=vectorstore, endpoint="/add_text", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                text=text)
+        text = clean_text(text)
+        if lim:
+            dct = {
+                "output": {"text": text},
+                "source": "Text",
+            }
+            num_token = len(text.translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="text", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username, "chatbotName": chat_bot_name, "dataSourceName": file_name,
+                 "numTokens": num_token, "sourceEndpoint": "/add_text",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200, {"message": "Successfully added the text."})
+            logger.info(f">>>Text added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_text: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/answer_query")
+async def answer_query(request: AnswerQueryRequest, req: Request):
+    logger.info(f">>>answer_query API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore, endpoint="/answer_query")
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        ip_address = req.client.host
+        city = get_ip_info(ip_address)
+        output, followup_questions, source = conversai_pipeline.answer_query_(query=request.query,
+                                                                              vectorstore=request.vectorstore,
+                                                                              llm_model=request.llm_model)
+        supa = supabase_client.table("ConversAI_ChatHistory").insert(
+            {"username": username, "chatbotName": chatbot_name, "llmModel": request.llm_model,
+             "question": request.query, "response": output, "IpAddress": ip_address, "ResponseTokenCount": len(output),
+             "vectorstore": request.vectorstore, "City": city}).execute()
+
+        response = create_success_response(200, data={"output": output, "follow_up_questions": followup_questions,
+                                                      "source": source})
+        logger.info(f">>>Query answered successfully for {request.vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in answer_query: {e} for {request.vectorstore}.<<<")
+        # raise HTTPException(status_code=500, detail="Internal Server Error")
+        raise e
+
+@conversai_api_router.post("/data_analyzer" )
+async def data_analyzer(query: str = Form(...), file: UploadFile = File(...)):
+    logger.info(f">>>data_analyzer API Triggered By {query}<<<")
+    try:
+        extension = file.filename.split(".")[-1]
+        if extension in ["xls", "xlsx", "xlsm", "xlsb"]:
+            df = pd.read_excel(io.BytesIO(await file.read()))
+        elif extension == "csv":
+            df = pd.read_csv(io.BytesIO(await file.read()))
+        else:
+            return {"output": "INVALID FILE TYPE"}
+
+        response = conversai_pipeline.data_analyzer(query=query, dataframe=df)
+        response = create_success_response(200, {"output": response})
+        logger.info(f">>>Data analyzed successfully for {query}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in data_analyzer: {e} for {query}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_links" )
+async def get_links(request: GetLinksRequest):
+    logger.info(f">>>get_links API Triggered By {request.url}<<<")
+    try:
+        response = conversai_pipeline.get_links_(url=request.url, timeout=30)
+        response = create_success_response(200, {"urls": response, "source": urlparse(request.url).netloc})
+        logger.info(f">>>Links fetched successfully for {request.url}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_links: {e} for {request.url}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/image_pdf_text_extraction" )
+async def image_pdf_text_extraction(vectorstore: str = Form(...)
+                                    , pdf: UploadFile = File(...)):
+    logger.info(f">>>image_pdf_text_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/image_pdf_text_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf_bytes = await pdf.read()
+        response = conversai_pipeline.image_pdf_text_extraction_(image_pdf=pdf_bytes)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+        logger.info(f"this is the {lim}")
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            num_tokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username,
+                 "chatbotName": chatbot_name,
+                 "dataSourceName": file_name,
+                 "numTokens": num_tokens,
+                 "sourceEndpoint": "/image_pdf_text_extraction",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200,
+                                               {"source": pdf.filename, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {pdf.filename}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in image_pdf_text_extraction: {e} for {pdf.filename}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/text_pdf_extraction" )
+async def text_pdf_extraction(vectorstore: str = Form(...)
+                              , pdf: UploadFile = File(...)):
+    logger.info(f">>>text_pdf_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/text_pdf_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf = await pdf.read()
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+            temp_file.write(pdf)
+            temp_file_path = temp_file.name
+
+        response = conversai_pipeline.text_pdf_extraction_(pdf=temp_file_path)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+        os.remove(temp_file_path)
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            numTokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": numTokens,
+                         "sourceEndpoint": "/text_pdf_extraction",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"source": source, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {source}.<<<")
+            return response
+
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in text_pdf_extraction: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/youtube_transcript" )
+async def youtube_transcript(request: YoutubeTranscriptRequest):
+    vectorstore, urls = request.vectorstore, request.urls
+    logger.info(f">>>youtube_transcript API Triggered By {urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/youtube_transcript")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        response = conversai_pipeline.youtube_transcript_(url=urls)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+        if lim:
+
+            dct = {
+                "output": response,
+                "source": "www.youtube.com"
+            }
+            num_tokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="youtube", username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_tokens,
+                         "sourceEndpoint": "/youtube_transcript",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the youtube transcript."})
+            logger.info(f">>>Youtube transcript fetched successfully for {urls}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in youtube_transcript: {e} for {urls}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/website_url_text_extraction" )
+async def add_website(request: AddWebsiteRequest):
+    vectorstore, website_urls, source = request.vectorstore, request.website_urls, request.source
+
+    logger.info(f">>>website_url_text_extraction API Triggered By {request.website_urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/fetch_text/urls")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        text = conversai_pipeline.website_url_text_extraction_list_(urls=website_urls)
+        text_corpus="\n".join(text.values())
+
+        logger.info(f">>>website_url_text_extraction len {type(text)}<<<")
+
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=text_corpus)
+        if not lim:
+
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+        else:
+            dct = {
+                "output": text,
+                "source": source
+            }
+            num_tokens = len(
+                " ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=urlparse(source).netloc,
+                                                                         username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_tokens,
+                         "sourceEndpoint": "/fetch_text/urls",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the website text."})
+            logger.info(f">>>Website text extracted successfully for {request.website_urls}.<<<")
+            return response
+    except Exception as e:
+        logger.error(f">>>Error in website_url_text_extraction: {e} for {request.website_urls}.<<<")
+        raise e
+        # raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+
+
+
+@conversai_api_router.get("/get_current_count" )
+async def get_count(vectorstore: str):
+    logger.info(f">>>get_current_count API Triggered By {vectorstore}<<<")
+    try:
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        current_count = user_management_pipeline.get_current_count_(username)
+
+        response = create_success_response(200, {"current_count": current_count})
+        logger.info(f">>>Current count fetched successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_current_count: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/list_chatbots" )
+async def list_chatbots(request: ListChatbotsRequest):
+    logger.info(f">>>list_chatbots API Triggered By {request.username}<<<")
+    try:
+        chatbots = user_management.list_tables(username=request.username)
+        response = create_success_response(200, {"chatbots": chatbots})
+        logger.info(f">>>Chatbots listed successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbots: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_chat_history" )
+async def chat_history(request: GetChatHistoryRequest):
+    logger.info(f">>>get_chat_history API Triggered By {request.vectorstore}<<<")
+    try:
+        _, username, chatbotName = request.vectorstore.split("$", 2)
+
+        history = supabase_client.table("ConversAI_ChatHistory").select(
+            "timestamp", "question", "response"
+        ).eq("username", username).eq("chatbotName", chatbotName).execute().data
+
+        response = create_success_response(200, {"history": history})
+        logger.info(f">>>Chat history fetched successfully for {request.vectorstore}.<<<")
+        return response
+
+
+    except IndexError:
+        logger.warning(f"Chat history not found for {request.vectorstore}")
+        return create_error_response(404, "Chat history not found for the given chatbot.")
+
+    except Exception as e:
+        logger.error(f">>>Error in get_chat_history: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot" )
+async def delete_chatbot(request: DeleteChatbotRequest):
+    logger.info(f">>>delete_chatbot API Triggered By {request.vectorstore}<<<")
+    try:
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        supabase_client.table('ConversAI_ChatbotInfo').delete().eq('user_id', username).eq('chatbotname',
+                                                                                           chatbot_name).execute()
+        all_sources = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName", chatbot_name).execute().data
+        all_sources = [x["sourceContentURL"].split("/")[-1] for x in all_sources]
+        supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("username", username).eq("chatbotName",
+                                                                                                   chatbot_name).execute()
+        supabase_client.table("ConversAI_ActivityLog").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        supabase_client.table("ConversAI_ChatHistory").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        for source in all_sources:
+            supabase_client.table("ConversAI_Chatbot")
+            supabase_client.storage.from_("ConversAI").remove(source)
+        user_management.delete_table(table_name=chatbot_name)
+        user_management.delete_qdrant_cluster(vectorstorename=request.vectorstore)
+        response = create_success_response(200, {"message": "Chatbot deleted successfully"})
+        logger.info(f">>>Chatbot deleted successfully for {request.vectorstore}.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/add_qa_pair" )
+async def add_qa_pair(request: AddQAPairRequest):
+    logger.info(f">>>add_qa_pair API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, question, answer = request.vectorstore, request.question, request.answer
+        track_usage(vectorstore=vectorstore, endpoint="/add_qa_pair", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        normal_text = f"\nQUESTION: {question}\nANSWER: {answer}\n"
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                text=normal_text)
+        if lim:
+            dct = {
+                "output": {"text": normal_text},
+                "source": "QA Pair",
+            }
+            num_token = len(normal_text.translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="qa_pair", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chat_bot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_token,
+                         "sourceEndpoint": "/add_qa_pair",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+
+            response = create_success_response(200, {"message": "Successfully added the qa pair."})
+            logger.info(f">>>QA Pair added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_qa_pair: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/load_edited_json" )
+async def load_edited_json(request: LoadEditedJson):
+    vectorstore, data_source_name, source_endpoint, json_data = request.vectorstore, request.data_source_name, request.source_endpoint, request.json_data
+    username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+
+    logger.info(f">>>loadEditedJson API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore,
+                    endpoint="/load_edited_json")
+        json_data = decode_base64(request.json_data)
+        json_data = json.dumps(json_data, indent=1).encode("utf-8", errors="replace")
+        file_name = user_management_pipeline.create_data_source_name(source_name=data_source_name,
+                                                                     username=username)
+        response = supabase_client.storage.from_("ConversAI").upload(file=json_data, path=f"{file_name}_data.json")
+        response = (
+            supabase_client.table("ConversAI_ChatbotDataSources")
+            .insert({"username": username,
+                     "chatbotName": chatbot_name,
+                     "dataSourceName": file_name,
+                     "sourceEndpoint": source_endpoint,
+                     "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                      f"{file_name}_data.json")})
+            .execute()
+        )
+        response = create_success_response(200, {"output": "Successfully loaded the edited json."})
+        logger.info(f">>>Edited json loaded successfully for {vectorstore}.<<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in loadEditedJson: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/list_chatbot_sources" )
+async def list_chatbot_sources(vectorstore: str):
+    try:
+        logger.info(f">>>list_chatbot_sources API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/list_chatbot_sources")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        result = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute().data
+
+        response = create_success_response(200, {"output": result})
+        logger.info(f">>>Chatbot listed successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbot_sources: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/get_data_source" )
+async def get_data_source(vectorstore: str, source_url: str):
+    try:
+        logger.info(f">>>get_data_source API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/get_data_source")
+        r = requests.get(source_url)
+        res = encode_to_base64(eval(r.content.decode("utf-8", errors="replace")))
+
+        response = create_success_response(200, {"output": res})
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_data_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot_source" )
+async def delete_chatbot_source(request: DeleteChatbotSourceRequest):
+    vectorstore, data_source_name = request.vectorstore, request.data_source_name
+    try:
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/delete_chatbot_source")
+        response = supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("dataSourceName",
+                                                                                     data_source_name).execute()
+        response = supabase_client.storage.from_('ConversAI').remove(f"{data_source_name}_data.json")
+
+        response = create_success_response(200, {"output": f"Successfully deleted the {data_source_name} data source."})
+
+        logger.info(f">>>Data source deleted successfully for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/train_chatbot")
+async def train_chatbot(request: TrainChatbotRequest):
+    vectorstore, url_sources = request.vectorstore, request.urls
+    logger.info(f">>>train_chatbot API Triggered By {vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/train_chatbot")
+        texts = []
+        sources = []
+        fileTypes = [
+            supabase_client.table("ConversAI_ChatbotDataSources").select("sourceEndpoint").eq("sourceContentURL",
+                                                                                              x).execute().data[0][
+                "sourceEndpoint"] for x in url_sources]
+        for source, fileType in zip(url_sources, fileTypes):
+            if ((fileType == "/text_pdf_extraction") | (fileType == "/image_pdf_text_extraction")):
+                logger.info(f"Source is {source}")
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                logger.info(f"content is {content}")
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+
+                sources.append(fileSource)
+            elif fileType == "/add_text" or fileType == "/add_qa_pair":
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]["text"]
+                fileSource = file["source"]
+                texts.append(content.replace("\n", " "))
+                sources.append(fileSource)
+            elif ((fileType == "/fetch_text/urls") | (fileType == "/youtube_transcript")):
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+                sources.append(fileSource)
+            else:
+                pass
+        texts = [(text, source) for text, source in zip(texts, sources)]
+        conversai_pipeline.add_document_(texts, vectorstore)
+        response = create_success_response(200, {"message": "Chatbot trained successfully."})
+        logger.info(f">>>Chatbot trained successfully for {vectorstore}.<<<")
+
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in train_chatbot: {e} for {vectorstore}.<<<")
+        raise e
+        # raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/activity_log" )
+async def activity_log(username: str):
+    logger.info(f">>>activityLog API Triggered By {username}<<<")
+    try:
+        response = supabase_client.table("ConversAI_ActivityLog").select("*").eq("username", username).execute().data
+
+        logger.info(f">>>Activity log fetched successfully for {username}.<<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in activityLog: {e} for {username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/new_chatbot" )
+async def new_chatbot(request: NewChatbotRequest):
+    logger.info(f">>> new_chatbot API Triggered <<<")
+    try:
+        response = user_management.new_chatbot_(chatbot_name=request.chatbot_name, username=request.username)
+        logger.info(f">>> Chatbot created successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in new_chatbot: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/jewel_mirror.py

@@ -0,0 +1,21 @@
+from fastapi.routing import APIRouter
+from fastapi.requests import Request
+
+from src.jewel_mirror.jewel_mirror import GoldPricingJewelMirror
+from src.models.apis_models import GoldPriceRequest
+from src import logging as logger
+
+goldprice_apirouter=APIRouter(tags=["GoldPrice"])
+
+goldpricing=GoldPricingJewelMirror()
+
+@goldprice_apirouter.post("/gold_price")
+async def answer_goldprice(request:GoldPriceRequest,req:Request):
+    logger.info(f">>>Gold Price API Triggered <<<")
+    try:
+        response=goldpricing.response_query(query=request.query,llmModel=request.llm_model)
+        logger.info(f">>>Gold Price API Success<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in Gold Price API: {e} <<<")
+        raise e

src/api/jwt_bearer.py

@@ -0,0 +1,29 @@
+"""
+Created By: ishwor subedi
+Date: 2024-09-03
+"""
+import os
+from fastapi import Depends
+from supabase import create_client
+from src import logging as logger
+from src.utils.error_handling import create_error_response, raise_http_exception
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+security = HTTPBearer()
+
+supabase_client = create_client(
+    os.getenv("SUPABASE_URL"),
+    os.getenv("SUPABASE_KEY")
+)
+
+
+async def access_check_bearer(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    access_token = credentials.credentials
+    try:
+        supabase_client.auth.get_user(access_token)
+
+    except Exception as e:
+        logger.info(f">>> Invalid access token {e}<<<")
+        raise_http_exception(code=401,
+                                   message="Invalid Access Token", details=[
+                {"info": "Invalid access token or access token expired please login again"}])

src/api/speech_api.py

@@ -0,0 +1,85 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import os
+import tempfile
+from fastapi import Form
+from fastapi import UploadFile, HTTPException, status
+from src.models.apis_models import TextToSpeechRequest
+from fastapi.routing import APIRouter
+from src.pipeline.speech_transcription_pipeline import SpeechTranscriptionPipeline
+from src import logging as logger
+from src.utils.error_handling import create_success_response, raise_http_exception
+
+speech_translator_router = APIRouter(tags=["SpeechTranscription"])
+pipeline = SpeechTranscriptionPipeline()
+
+
+@speech_translator_router.post(
+    "/text_to_speech",
+)
+async def text_to_speech(request: TextToSpeechRequest):
+    logger.info(f">>>text_to_speech API Triggered <<<")
+    try:
+        audio_bytes = pipeline.text_to_speech(request.text, request.lang, request.tld)
+        if not audio_bytes:
+            raise ValueError("Audio generation failed.")
+        response = create_success_response(code=200, data={"audio": audio_bytes})
+        logger.info(f">>>text_to_speech API success <<<")
+        return response
+    except ValueError as ve:
+        logger.info(f">>>text_to_speech API failed {ve}<<<")
+        raise_http_exception(code=400, message="Text to speech failed")
+
+    except Exception as e:
+        logger.error(f">>> Error processing text-to-speech {e}<<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+
+@speech_translator_router.post(
+    "/speech_to_text",
+
+)
+async def speech_to_text(audio: UploadFile, lang: str = Form(...)):
+    logger.info(f">>>speech_to_text API Triggered <<<")
+    try:
+        audio_bytes = await audio.read()
+        if not audio_bytes:
+            logger.error(f">>> Empty audio file <<<")
+            raise ValueError("Empty audio file")
+    except Exception as e:
+        logger.error(f">>> Invalid audio file {e}<<<")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid audio file"
+        )
+
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
+            temp_audio_file.write(audio_bytes)
+            temp_audio_file_path = temp_audio_file.name
+    except Exception as e:
+        logger.error(f">>> Error creating temporary file{e} <<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+    try:
+        transcript = pipeline.speech_to_text(temp_audio_file_path, lang)
+        response = create_success_response(code=200, data={"transcript": transcript})
+        logger.info(f">>>speech_to_text API success <<<")
+
+        return response
+
+    except FileNotFoundError:
+        logger.error(f">>> Temporary file not found <<<")
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Temporary file not found"
+        )
+    except Exception as e:
+        logger.error(f">>> Error processing speech-to-text {e}<<<")
+        raise_http_exception(code=500, message="Error processing speech-to-text")
+
+    finally:
+        if os.path.exists(temp_audio_file_path):
+            os.remove(temp_audio_file_path)

src/api/user_management_api.py

@@ -0,0 +1,157 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from fastapi import Depends
+from src import logging as logger
+from src.models.apis_models import *
+from fastapi.routing import APIRouter
+from src.api.jwt_bearer import access_check_bearer, supabase_client
+from src.pipeline.user_management_pipeline import SupabaseUserManagementPipeline
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.utils.error_handling import raise_http_exception, create_success_response, create_error_response
+
+user_management_api_router = APIRouter(tags=["User Management"])
+
+user_management = SupabaseUserManagementPipeline()
+
+
+@user_management_api_router.post("/user_signup")
+async def user_signup(request: UserSignupRequest):
+    logger.info(f">>>user_signup API Triggered <<<")
+    response = user_management.user_signup_(username=request.username, email=request.email, password=request.password)
+    logger.info(f">>>user_signup API Success<<<")
+
+    return response
+
+
+@user_management_api_router.post("/user_signin")
+async def user_signin(request: UserSigninRequest):
+    logger.info(f">>>user_signin API Triggered <<<")
+
+    response = user_management.user_signin_(email=request.email, password=request.password)
+    if response != None:
+        logger.info(f">>>user_signin API Success.<<<")
+        return response
+    else:
+        logger.info(f">>> Email or password is incorrect please try again.<<<")
+        response = create_error_response(400, "Email or password is incorrect please try again.")
+        return response
+
+
+@user_management_api_router.post("/get_user_data")
+async def get_user_data(request: GetUserDataRequest):
+    logger.info(f">>>get_user_data API Triggered <<<")
+    response = user_management.get_user_data_(access_token=request.access_token)
+    return response
+
+
+@user_management_api_router.post("/login_with_access_token")
+async def login_with_access_token(request: LoginWithAccessTokenRequest):
+    logger.info(f">>>login_with_access_token API Triggered <<<")
+
+    response = user_management.login_with_access_token_(access_token=request.access_token,
+                                                        refresh_token=request.refresh_token)
+    logger.info(f">>>login_with_access_token API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/set_session_data")
+async def set_session_data(request: SetSessionDataRequest):
+    logger.info(f">>> set_session_data API Triggered <<<")
+
+    response = user_management.set_session_data_(access_token=request.access_token, refresh_token=request.refresh_token,
+                                                 user_id=request.user_id)
+    return response
+
+
+@user_management_api_router.post("/sign_out")
+async def sign_out():
+    logger.info(f">>> sign_out API Triggered <<<")
+
+    response = user_management.sign_out_()
+    logger.info(f">>>sign_out API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/oauth_signin")
+async def oauth_signin():
+    logger.info(f">>> oauth_signin API Triggered <<<")
+    response = user_management.oauth_signin_()
+    logger.info(f">>>oauth_signin API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/check_session")
+async def check_session():
+    logger.info(f">>>check_session API Triggered <<<")
+
+    response = user_management.check_session_()
+    return response
+
+
+@user_management_api_router.get("/get_public_chatbot")
+async def get_public_chatbots():
+    logger.info(f">>>get_public_chatbot API Triggered<<<")
+    try:
+        response = supabase_client.table("ConversAI_ChatbotInfo").select("*").eq("isPrivate", False).execute().data
+        logger.info(f">>>Public chatbots fetched successfully.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in get_public_chatbot: {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/public_private_check")
+async def public_or_private(request: PublicPrivateCheckRequest):
+    vectorstore, mode = request.vectorstore, request.mode
+    logger.info(f">>>public_private_check API Triggered for {vectorstore}.<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/public_private_check")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        if len(mode) == 0:
+            value = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .select("isPrivate")
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            value = value.data[0]["isPrivate"]
+            response = create_success_response(200, {"output": value})
+        else:
+            response = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .update({"isPrivate": mode})
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            response = create_success_response(200, {"output": response})
+        logger.info(f">>>Public/Private check successful for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in public_private_check: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/refresh_session", dependencies=[Depends(access_check_bearer)])
+async def refresh_session(request: RefreshSessionRequest):
+    logger.info(f">>>refresh_session API Triggered <<<")
+    response = user_management.refresh_session__(refresh_token=request.refresh_token)
+    logger.info(f">>>refresh token fetched successfully.<<<")
+
+    return response
+
+
+@user_management_api_router.post("/username_creation_oauth", dependencies=[Depends(access_check_bearer)])
+async def username_creation_oauth(request: UsernameCreationOauthRequest):
+    logger.info(f">>> username_creation_oauth API Triggered <<<")
+
+    response = user_management.username_creation_oauth_(username=request.username, user_id=request.user_id,
+                                                        email=request.email)
+
+    logger.info(f">>>username creation successful.<<<")
+    return response

src/jewel_mirror/jewel_mirror.py

@@ -0,0 +1,69 @@
+# Todo: class and object (gold_price) ,
+from langchain_community.document_loaders import WebBaseLoader
+from langchain.schema import Document
+from langchain.prompts import PromptTemplate
+from langchain_groq import ChatGroq
+from langchain.chains.question_answering import load_qa_chain
+
+
+class GoldPricingJewelMirror:
+    def __init__(self):
+        self.template=(
+                    "You are an elite financial analyst with extensive expertise in finance and investment strategies. "
+                    "Provide a **direct and concise** answer to the question below **without** using any introductory phrases such as "
+                    "'based on the context provided,' 'according to the information given,' or similar expressions. "
+                    "Your response must rely solely on the information from the provided context and should **not** include any additional commentary, explanations, or opinions.\n\n"
+                    "📊 **Context:**\n{context}\n\n"
+                    "❓ **Question:**\n{question}\n\n"
+                    "**✅ Answer:**"
+                )
+
+    def fetch_gold_price(self):
+        loader = WebBaseLoader("https://groww.in/gold-rates")
+        documents = loader.load()
+        content = " ".join([doc.page_content for doc in documents])
+        return content
+    def response_query(self,query:str,llmModel: str = "llama-3.1-70b-versatile"):
+        gold_price_context = self.fetch_gold_price()
+        gold_document = Document(page_content=gold_price_context, metadata={"source": "GoldReturns"})
+        custom_prompt = PromptTemplate(
+            input_variables=["context", "question"],
+            template=(
+                "You are an elite financial analyst with extensive expertise in finance and investment strategies. "
+                "Provide a **direct and concise** answer to the question below **without** using any introductory phrases such as "
+                "'based on the context provided,' 'according to the information given,' or similar expressions. "
+                "Your response must rely solely on the information from the provided context and should **not** include any additional commentary, explanations, or opinions.\n\n"
+                "📊 **Context:**\n{context}\n\n"
+                "❓ **Question:**\n{question}\n\n"
+                "**✅ Answer:**"
+            )
+        )
+        llm = ChatGroq(temperature=0, model_name=llmModel)
+        chain=custom_prompt | llm
+        result = chain.invoke({
+            "context": [gold_document],
+            "question": query
+        })
+        return result.content
+
+if __name__=="__main__":
+    gold_price_jewel_mirror = GoldPricingJewelMirror()
+    query = "What is the current gold price of bangalore?"
+    result= gold_price_jewel_mirror.response_query(query)
+    print(result)
+
+# future_use: For grading
+
+# class gradeIntent(BaseModel):
+#binary_score: str = Field(description="The intent related to gold pricing 'yes' or 'no'")
+#
+# self.intent_prompt = ChatPromptTemplate.from_template(
+#         template=(
+#             "You are an intelligent assistant responsible for evaluating whether a user's query pertains to pricing or price-related topics, "
+#             "specifically about commodities such as gold, silver, or other market prices.\n\n"
+#             "User query: {query}\n\n"
+#             "Does this query relate to gold pricing or similar price-related topics? Respond with '1' for yes and '0' for no. "
+#             "Your response must be exactly one character long, either '0' or '1'. Do not include any other text."
+#         )
+#     )
+#     self.intent_grading_chain = self.intent_prompt | self.llm

src/models/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/models/apis_models.py

@@ -0,0 +1,207 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from pydantic import BaseModel, EmailStr
+from typing import List, Optional
+
+
+## ---------------------------------- SpeechTranscription API Models ----------------------------------
+class TextToSpeechRequest(BaseModel):
+    text: str
+    lang: str
+    tld: str
+
+
+class SpeechToTextRequest(BaseModel):
+    lang: str
+
+
+## ---------------------------------- Chatbot API Models ----------------------------------
+
+class AddTextRequest(BaseModel):
+    vectorstore: str
+    text: str
+
+
+class AddWebsiteRequest(BaseModel):
+    website_urls: List[str]
+    vectorstore: str
+    source: str
+
+
+class AnswerQueryRequest(BaseModel):
+    query: str
+    vectorstore: str
+    llm_model: str = "llama3-70b-8192"
+
+
+class DataAnalyzerRequest(BaseModel):
+    query: str
+
+
+class GetLinksRequest(BaseModel):
+    url: str
+
+
+class YoutubeTranscriptRequest(BaseModel):
+    vectorstore: str
+    urls: List[str]
+
+
+class WebsiteUrlTextExtractionRequest(BaseModel):
+    url: str
+
+
+class WebsiteUrlTextExtractionListRequest(BaseModel):
+    urls: List[str]
+
+
+class GetCurrentCountRequest(BaseModel):
+    vectorstore: str
+
+
+class ListChatbotsRequest(BaseModel):
+    username: str
+
+
+class GetChatHistoryRequest(BaseModel):
+    vectorstore: str
+
+
+class ChatHistoryItem(BaseModel):
+    timestamp: str
+    question: str
+    response: str
+
+
+class DeleteChatbotRequest(BaseModel):
+    vectorstore: str
+
+
+class AddQAPairRequest(BaseModel):
+    vectorstore: str
+    question: str
+    answer: str
+
+
+class TrainChatbotRequest(BaseModel):
+    vectorstore: str
+    urls: list[str]
+
+
+class LoadPDFRequest(BaseModel):
+    vectorstore: str
+
+
+class LoadEditedJson(BaseModel):
+    vectorstore: str
+    data_source_name: str
+    source_endpoint: str
+    json_data: dict
+
+
+class PublicPrivateCheckRequest(BaseModel):
+    vectorstore: str
+    mode: str | None = None
+
+
+class DeleteChatbotSourceRequest(BaseModel):
+    vectorstore: str
+    data_source_name: str
+
+
+## ---------------------------------- User Management API Models ----------------------------------
+
+class UserSignupRequest(BaseModel):
+    username: str
+    email: EmailStr
+    password: str
+
+
+class UserSigninRequest(BaseModel):
+    email: EmailStr
+    password: str
+
+
+class CheckSessionRequest(BaseModel):
+    user_id: str
+
+
+class GetUserDataRequest(BaseModel):
+    access_token: str
+
+
+class RefreshSessionRequest(BaseModel):
+    refresh_token: str
+
+
+class LoginWithAccessTokenRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+
+
+class UsernameCreationOauthRequest(BaseModel):
+    username: str
+    user_id: str
+    email: str
+
+
+class SetSessionDataRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+    user_id: str
+
+
+class SignOutRequest(BaseModel):
+    user_id: str
+
+
+class NewChatbotRequest(BaseModel):
+    chatbot_name: str
+    username: str
+
+
+## ---------------------------------- Analytics API Models ----------------------------------
+
+
+class FeedbackRequest(BaseModel):
+    feedback: str
+    user_id: str
+    vectorstore: Optional[str]
+
+
+class UserSatisfactionRateRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class TokenUsageRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class AverageSessionInteractionRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyActiveEndUserRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyChatCountRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+##---------------------GOldPr---------------------------
+
+class GoldPriceRequest(BaseModel):
+    query:str
+    llm_model:str="llama3-70b-8192"

src/models/response_handling_models.py

@@ -0,0 +1,27 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+
+## ---------------------------------- Response Handling API Models ----------------------------------
+class SuccessResponse(BaseModel):
+    status: str
+    code: int
+    data: Dict[str, Any]
+
+
+class ErrorResponse(BaseModel):
+    status: str
+    code: int
+    error: Dict[str, Any]
+
+
+class SuccessResponseUsermanagement(BaseModel):
+    status: str
+    code: int
+    message: str
+    data: Dict[str, Any]

src/models/utls.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-29
+"""
+from pydantic import BaseModel, Field
+
+
+class FollowUps(BaseModel):
+    q1: str = Field(description="First Follow-up Question")
+    q2: str = Field(description="Second Follow-up Question")
+    q3: str = Field(description="Third Follow-up Question")

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/pipeline/conversai_analytic_pipeline.py

@@ -0,0 +1,34 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+import requests
+
+from src.api.conversai_api import supabase_client
+from src.services.supabase.analytics.analytic_tables import feedback_table, chat_history_table, add_feedback, \
+    track_usage
+
+
+class ConversAIAnalyticPipeline:
+    def __init__(self):
+        self.supabase_client = supabase_client
+
+    def feedback_table_(self, vectorstore):
+        return feedback_table(self.supabase_client, vectorstore)
+
+    def chat_history_table_(self, vectorstore):
+        return chat_history_table(self.supabase_client, vectorstore)
+
+    def add_feedback_(self, feedback, user_id, city, client_ip, vectorstore):
+        return add_feedback(self.supabase_client, feedback, user_id, city, client_ip, vectorstore)
+
+    def track_usage_(self, vectorstore: str, endpoint: str):
+        return track_usage(supabase_client=self.supabase_client, vectorstore=vectorstore, endpoint=endpoint)
+
+    def get_ip_info(self, ip: str):
+        try:
+            response = requests.get(f"https://ipinfo.io/{ip}/json")
+            data = response.json()
+            return data.get("city", "Unknown")
+        except Exception as e:
+            return "Unknown"

src/pipeline/conversai_pipeline.py

@@ -0,0 +1,73 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.prompts import PromptTemplate
+from src.services.embeddings.Qdrant_BM25_embedding import qdrant_bm25_embedding
+from src.services.document.add_document import AddDocument
+from src.services.answer_query.answerquery import AnswerQuery
+from src.services.file_analyzer.data_analyzer import DataAnalyzer
+from src.services.get_links.web_scraper import WebScraper
+# from src.services.ocr.easy_ocr.easy_ocr_ import EasyOCR_ as OCRService
+from src.services.pdf_extraction.image_pdf.image_pdf_text_extraction import get_text_from_image_pdf
+from src.services.pdf_extraction.text_pdf.text_pdf_extraction import extract_text_from_pdf
+from src.services.video_transcription.youtube_video_transcription.youtube_video_transcript import \
+    youtube_video_transcription
+from src.services.website_url.text_extraction_urls import extract_text_from_url_list, extract_text_from_url
+from src.utils.utils import json_parser
+from src.prompts.custom_prompts import  _custom_prompts
+from src.services.ocr.replicate_ocr.replicate_ocr import ReplicateOCR as OCRService
+from src.services.embeddings.jina_embeddings import jina_embedding
+
+class ConversAIPipeline:
+    def __init__(self):
+        prompt_template=_custom_prompts["RAG_ANSWER_PROMPT"]
+        follow_up_prompt_template = _custom_prompts["FOLLOW_UP_PROMPT"]
+        prompt = ChatPromptTemplate.from_template(prompt_template)
+        json_parser_ = json_parser()
+        follow_up_prompt = PromptTemplate(
+            template=follow_up_prompt_template,
+            input_variables=["context"],
+            partial_variables={"format_instructions": json_parser_.get_format_instructions()},
+        )
+        self.vector_embedding = jina_embedding()
+        self.sparse_embedding = qdrant_bm25_embedding()
+        self.add_document_service = AddDocument(self.vector_embedding, self.sparse_embedding)
+
+        self.answer_query_service = AnswerQuery(vector_embedding=self.vector_embedding,
+                                                                 sparse_embedding=self.sparse_embedding, prompt=prompt,
+                                                                          follow_up_prompt=follow_up_prompt, json_parser=json_parser_)
+        self.data_analyzer = DataAnalyzer()
+        self.get_website_links = WebScraper()
+        self.ocr_service = OCRService()
+
+    def add_document_(self, texts: list[tuple[str]], vectorstore: str):
+        return self.add_document_service.add_documents(texts=texts, vectorstore=vectorstore)
+
+    def answer_query_(self, query: str, vectorstore: str, llm_model: str = "llama-3.1-70b-versatile"):
+        output, follow_up_questions, source = self.answer_query_service.answer_query(query=query,
+                                                                                     vectorstore=vectorstore,
+                                                                                     llmModel=llm_model)
+        return output, follow_up_questions, source
+
+    def data_analyzer_(self, query: str, dataframe):
+        return self.data_analyzer.analyze_data(query=query, dataframe=dataframe)
+
+    def get_links_(self, url: str, timeout: int):
+        return self.get_website_links.get_links(url=url, timeout=timeout)
+
+    def image_pdf_text_extraction_(self, image_pdf: bytes):
+        return get_text_from_image_pdf(pdf_bytes=image_pdf)
+
+    def text_pdf_extraction_(self, pdf: str):
+        return extract_text_from_pdf(pdf_path=pdf)
+
+    def youtube_transcript_(self, url: list):
+        return youtube_video_transcription(youtube_video_url=url)
+
+    def website_url_text_extraction_(self, url: str):
+        return extract_text_from_url(url=url)
+
+    def website_url_text_extraction_list_(self, urls: list):
+        return extract_text_from_url_list(urls=urls)

src/pipeline/speech_transcription_pipeline.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from src.services.speech_to_text.speech_to_text_replicate import SpeechToTextReplicate
+from src.services.text_to_speech.text_to_speech_gtts import TextToSpeech
+
+
+class SpeechTranscriptionPipeline:
+    def __init__(self):
+        self.speech_to_text_ = SpeechToTextReplicate()
+        self.text_to_speech_ = TextToSpeech()
+
+    def text_to_speech(self, text: str, lang: str, tld: str) -> str:
+        speech = self.text_to_speech_.conversion(text, lang, tld)
+        return speech
+
+    def speech_to_text(self, audio, lang: str) -> str:
+        transcript_with_timestamp, transcript = self.speech_to_text_.transcribe_audio(audio=audio, language=lang)
+        return transcript

src/pipeline/user_management_pipeline.py

@@ -0,0 +1,80 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.supabase.user_management.user_service import UserManagement
+from src.services.supabase.user_management.chatbot_management import SupabaseChatoBotManagement
+from src.services.supabase.conversai_setup.conversai_user_db_setup import ConversAIUserDBSetup
+from src.services.supabase.user_management.chat_history import get_chat_history
+from supabase.client import create_client
+from qdrant_client import QdrantClient
+import os
+
+SUPABASE_URL = os.getenv("SUPABASE_URL")
+SUPABASE_KEY = os.getenv("SUPABASE_KEY")
+
+QDRANT_URL = os.getenv("QDRANT_URL")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+
+
+class SupabaseUserManagementPipeline:
+    def __init__(self):
+        self.supabase_client = create_client(SUPABASE_URL, SUPABASE_KEY)
+        self.qdrant_client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+        self.user_management = UserManagement(supabase_client=self.supabase_client)
+        self.chatbot_management = SupabaseChatoBotManagement(supabase_client=self.supabase_client,
+                                                             qdrant_client=self.qdrant_client)
+        self.user_db_setup = ConversAIUserDBSetup(supabase_client=self.supabase_client)
+
+    def user_signup_(self, username: str, email: str, password: str) -> dict:
+        return self.user_management.user_signup(username=username, email=email, password=password)
+
+    def user_signin_(self, email: str, password: str) -> dict:
+        return self.user_management.user_signin(email=email, password=password)
+
+    def check_session_(self) -> dict:
+        return self.user_management.check_session()
+
+    def get_user_data_(self, access_token: str) -> dict:
+        return self.user_management.get_user_data(access_token=access_token)
+
+    def refresh_session__(self, refresh_token: str) -> dict:
+        return self.user_management.refresh_session_(refresh_token=refresh_token)
+
+    def login_with_access_token_(self, access_token: str, refresh_token: str) -> dict:
+        return self.user_management.login_with_access_token(access_token=access_token, refresh_token=refresh_token)
+
+    def username_creation_oauth_(self, username: str, user_id: str, email: str):
+        return self.user_management.user_name_creation_oauth(user_id=user_id, username=username, email=email)
+
+    def set_session_data_(self, access_token: str, refresh_token: str, user_id: str):
+        return self.user_management.set_session_data(access_token=access_token, refresh_token=refresh_token,
+                                                     user_id=user_id)
+
+    def sign_out_(self):
+        return self.user_management.sign_out_()
+
+    def oauth_signin_(self) -> dict:
+        return self.user_management.oauth()
+
+    def new_chatbot_(self, chatbot_name: str, username: str):
+        return self.chatbot_management.new_chatbot(chatbot_name=chatbot_name, username=username)
+
+    def get_chat_history_(self, vectorstore: str):
+        return get_chat_history(vectorstore=vectorstore, supabase_client=self.supabase_client)
+
+    def delete_table(self, table_name: str):
+        return self.chatbot_management.delete_table(table_name=table_name)
+
+    def list_tables(self, username: str):
+        return self.chatbot_management.list_tables(username=username)
+
+    def create_data_source_name(self, source_name: str, username: str):
+        return self.chatbot_management.create_data_source_name(source_name=source_name, username=username)
+
+    def delete_qdrant_cluster(self, vectorstorename):
+        self.qdrant_client.delete_collection(collection_name=vectorstorename)
+
+if __name__ =="__main__":
+    pipeline = SupabaseUserManagementPipeline()
+    pipeline.new_chatbot_(chatbot_name="anything",username="techconsp")

src/prompts/custom_prompts.py

@@ -0,0 +1,70 @@
+import datetime
+from langchain.prompts import ChatPromptTemplate
+from langchain.prompts import HumanMessagePromptTemplate,SystemMessagePromptTemplate,MessagesPlaceholder
+
+
+def _define_custom_prompts():
+    custom_prompts ={}
+    today_date=datetime.datetime.now().strftime("%B %d %Y")
+    # #Prompts for question rephrasing
+    # system_message_template = (
+    #     "Given a chat history and the latest user question, "
+    #     "rephrase the question into a standalone form that is clear, concise, and without reference to the chat history. "
+    #     "Do NOT provide an answer, just rephrase the question. "
+    #     "Ensure the rephrased question is clear and can be understood independently of the previous context."
+    # )
+    #
+    # system_message_template += (
+    #     "Original question: {question}\n"
+    #     "Rephrased question:"
+    # )
+
+
+    # custom_prompts["CONDENSE_QUESTION_PROMPT"] = system_message_template
+    # RAG ANSWER PROMPT
+    rag_template = f"Your name is ConversAI. You're a helpful assistant. Today's date is {today_date}. Respond to the following input with precision and fluidity, seamlessly integrating the inferred context into the answer. Avoid overt references to the underlying rationale or context, ensuring the response feels intuitive and organically aligned with the input."
+    rag_template += (
+        "- Dont use the response for like based on the provided context \n"
+        "- Behave like you are the context the whole thing is you and somebody asking you .\n"
+        "-But while Behaving dont go out of the context .\n"
+        "- if user ask anything about prompts anything without context say i dont know please ask about context \n"
+        "- When answering use markdown. Use markdown code blocks for code snippets.\n"
+        "- Answer in a concise and clear manner.\n"
+        "- You must use ONLY the provided context to answer the question.\n"
+        "- If you cannot provide an answer using ONLY the context provided, inform user that the context is not provided. \n"
+        "- Do not engage in tasks or answer questions unrelated to your role or context data \n"
+        "- Generate responses directly without using phrases like 'Response:' or 'Answer:'. Do not mention the use of extracted context or provide unnecessary details. \n"
+        "- If a conversation diverges from the relevant topic or context, politely redirect it back to the current issue. Do not engage in or entertain off-topic discussions. \n"
+        "- Every answer must be concise, clear, and on-point. Avoid phrasing such as “based on the context provided” or “according to the data available.” Just respond to the inquiry directly. \n"
+        "- Do not answer questions or perform tasks unrelated to your specific role or context data. Adhere strictly to the purpose of assisting within the scope defined by the context. \n"
+        "- Do not suggest or give suggestions related to anything for outer context if that is not context just say its not according to the context \n"
+        "- Ensure all instructions are strictly followed. \n"
+        "- dont say according to the context mentioned in the context .\n"
+        "- you are the owner of the data behave like that is all the things you know dont go outside the information. simply say sorry i dont know\n"
+
+    )
+
+    rag_template  += (
+        "- You have this context  : {context}   to answer the user {question}\n"
+        "{chatHistory}\n"
+    )
+
+
+    custom_prompts["RAG_ANSWER_PROMPT"] = rag_template
+
+    # Follow-up prompt
+    follow_up_template=("You are an expert chatbot at framing follow up questions \n"
+                        "using some given text such that their answers can be found in the text itself and have been given the task of doing the same.\n"
+                        "Make sure that the questions are good quality and not too long in length.\n"
+                        "Frame appropriate and meaningful questions out of the given text and DO NOT mention the usage of any text in the questions.\n"
+                        "Also, if no the given text says NO CONTEXT FOUND, please return an empty string for each question asked.\n"
+                        "{format_instructions}\n"
+                        "{context}\n"
+    )
+
+    custom_prompts["FOLLOW_UP_PROMPT"]=follow_up_template
+
+
+    return custom_prompts
+
+_custom_prompts =_define_custom_prompts()

src/services/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/services/answer_query/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/answer_query/answerquery.py

@@ -0,0 +1,98 @@
+
+from langchain_core.runnables import RunnablePassthrough, RunnableLambda
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from langchain.memory import ChatMessageHistory
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain_community.document_compressors import JinaRerank
+from langchain_core.chat_history import BaseChatMessageHistory
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection, \
+    answer_query_from_existing_collection
+from langchain_groq import ChatGroq
+import os
+os.environ["JINA_API_KEY"] = os.getenv("JINA_API")
+
+
+class AnswerQuery:
+    def __init__(self, prompt, vector_embedding, sparse_embedding, follow_up_prompt, json_parser):
+        self.chat_history_store = {}
+        self.compressor = JinaRerank(model="jina-reranker-v2-base-multilingual")
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+        self.prompt = prompt
+        self.follow_up_prompt = follow_up_prompt
+        self.json_parser = json_parser
+
+    def format_docs(self, docs: str):
+        global sources
+        global temp_context
+        sources = []
+        context = ""
+        for doc in docs:
+            context += f"{doc.page_content}\n\n\n"
+            source = doc.metadata
+            source = source["source"]
+            sources.append(source)
+        if context == "":
+            context = "No context found"
+        else:
+            pass
+        sources = list(set(sources))
+        temp_context = context
+        return context
+
+
+
+    def answer_query(self, query: str, vectorstore: str, llmModel: str = "llama-3.1-70b-versatile"):
+        global sources
+        global temp_context
+        vector_store_name = vectorstore
+        vector_store = answer_query_from_existing_collection(vector_embed=self.vector_embed,
+                                                             sparse_embed=self.sparse_embed,
+                                                             vectorstore=vectorstore)
+
+        retriever = vector_store.as_retriever(search_type="mmr", search_kwargs={"k": 10, "fetch_k": 20})
+        compression_retriever = ContextualCompressionRetriever(
+            base_compressor = self.compressor, base_retriever = retriever
+        )
+        brain_chain = (
+                {"context": RunnableLambda(lambda x: x["question"]) | compression_retriever | RunnableLambda(self.format_docs),
+                 "question": RunnableLambda(lambda x: x["question"]),
+                 "chatHistory": RunnableLambda(lambda x: x["chatHistory"])}
+                | self.prompt
+                | ChatGroq(model=llmModel, temperature=0.75, max_tokens=512)
+                | StrOutputParser()
+        )
+        message_chain = RunnableWithMessageHistory(
+            brain_chain,
+            self.get_session_history,
+            input_messages_key="question",
+            history_messages_key="chatHistory"
+        )
+        chain = RunnablePassthrough.assign(messages_trimmed=self.trim_messages) | message_chain
+        follow_up_chain = self.follow_up_prompt | ChatGroq(model_name="llama-3.1-70b-versatile",
+                                                           temperature=0) | self.json_parser
+
+        output = chain.invoke(
+            {"question": query},
+            {"configurable": {"session_id": vector_store_name}}
+        )
+        follow_up_questions = follow_up_chain.invoke({"context": temp_context})
+
+        return output, follow_up_questions, sources
+
+    def trim_messages(self, chain_input):
+        for store_name in self.chat_history_store:
+            messages = self.chat_history_store[store_name].messages
+            if len(messages) <= 1:
+                pass
+            else:
+                self.chat_history_store[store_name].clear()
+                for message in messages[-1:]:
+                    self.chat_history_store[store_name].add_message(message)
+        return True
+
+    def get_session_history(self, session_id: str) -> BaseChatMessageHistory:
+        if session_id not in self.chat_history_store:
+            self.chat_history_store[session_id] = ChatMessageHistory()
+        return self.chat_history_store[session_id]

src/services/document/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/document/add_document.py

@@ -0,0 +1,30 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import string
+from uuid import uuid4
+from langchain.docstore.document import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection
+
+
+class AddDocument:
+    def __init__(self, vector_embedding, sparse_embedding):
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+
+    def add_documents(self, texts: list[tuple[str]], vectorstore: str):
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=400,
+            chunk_overlap=100,
+            add_start_index=True
+        )
+        sources = [textTuple[1] for textTuple in texts]
+        texts = [textTuple[0].replace("\n", " ") for textTuple in texts]
+        texts = [text.translate(str.maketrans('', '', string.punctuation.replace(".", ""))) for text in texts]
+        texts = [Document(page_content=text, metadata={"source": source}) for text, source in zip(texts, sources)]
+        documents = splitter.split_documents(texts)
+        upload_document_existing_collection(vector_embed=self.vector_embed,
+                                            sparse_embed=self.sparse_embed,
+                                            vectorstore=vectorstore, documents=documents)

src/services/embeddings/BGE-M3_vector_embedding.py

@@ -0,0 +1,13 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_huggingface import HuggingFaceEmbeddings
+
+
+def bge_m3_vector_embedding():
+    return HuggingFaceEmbeddings(
+        model_name="BAAI/bge-m3",
+        model_kwargs={"device": "cuda"},
+        encode_kwargs={"normalize_embeddings": True}
+    )

src/services/embeddings/Qdrant_BM25_embedding.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_qdrant import FastEmbedSparse
+
+
+def qdrant_bm25_embedding():
+    instance = FastEmbedSparse(model="Qdrant/BM25", threads=20, parallel=0)
+
+    return instance

src/services/embeddings/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/embeddings/jina_embeddings.py

@@ -0,0 +1,8 @@
+from langchain_community.embeddings import JinaEmbeddings
+import os
+
+def jina_embedding():
+    text_embeddings=JinaEmbeddings(
+    jina_api_key=os.getenv('JINA_API'),
+    model_name="jina-embeddings-v3")
+    return text_embeddings

src/services/embeddings/sentence_transformers_all_MiniLM_L6_v2_vector_embedding.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_huggingface import HuggingFaceEmbeddings
+from src.utils.utils import load_ini_config
+
+
+def all_minilm_l6_v2_vector_embedding():
+    config = load_ini_config("config.ini")
+
+    model_kwargs = {"device": config.get('all_mini_l6_v2_vector_embedding', 'device')}
+    encode_kwargs = {"normalize_embeddings": config.get('all_mini_l6_v2_vector_embedding', 'normalize_embeddings')}
+
+    all_minilm_object = HuggingFaceEmbeddings(
+        model_name="BAAI/bge-base-en-v1.5",
+        model_kwargs=model_kwargs,
+        encode_kwargs=encode_kwargs
+    )
+    return all_minilm_object

src/services/file_analyzer/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/file_analyzer/data_analyzer.py

@@ -0,0 +1,28 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import os
+import base64
+from langchain_groq import ChatGroq
+from pandasai import SmartDataframe
+from src.utils.utils import load_ini_config
+
+
+class DataAnalyzer:
+    def __init__(self):
+        self.config = load_ini_config("config.ini")
+
+        self.llm_config = ChatGroq(name=self.config.get('data_analyzer', 'groq_llm_name'))
+        self.additional_query = self.config.get('data_analyzer', 'additional_query')
+
+    def analyze_data(self, query, dataframe):
+        query += self.additional_query
+        df = SmartDataframe(dataframe,config={"llm": self.llm_config, "verbose": bool(self.config.get("data_analyzer", "verbose"))})
+        response = df.chat(query)
+        if os.path.isfile(response):
+            with open(response, "rb") as file:
+                b64string = base64.b64encode(file.read()).decode("utf-8")
+            return f"data:image/png;base64,{b64string}"
+        else:
+            return response

src/services/get_links/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/get_links/web_scraper.py

@@ -0,0 +1,49 @@
+import time
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+from concurrent.futures import ThreadPoolExecutor
+
+
+
+class WebScraper:
+    def __init__(self):
+        pass
+
+    def get_links(self,url: str, timeout=4):
+        start = time.time()
+
+        def get_links_from_page(url: str) -> list:
+            response = requests.get(url)
+            soup = BeautifulSoup(response.content, "lxml")
+            anchors = soup.find_all("a")
+            links = []
+            for anchor in anchors:
+                if "href" in anchor.attrs:
+                    if urlparse(anchor.attrs["href"]).netloc == urlparse(url).netloc:
+                        links.append(anchor.attrs["href"])
+                    elif not anchor.attrs["href"].startswith(("//", "file", "javascript", "tel", "mailto", "http")):
+                        links.append(urljoin(url + "/", anchor.attrs["href"]))
+                    else:
+                        pass
+                    links = [link for link in links if "#" not in link]
+                    links = list(set(links))
+                else:
+                    continue
+            return links
+
+        links = get_links_from_page(url)
+        unique_links = set()
+        for link in links:
+            now = time.time()
+            if now - start > timeout:
+                break
+            else:
+                unique_links = unique_links.union(set(get_links_from_page(link)))
+        return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in unique_links]))
+
+if __name__ == "__main__":
+    scraper = WebScraper()
+    links=scraper.get_links("https://www.additudemag.com/")
+    print(len(links))
+    print(type(links))

src/services/ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/ocr/easy_ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/ocr/easy_ocr/easy_ocr_.py

@@ -0,0 +1,19 @@
+# """
+# Created By: ishwor subedi
+# Date: 2024-08-23
+# """
+# import cv2
+# import easyocr
+# import numpy as np
+# from src.utils.utils import load_ini_config
+#
+#
+# class EasyOCR_:
+#     def __init__(self):
+#         self.config = load_ini_config("config.ini")
+#         self.reader = easyocr.Reader([self.config.get('easy_ocr', 'language')],
+#                                      gpu=bool(self.config.get('easy_ocr', 'gpu')),
+#                                      model_storage_directory=self.config.get('easy_ocr', 'model_path'))
+#
+#     def read_text(self, image: np.array):
+#         return self.reader.readtext(image, paragraph=True)