Improve the code bot development

.gitignore

@@ -397,4 +397,5 @@ FodyWeavers.xsd
 # JetBrains Rider
 *.sln.iml
 
-.env
+.env
+*.pem

alembic.ini

@@ -0,0 +1,117 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = placeholder_url
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# content and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,86 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+from dotenv import load_dotenv
+from alembic import context
+from db.models import Base
+
+import os
+
+load_dotenv()
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+config.set_main_option(
+    "sqlalchemy.url", os.getenv("DB_URI_SQL_ALCHEMY")
+)
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/404f8a028e0e_add_bot_name.py

@@ -0,0 +1,30 @@
+"""add bot_name
+
+Revision ID: 404f8a028e0e
+Revises: 98b1b4a0de39
+Create Date: 2024-10-04 14:03:44.098762
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '404f8a028e0e'
+down_revision: Union[str, None] = '98b1b4a0de39'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('bot', sa.Column('bot_name', sa.String(length=200), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('bot', 'bot_name')
+    # ### end Alembic commands ###

alembic/versions/426e52aa13aa_migration_description.py

@@ -0,0 +1,91 @@
+"""migration description
+
+Revision ID: 426e52aa13aa
+Revises: 
+Create Date: 2024-10-02 14:32:47.859996
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = '426e52aa13aa'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('Category')
+    op.drop_table('Role')
+    op.drop_table('Metadata')
+    op.drop_table('_prisma_migrations')
+    op.drop_index('email', table_name='User')
+    op.drop_table('User')
+    op.add_column('bot', sa.Column('user_id', sa.Integer(), nullable=True))
+    op.add_column('metadata', sa.Column('title', sa.String(length=100), nullable=True))
+    op.create_foreign_key(None, 'bot', 'user', ['user_id'], ['id'])
+    op.alter_column('session', 'id',
+               existing_type=mysql.CHAR(length=36),
+               type_=sa.String(length=36),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('session', 'id',
+               existing_type=sa.String(length=36),
+               type_=mysql.CHAR(length=36),
+               existing_nullable=False)
+    op.drop_constraint(None, 'bot', type_='foreignkey')
+    op.drop_column('bot', 'user_id')
+    op.create_table('User',
+    sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('name', mysql.VARCHAR(length=50), nullable=False),
+    sa.Column('email', mysql.VARCHAR(length=100), nullable=False),
+    sa.Column('password_hash', mysql.VARCHAR(length=100), nullable=False),
+    sa.Column('created_at', mysql.DATETIME(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', mysql.DATETIME(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('email', 'User', ['email'], unique=True)
+    op.create_table('_prisma_migrations',
+    sa.Column('id', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=36), nullable=False),
+    sa.Column('checksum', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=64), nullable=False),
+    sa.Column('finished_at', mysql.DATETIME(fsp=3), nullable=True),
+    sa.Column('migration_name', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=255), nullable=False),
+    sa.Column('logs', mysql.TEXT(collation='utf8mb4_unicode_ci'), nullable=True),
+    sa.Column('rolled_back_at', mysql.DATETIME(fsp=3), nullable=True),
+    sa.Column('started_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False),
+    sa.Column('applied_steps_count', mysql.INTEGER(unsigned=True), server_default=sa.text("'0'"), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('Metadata',
+    sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('title', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=191), nullable=False),
+    sa.Column('category', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=191), nullable=False),
+    sa.Column('author', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=191), nullable=False),
+    sa.Column('year', mysql.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('publisher', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=191), nullable=False),
+    sa.Column('createdAt', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False),
+    sa.Column('updatedAt', mysql.DATETIME(fsp=3), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('Role',
+    sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('role_name', mysql.VARCHAR(length=100), nullable=False),
+    sa.Column('description', mysql.VARCHAR(length=100), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('Category',
+    sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('category', mysql.VARCHAR(length=100), nullable=True),
+    sa.Column('created_at', mysql.DATETIME(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###

alembic/versions/4dd226fee84e_add_updated_data.py

@@ -0,0 +1,30 @@
+"""add updated data
+
+Revision ID: 4dd226fee84e
+Revises: 6bd7ce57dca0
+Create Date: 2024-10-07 14:44:03.718963
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '4dd226fee84e'
+down_revision: Union[str, None] = '6bd7ce57dca0'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('category', sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('category', 'updated_at')
+    # ### end Alembic commands ###

alembic/versions/6bd7ce57dca0_add_updated_at.py

@@ -0,0 +1,30 @@
+"""add updated at
+
+Revision ID: 6bd7ce57dca0
+Revises: 404f8a028e0e
+Create Date: 2024-10-07 14:02:25.722428
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6bd7ce57dca0'
+down_revision: Union[str, None] = '404f8a028e0e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('category', sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('category', 'updated_at')
+    # ### end Alembic commands ###

alembic/versions/6e972ae8b93b_add_updated_at_session.py

@@ -0,0 +1,32 @@
+"""add updated at session
+
+Revision ID: 6e972ae8b93b
+Revises: 82908d30ae5a
+Create Date: 2024-10-10 14:10:14.433336
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6e972ae8b93b'
+down_revision: Union[str, None] = '82908d30ae5a'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('session', sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    op.add_column('session_publisher', sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('session_publisher', 'updated_at')
+    op.drop_column('session', 'updated_at')
+    # ### end Alembic commands ###

alembic/versions/82908d30ae5a_add_updated_data.py

@@ -0,0 +1,50 @@
+"""add updated data
+
+Revision ID: 82908d30ae5a
+Revises: fcc0580a4769
+Create Date: 2024-10-09 14:12:01.147393
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = '82908d30ae5a'
+down_revision: Union[str, None] = 'fcc0580a4769'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('session_publisher',
+    sa.Column('id', sa.String(length=36), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('metadata_id', sa.Integer(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.ForeignKeyConstraint(['metadata_id'], ['metadata.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_session_publisher_id'), 'session_publisher', ['id'], unique=False)
+    op.add_column('metadata', sa.Column('thumbnail', sa.LargeBinary(), nullable=True))
+    op.add_column('planning', sa.Column('trials_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'planning', 'trials', ['trials_id'], ['id'])
+    op.drop_column('planning', 'token_planned')
+    op.drop_column('planning', 'token_used')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('planning', sa.Column('token_used', mysql.INTEGER(), autoincrement=False, nullable=False))
+    op.add_column('planning', sa.Column('token_planned', mysql.INTEGER(), autoincrement=False, nullable=False))
+    op.drop_constraint(None, 'planning', type_='foreignkey')
+    op.drop_column('planning', 'trials_id')
+    op.drop_column('metadata', 'thumbnail')
+    op.drop_index(op.f('ix_session_publisher_id'), table_name='session_publisher')
+    op.drop_table('session_publisher')
+    # ### end Alembic commands ###

alembic/versions/98b1b4a0de39_add_username.py

@@ -0,0 +1,32 @@
+"""add username
+
+Revision ID: 98b1b4a0de39
+Revises: c818e5b84075
+Create Date: 2024-10-03 14:27:43.877725
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '98b1b4a0de39'
+down_revision: Union[str, None] = 'c818e5b84075'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('user', sa.Column('username', sa.String(length=100), nullable=False))
+    op.create_unique_constraint(None, 'user', ['username'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'user', type_='unique')
+    op.drop_column('user', 'username')
+    # ### end Alembic commands ###

alembic/versions/b1df0377cbe2_change_time_zome.py

@@ -0,0 +1,30 @@
+"""change time zome
+
+Revision ID: b1df0377cbe2
+Revises: c0e9d62caae8
+Create Date: 2024-10-10 14:44:44.626184
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'b1df0377cbe2'
+down_revision: Union[str, None] = 'c0e9d62caae8'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/c0e9d62caae8_change_time_zome.py

@@ -0,0 +1,30 @@
+"""change time zome
+
+Revision ID: c0e9d62caae8
+Revises: 6e972ae8b93b
+Create Date: 2024-10-10 14:37:41.728523
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c0e9d62caae8'
+down_revision: Union[str, None] = '6e972ae8b93b'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/c818e5b84075_add_role_id.py

@@ -0,0 +1,108 @@
+"""add role id
+
+Revision ID: c818e5b84075
+Revises: 426e52aa13aa
+Create Date: 2024-10-03 09:35:53.882054
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = 'c818e5b84075'
+down_revision: Union[str, None] = '426e52aa13aa'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('_prisma_migrations')
+    op.alter_column('category', 'category',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=True)
+    op.alter_column('feedback', 'comment',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=1000),
+               existing_nullable=True)
+    op.alter_column('message', 'goal',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=True)
+    op.alter_column('metadata', 'title',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=True)
+    op.alter_column('metadata', 'author',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=True)
+    op.alter_column('role', 'role_name',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=False)
+    op.alter_column('role', 'description',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               type_=sa.String(length=200),
+               existing_nullable=True)
+    op.add_column('user', sa.Column('role_id', sa.Integer(), nullable=True))
+    op.alter_column('user', 'name',
+               existing_type=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=50),
+               type_=sa.String(length=100),
+               existing_nullable=False)
+    op.create_foreign_key(None, 'user', 'role', ['role_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'user', type_='foreignkey')
+    op.alter_column('user', 'name',
+               existing_type=sa.String(length=100),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=50),
+               existing_nullable=False)
+    op.drop_column('user', 'role_id')
+    op.alter_column('role', 'description',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.alter_column('role', 'role_name',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=False)
+    op.alter_column('metadata', 'author',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.alter_column('metadata', 'title',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.alter_column('message', 'goal',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.alter_column('feedback', 'comment',
+               existing_type=sa.String(length=1000),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.alter_column('category', 'category',
+               existing_type=sa.String(length=200),
+               type_=mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=100),
+               existing_nullable=True)
+    op.create_table('_prisma_migrations',
+    sa.Column('id', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=36), nullable=False),
+    sa.Column('checksum', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=64), nullable=False),
+    sa.Column('finished_at', mysql.DATETIME(fsp=3), nullable=True),
+    sa.Column('migration_name', mysql.VARCHAR(collation='utf8mb4_unicode_ci', length=255), nullable=False),
+    sa.Column('logs', mysql.TEXT(collation='utf8mb4_unicode_ci'), nullable=True),
+    sa.Column('rolled_back_at', mysql.DATETIME(fsp=3), nullable=True),
+    sa.Column('started_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False),
+    sa.Column('applied_steps_count', mysql.INTEGER(unsigned=True), server_default=sa.text("'0'"), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###

alembic/versions/fcc0580a4769_add_updated_model_planning.py

@@ -0,0 +1,48 @@
+"""add updated model planning
+
+Revision ID: fcc0580a4769
+Revises: 4dd226fee84e
+Create Date: 2024-10-08 10:01:41.964473
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'fcc0580a4769'
+down_revision: Union[str, None] = '4dd226fee84e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('bot_meta', sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    op.add_column('bot_meta', sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    op.add_column('planning', sa.Column('planning_name', sa.String(length=200), nullable=False))
+    op.add_column('planning', sa.Column('duration', sa.Integer(), nullable=False))
+    op.add_column('planning', sa.Column('token_used', sa.Integer(), nullable=False))
+    op.add_column('planning', sa.Column('token_planned', sa.Integer(), nullable=False))
+    op.add_column('planning', sa.Column('start_date', sa.DateTime(), nullable=False))
+    op.add_column('planning', sa.Column('end_date', sa.DateTime(), nullable=False))
+    op.add_column('planning', sa.Column('is_activated', sa.Boolean(), nullable=True))
+    op.add_column('planning', sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('planning', 'created_at')
+    op.drop_column('planning', 'is_activated')
+    op.drop_column('planning', 'end_date')
+    op.drop_column('planning', 'start_date')
+    op.drop_column('planning', 'token_planned')
+    op.drop_column('planning', 'token_used')
+    op.drop_column('planning', 'duration')
+    op.drop_column('planning', 'planning_name')
+    op.drop_column('bot_meta', 'updated_at')
+    op.drop_column('bot_meta', 'created_at')
+    # ### end Alembic commands ###

api/auth.py

@@ -0,0 +1,71 @@
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+from fastapi.security import OAuth2PasswordBearer
+from dotenv import load_dotenv
+
+from sqlalchemy.orm import Session
+from db.models import User
+from starlette import status
+from datetime import timedelta, datetime, timezone
+from db.database import get_db
+from passlib.context import CryptContext
+from typing import Annotated
+from jose import jwt, JWTError
+import os
+
+load_dotenv()
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="login")
+
+# Custom OAuth2 request form to accept email, username, password, and role_id
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+SECRET_KEY = os.getenv("SECRET_KEY")
+ALGORITHM = "HS256"
+
+bcrypt_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+# Database dependency
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+def authenticate_user(email: str, password: str, db):
+    user = db.query(User).filter(User.email == email).first()
+    if not user:
+        return False
+
+    if not bcrypt_context.verify(password, user.hashed_password):
+        return False
+    return user
+
+
+def create_access_token(
+    username: str, name: str, user_id: int, role_id: int, expires_delta: timedelta, email: str
+):
+    encode = {"sub": username, "name":name, "id": user_id, "role_id": role_id, "email": email}
+    expires = datetime.now(timezone.utc) + expires_delta
+    encode.update({"exp": expires})
+    return jwt.encode(encode, SECRET_KEY, algorithm=ALGORITHM)
+
+
+async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]):
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        username: str = payload.get("sub")
+        name: str = payload.get("name")
+        user_id: int = payload.get("id")
+        role_id: int = payload.get("role_id")
+        email: str = payload.get("email")
+
+        if username is None or user_id is None:
+            return JSONResponse(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                content="Could not validate user.",
+            )
+
+        return {"username": username, "name" : name, "id": user_id, "role_id": role_id, "email": email}
+
+    except JWTError:
+        return JSONResponse(
+            status_code=status.HTTP_401_UNAUTHORIZED, content="Could not validate user."
+        )

api/events.py

@@ -1,14 +1,18 @@
 from fastapi import FastAPI
-from api.router.topic import db_conn
+from db.models import Base
+from db.database import engine
+from api.router.book import db_conn
 from llama_index.core import set_global_handler
-import os
 from dotenv import load_dotenv
+import os
 
 
 load_dotenv()
 
 
 async def startup() -> None:
+    Base.metadata.create_all(engine)
+    print("table added")
     await db_conn.connect()
     os.environ["LANGFUSE_SECRET_KEY"] = os.getenv("LANGFUSE_SECRET_KEY")
     os.environ["LANGFUSE_PUBLIC_KEY"] = os.getenv("LANGFUSE_PUBLIC_KEY")
@@ -19,7 +23,6 @@ async def startup() -> None:
 async def shutdown() -> None:
     await db_conn.disconnect()
 
-
 def register_events(app: FastAPI) -> FastAPI:
     app.add_event_handler("startup", startup)
     app.add_event_handler("shutdown", shutdown)

api/function.py

@@ -1,75 +1,69 @@
 from script.vector_db import IndexManager
 from script.document_uploader import Uploader
-from db.save_data import InsertDatabase
 from db.get_data import GetDatabase
 from db.delete_data import DeleteDatabase
 from db.update_data import UpdateDatabase
 
-from typing import Any, Optional, List
+from typing import Any
 from fastapi import UploadFile
 from fastapi import HTTPException
+from fastapi.responses import JSONResponse
 
-from service.dto import ChatMessage
+from llama_index.core.llms import MessageRole
 from core.chat.engine import Engine
 from core.chat.chatstore import ChatStore
-from core.parser import clean_text, update_response, renumber_sources, seperate_to_list
-from llama_index.core.llms import MessageRole
-from service.dto import BotResponseStreaming
+from core.parser import clean_text, update_response, renumber_sources
+from service.dto import BotResponseStreaming, ChatMessage
 from service.aws_loader import Loader
 
+from pymongo.mongo_client import MongoClient
+from dotenv import load_dotenv
+
+from typing import List
+from datetime import datetime
+import redis
 import logging
 import re
 import json
+import os
 
+load_dotenv()
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
 
-# async def data_ingestion(
-#     db_conn, reference, file: UploadFile, content_table: UploadFile
-# ) -> Any:
 
-
-async def data_ingestion(db_conn, reference, file: UploadFile) -> Any:
+async def data_ingestion(category_id, reference, file: UploadFile) -> Any:
     try:
-
-        # insert_database = InsertDatabase(db_conn)
-
+        # Upload to AWS
         file_name = f"{reference['title']}"
         aws_loader = Loader()
 
         file_obj = file
         aws_loader.upload_to_s3(file_obj, file_name)
 
-        print("Uploaded Success")
-        
-        response = json.dumps({"status": "success", "message": "Vector Index loaded successfully."})
-
-        # Insert data into the database
-        # await insert_database.insert_data(reference)
-
-        # # uploader = Uploader(reference, file, content_table)
-        # uploader = Uploader(reference, file)
-        # print("uploader : ", uploader)
+        uploader = Uploader(reference, file)
 
-        # nodes_with_metadata = await uploader.process_documents()
+        nodes_with_metadata = await uploader.process_documents()
 
-        # # Build indexes using IndexManager
-        # index = IndexManager()
-        # response = index.build_indexes(nodes_with_metadata)
+        # Build indexes using IndexManager
+        index = IndexManager()
+        index.build_indexes(nodes_with_metadata)
 
-        return response
+        return json.dumps(
+            {"status": "success", "message": "Vector Index loaded successfully."}
+        )
 
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI
-        logging.error(f"An error occurred in data ingestion: {e}")
-        raise HTTPException(
+        logging.error("An error occurred in data ingestion: %s", e)
+        return JSONResponse(
             status_code=500,
-            detail="An internal server error occurred in data ingestion.",
+            content="An internal server error occurred in data ingestion.",
         )
 
 
-async def get_data(db_conn, title="", fetch_all_data=True):
+async def get_data(db_conn, title=None, fetch_all_data=True):
     get_database = GetDatabase(db_conn)
     print(get_database)
     try:
@@ -85,9 +79,9 @@ async def get_data(db_conn, title="", fetch_all_data=True):
 
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI
-        logging.error(f"An error occurred in get data.: {e}")
-        raise HTTPException(
-            status_code=500, detail="An internal server error occurred in get data."
+        logging.error("An error occurred in get data: %s", e)
+        return JSONResponse(
+            status_code=500, content="An internal server error occurred in get data."
         )
 
 
@@ -103,9 +97,9 @@ async def update_data(id: int, reference, db_conn):
         return response
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI
-        logging.error(f"An error occurred in update data.: {e}")
-        raise HTTPException(
-            status_code=500, detail="An internal server error occurred in update data."
+        logging.error("An error occurred in update data: %s", e)
+        return JSONResponse(
+            status_code=500, content="An internal server error occurred in update data."
         )
 
 
@@ -118,43 +112,48 @@ async def delete_data(id: int, db_conn):
         return response
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI
-        logging.error(f"An error occurred in get data.: {e}")
-        raise HTTPException(
-            status_code=500, detail="An internal server error occurred in delete data."
+        logging.error("An error occurred in get data: %s", e)
+        return JSONResponse(
+            status_code=500, content="An internal server error occurred in delete data."
         )
 
 
 def generate_completion_non_streaming(
-    session_id, user_request, chat_engine, title=None, category=None, type="general"
+    session_id, user_request, titles: List = None, type_bot="general"
 ):
+    uri = os.getenv("MONGO_URI")
+    engine = Engine()
+    index_manager = IndexManager()
+    chatstore = ChatStore()
+    client = MongoClient(uri)
+
+    try:
+        client.admin.command("ping")
+        print("Pinged your deployment. You successfully connected to MongoDB!")
+    except Exception as e:
+        return JSONResponse(status_code=500, content=f"Database Error as {e}")
+
     try:
-        engine = Engine()
-        index_manager = IndexManager()
-        chatstore = ChatStore()
 
         # Load existing indexes
         index = index_manager.load_existing_indexes()
 
-        if type == "general":
+        if type_bot == "general":
             # Retrieve the chat engine with the loaded index
             chat_engine = engine.get_chat_engine(session_id, index)
         else:
             # Retrieve the chat engine with the loaded index
-            chat_engine = engine.get_chat_engine(
-                session_id, index, title=title, category=category
-            )
+            chat_engine = engine.get_chat_engine(session_id, index, titles, type_bot)
 
         # Generate completion response
         response = chat_engine.chat(user_request)
 
         sources = response.sources
-        print(sources)
 
         number_reference = list(set(re.findall(r"\[(\d+)\]", str(response))))
         number_reference_sorted = sorted(number_reference)
 
         contents = []
-        raw_contents = []
         metadata_collection = []
         scores = []
 
@@ -170,9 +169,6 @@ def generate_completion_non_streaming(
                     # Pastikan number valid sebagai indeks
                     if 0 <= number - 1 < len(node):
 
-                        raw_content = seperate_to_list(node[number - 1].node.get_text())
-                        raw_contents.append(raw_content)
-
                         content = clean_text(node[number - 1].node.get_text())
                         contents.append(content)
 
@@ -206,18 +202,47 @@ def generate_completion_non_streaming(
         chatstore.delete_last_message(session_id)
         chatstore.add_message(session_id, message)
         chatstore.clean_message(session_id)
-
-        return str(response), raw_contents, contents, metadata_collection, scores
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI
-        logging.error(f"An error occurred in generate text: {e}")
-        raise HTTPException(
+        logging.error("An error occurred in generate text: %s", e)
+        return JSONResponse(
             status_code=500,
-            detail="An internal server error occurred in generate text.",
-        )
+            content=f"An internal server error occurred in generate text as {e}.")
+
+    try :
+        chat_history_db = [
+                ChatMessage(role=MessageRole.SYSTEM, 
+                            content=user_request, 
+                            timestamp=datetime.now(),
+                            payment = "free" if type_bot=="general" else None
+                            ),
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=response,
+                    metadata=metadata_collection,
+                    timestamp=datetime.now(),
+                    payment = "free" if type_bot=="general" else None 
+                )
+            ]
+        
+        chat_history_json = [message.model_dump() for message in chat_history_db]
+
+        db = client["bot_database"]  # Replace with your database name
+        collection = db[session_id]  # Replace with your collection name
+        
+        result = collection.insert_many(chat_history_json)
+        print("Data inserted with record ids", result.inserted_ids)
 
+        return str(response), metadata_collection, scores
+    
+    except Exception as e:
+        # Log the error and raise HTTPException for FastAPI
+        logging.error("An error occurred in generate text: %s", e)
+        return JSONResponse(
+            status_code=500,
+            content=f"An internal server error occurred in generate text as {e}.")
 
-async def generate_streaming_completion(user_request, chat_engine):
+async def generate_streaming_completion(user_request, session_id):
     try:
         engine = Engine()
         index_manager = IndexManager()
@@ -226,7 +251,7 @@ async def generate_streaming_completion(user_request, chat_engine):
         index = index_manager.load_existing_indexes()
 
         # Retrieve the chat engine with the loaded index
-        chat_engine = engine.get_chat_engine(index)
+        chat_engine = engine.get_chat_engine(index, session_id)
         # Generate completion response
         response = chat_engine.stream_chat(user_request)
 
@@ -258,4 +283,4 @@ async def generate_streaming_completion(user_request, chat_engine):
         raise HTTPException(
             status_code=500,
             detail="An internal server error occurred in generate text.",
-        )
+        ) from e

api/router/book.py

@@ -0,0 +1,278 @@
+import logging
+import base64
+
+from typing import Annotated, Optional
+from api.function import data_ingestion, get_data, delete_data, update_data
+from api.router.user import user_dependency
+from fastapi import Form, APIRouter, File, UploadFile, Depends
+from fastapi.responses import JSONResponse
+from db.repository import get_db_conn
+from db.get_data import GetDatabase
+from db.models import Category, Metadata
+from db.database import get_db
+from langfuse.llama_index import LlamaIndexCallbackHandler
+from config import MYSQL_CONFIG
+from script.vector_db import IndexManager
+from service.dto import MetadataRequest, MetadataResponse
+from sqlalchemy.orm import Session
+from sqlalchemy.future import select
+from sqlalchemy.exc import SQLAlchemyError
+
+
+router = APIRouter(tags=["Book"])
+
+db_conn = get_db_conn(MYSQL_CONFIG)
+get_database = GetDatabase(db_conn)
+index_manager = IndexManager()
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+@router.post("/book")
+async def upload_file(
+    user: user_dependency,
+    db: db_dependency,
+    title: str = Form(...),
+    author: str = Form(...),
+    category_id: int = Form(...),
+    year: int = Form(...),
+    publisher: str = Form(...),
+    file: UploadFile = File(...),
+    thumbnail: Optional[UploadFile] = File(None),
+):
+    print(user.get("role_id"))
+    # if user is None or user.get('role_id') != 1:
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    # Create a new Metadata object
+    new_book = Metadata(
+        title=title,
+        author=author,
+        category_id=category_id,
+        year=year,
+        publisher=publisher,
+    )
+
+    db.add(new_book)
+    db.commit()
+    logging.info("Database Inserted")
+
+    try:
+        # Query the category based on category_id
+        category_query = select(Category.category).where(Category.id == category_id)
+        result = db.execute(category_query)
+        category = result.scalar_one_or_none()
+
+        # Check if the category exists
+        if category is None:
+            return JSONResponse(status_code=404, content="Category not found")
+
+    except SQLAlchemyError as db_exc:
+        # Handle any database-related errors (e.g., connection issues, query issues)
+        print(f"Database error: {db_exc}")
+        return JSONResponse(status_code=500, content="Database error occurred")
+
+    except Exception as e:
+        # Catch any other general exceptions
+        print(f"Error: {e}")
+        return JSONResponse(
+            status_code=500, content="An error occurred while processing your request"
+        )
+
+    try:
+        # Assuming you have a Langfuse callback handler
+        langfuse_callback_handler = LlamaIndexCallbackHandler()
+        langfuse_callback_handler.set_trace_params(
+            user_id="admin_book_uploaded",
+        )
+
+        # Construct the reference dictionary
+        reference = {
+            "title": title,
+            "author": author,
+            "category": category,
+            "year": year,
+            "publisher": publisher,
+        }
+
+        # Process the file and handle data ingestion
+        response = await data_ingestion(category_id, reference, file)
+
+    except Exception as e:
+        # Handle any errors related to file processing or data ingestion
+        print(f"File processing error: {e}")
+        return JSONResponse(status_code=500, content="File processing error")
+
+    # Return a successful response with the uploaded filename and response from data ingestion
+    return {
+        "filename": file.filename,
+        "response": response,
+        "info": "upload file successfully",
+    }
+
+
+@router.get("/book")
+async def get_metadata(user: user_dependency, db: db_dependency):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+    try:
+        # Join Metadata with Category to get the category name
+        results = (
+            db.query(
+                Metadata.id,
+                Metadata.title,
+                Metadata.author,
+                Category.category,  # Assuming this is the correct field for category name
+                Category.id,
+                Metadata.year,
+                Metadata.publisher,
+                Metadata.thumbnail,
+            )
+            .join(Category, Metadata.category_id == Category.id)
+            .all()
+        )
+
+        # Transform results into MetadataResponse model with optional thumbnail handling
+        return [
+            MetadataResponse(
+                id = id,
+                title=title,
+                author=author,
+                category=category,
+                category_id = category_id,
+                year=year,
+                publisher=publisher,
+                thumbnail=(
+                    thumbnail if thumbnail else None
+                ),  # Ensure None if thumbnail is not present
+            )
+            for id, title, author, category, category_id, year, publisher, thumbnail in results
+        ]
+    except SQLAlchemyError as db_exc:
+        print(f"Database error: {db_exc}")
+        return JSONResponse(status_code=500, content="Database error occurred")
+
+
+@router.put("/book/{metadata_id}")
+async def update_metadata(
+    user: user_dependency,
+    db: db_dependency,
+    metadata_id: int,
+    title: str = Form(...),
+    author: str = Form(...),
+    category_id: int = Form(...),
+    year: int = Form(...),
+    publisher: str = Form(...),
+    thumbnail: Optional[UploadFile] = File(None),
+):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # old_metadata = await get_database.get_data_by_id(metadata_id)
+
+        old_metadata = db.execute(
+            select(Metadata).where(Metadata.id == metadata_id)
+        ).scalar_one_or_none()
+
+        if old_metadata is None:
+            return JSONResponse(status_code=404, content="Metadata not found.")
+
+        # Fetch old and new categories
+        old_category = db.execute(
+            select(Category.category).join(Metadata).where(Metadata.id == metadata_id)
+        ).scalar_one_or_none()
+
+        new_category = db.execute(
+            select(Category.category).where(Category.id == category_id)
+        ).scalar_one_or_none()
+
+        if old_category is None or new_category is None:
+            return JSONResponse(status_code=404, content="Category not found.")
+
+        # Prepare the references
+        old_reference = {
+            "title": old_metadata.title,
+            "author": old_metadata.author,
+            "category": old_category,
+            "year": old_metadata.year,
+            "publisher": old_metadata.publisher,
+        }
+
+        new_reference = {
+            "title": title,
+            "author": author,
+            "category": new_category,
+            "year": year,
+            "publisher": publisher,
+        }
+
+        index_manager.update_vector_database(old_reference, new_reference)
+
+        # Update existing metadata entry
+        metadata = db.query(Metadata).filter(Metadata.id == metadata_id).first()
+
+        if not metadata:
+            return JSONResponse(status_code=404, content="Metadata not found")
+
+        if thumbnail is not None:
+            thumbnail_data = await thumbnail.read()  # Read the file data
+            metadata.thumbnail = base64.b64encode(thumbnail_data).decode("utf-8")
+
+        # Update fields only if provided in the request
+        metadata.title = title
+        metadata.author = author
+        metadata.category_id = category_id
+        metadata.year = year
+        metadata.publisher = publisher
+
+        # Commit the changes to the database
+        db.commit()
+        db.refresh(metadata)  # Refresh to get the updated data
+
+        category_query = select(Category.category).where(
+            Category.id == metadata.category_id
+        )
+        result = db.execute(category_query)
+        category = result.scalar_one_or_none()
+
+        return MetadataResponse(
+            id=metadata_id,
+            title=metadata.title,
+            author=metadata.author,
+            category=category,  # Assuming category relationship is available
+            category_id=metadata.category_id,
+            year=metadata.year,
+            publisher=metadata.publisher,
+            thumbnail=metadata.thumbnail if metadata.thumbnail is not None else None,
+        )
+
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An error occurred while updating metadata as {e}"
+        )
+
+
+@router.delete("/book/{metadata_id}")
+async def delete_metadata(user: user_dependency, db: db_dependency, metadata_id: int):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        old_reference = await get_database.get_data_by_id(metadata_id)
+        
+        old_metadata = db.execute(
+            select(Metadata).where(Metadata.id == metadata_id)
+        ).scalar_one_or_none()
+        
+        index_manager.delete_vector_database(old_reference)
+
+        await delete_data(metadata_id, db_conn)
+
+        return {"Status": "delete successfully"}
+
+    except Exception as e:
+        print(e)
+        return JSONResponse(
+            status_code=500, content="An error occurred while delete metadata"
+        )

api/router/book_collection.py

@@ -0,0 +1,188 @@
+from typing import Annotated, List, Optional
+from api.router.user import user_dependency
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+from db.models import User_Meta, Metadata, Category
+from db.database import get_db
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+
+router = APIRouter(tags=["Book_Collection"])
+
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+@router.get("/book_collection")
+async def get_book_collection(
+    user: user_dependency,
+    db: db_dependency,
+):
+    """This function will return a BookCollection"""
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Fetch all User_Meta entries for the user and their associated Metadata
+        user_meta_entries = (
+            db.query(User_Meta, Metadata, Category)  # Select User_Meta, Metadata, and Category
+            .join(Metadata, Metadata.id == User_Meta.metadata_id)  # Join on metadata_id
+            .join(Category, Category.id == Metadata.category_id)  # Join on category_id
+            .filter(User_Meta.user_id == user.get("id"))
+            .all()
+        )
+
+        if not user_meta_entries:
+            return {"info": "No book collection found"}
+
+        # Extract relevant data from the user_meta_entries
+        results = [
+            {
+                "user_id": user_meta.user_id,
+                "metadata_id": metadata.id,  # Access Metadata fields
+                "title": metadata.title,  # Replace with actual field names in Metadata
+                "author": metadata.author,  # Replace with actual field names in Metadata
+                "category_name": category.category,  # Replace with actual field names in Category
+                "year": metadata.year,
+                "publisher": metadata.publisher
+                # Add other Metadata and Category fields as needed
+            }
+            for user_meta, metadata, category in user_meta_entries  # Unpack the tuple
+        ]
+
+        return {
+            "status": "success",
+            "book_collection": results,  # Return the list of metadata
+        }
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+
+@router.post("/book_collection")
+async def request_book_collection(
+    user: user_dependency,
+    db: db_dependency,
+    metadata_id: List[Optional[int]],
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Create User_Meta instances for each metadata_id
+        user_meta_entries = [
+            User_Meta(user_id=user.get("id"), metadata_id=mid) for mid in metadata_id
+        ]
+
+        # Insert all entries into the database
+        db.add_all(user_meta_entries)
+        db.commit()  # Commit the transaction
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    return {"status": "success", "user_meta": [entry.id for entry in user_meta_entries]}
+
+
+@router.put("/book_collection")
+async def update_book_collection(
+    user: user_dependency,
+    db: db_dependency,
+    metadata_id: List[Optional[int]],  # Use the Pydantic model
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Fetch and delete existing User_Meta entries
+        db.query(User_Meta).filter(User_Meta.user_id == user.get("id")).delete(
+            synchronize_session=False
+        )
+        db.commit()
+
+        # Insert new User_Meta entries
+        user_meta_entries = [
+            User_Meta(user_id=user.get("id"), metadata_id=mid) for mid in metadata_id
+        ]
+        db.add_all(user_meta_entries)
+        db.commit()
+    except SQLAlchemyError as e:
+        db.rollback()
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(status_code=500, content=f"Unexpected error: {str(e)}")
+
+    return {"status": "success", "user_meta": [entry.id for entry in user_meta_entries]}
+
+
+@router.delete("/book_collection/{metadata_id}")
+async def delete_book_collection(
+    user: user_dependency,
+    db: db_dependency,
+    metadata_id: int
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Query to delete the entry based on metadata_id for the authenticated user
+        entry_to_delete = db.query(User_Meta).filter(
+            User_Meta.user_id == user.get("id"),
+            User_Meta.metadata_id == metadata_id
+        ).first()
+
+        if not entry_to_delete:
+            return JSONResponse(status_code=404, content="Entry not found to delete.")
+
+        # Delete the entry
+        db.delete(entry_to_delete)
+        db.commit()
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Rollback in case of any database error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    return {
+        "status": "success",
+        "deleted_entry": entry_to_delete.id,
+    }
+
+
+@router.delete("/all_collections")
+async def delete_all_book(
+    user: user_dependency,
+    db: db_dependency,
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        db.query(User_Meta).filter(User_Meta.user_id == user.get("id")).delete(
+            synchronize_session="fetch"
+        )
+        
+        db.commit()  # Commit all deletions
+
+        return {
+            "status": "success",
+            "delete book collection from": user.get("id"),
+        }
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Rollback in case of any database error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )

api/router/bot_general.py

@@ -0,0 +1,94 @@
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+from service.dto import UserPromptRequest, BotResponse
+from core.chat.chatstore import ChatStore
+from db.database import get_db
+from db.models import Session_Publisher
+from sqlalchemy.orm import Session
+from api.function import (
+    generate_streaming_completion,
+    generate_completion_non_streaming,
+)
+from api.router.user import user_dependency
+from sse_starlette.sse import EventSourceResponse
+from utils.utils import generate_uuid
+from typing import Annotated
+from langfuse.llama_index import LlamaIndexCallbackHandler
+
+
+router = APIRouter(tags=["Bot_General"])
+
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+def get_chat_store():
+    return ChatStore()
+
+
+@router.post("/bot_general/new")
+async def create_session_general():
+    session_id = generate_uuid()
+    return {"session_id": session_id}
+
+
+@router.get("/bot/{session_id}")
+async def get_session_id(
+    user: user_dependency,
+    session_id: str,
+    chat_store: ChatStore = Depends(get_chat_store),
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    chat_history = chat_store.get_all_messages_mongodb(session_id)
+    # if not chat_history:
+    #     return JSONResponse(status_code=404, content="Session not found or empty.")
+
+    return chat_history
+
+
+@router.post("/bot/{session_id}")
+async def bot_generator_general(
+    session_id: str, user_prompt_request: UserPromptRequest
+):
+
+    langfuse_callback_handler = LlamaIndexCallbackHandler()
+    langfuse_callback_handler.set_trace_params(user_id="guest", session_id=session_id)
+
+    if user_prompt_request.streaming:
+        return EventSourceResponse(
+            generate_streaming_completion(
+                user_prompt_request.prompt,
+                session_id
+            )
+        )
+    else:
+        response, metadata, scores = generate_completion_non_streaming(
+            session_id,
+            user_prompt_request.prompt,
+        )
+
+        return BotResponse(
+            content=response,
+            metadata=metadata,
+            scores=scores,
+        )
+
+
+@router.delete("/bot/{session_id}")
+async def delete_bot(db:db_dependency, session_id: str, chat_store: ChatStore = Depends(get_chat_store)):
+    try:
+        chat_store.delete_messages(session_id)
+        
+        # Delete session from database
+        session = db.query(Session_Publisher).filter(Session_Publisher.id == session_id).first()
+        if session:
+            db.delete(session)
+            db.commit()  # Commit the transaction
+        else:
+            return JSONResponse(status_code=404, content="Session not found")
+        return {"info": f"Delete {session_id} successful"}
+    except Exception as e:
+        # Log the error and return JSONResponse for FastAPI
+        print(f"An error occurred in update data.: {e}")
+        return JSONResponse(status_code=400, content="the error when deleting message")

api/router/bot_one.py

@@ -0,0 +1,151 @@
+from typing import Annotated
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+from service.dto import UserPromptRequest, BotResponse
+from core.chat.chatstore import ChatStore
+from db.database import get_db
+from db.models import Metadata, Session_Publisher
+from db.models import Session as SessionModel
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import select
+from api.function import (
+    generate_streaming_completion,
+    generate_completion_non_streaming,
+)
+from datetime import datetime
+from api.router.user import user_dependency
+from sse_starlette.sse import EventSourceResponse
+from utils.utils import generate_uuid
+from langfuse.llama_index import LlamaIndexCallbackHandler
+
+
+router = APIRouter(tags=["Bot_One"])
+
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+def get_chat_store():
+    return ChatStore()
+
+
+@router.post("/bot_one/{metadata_id}")
+async def create_bot_one(user: user_dependency, db: db_dependency, metadata_id: int):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+    # Generate a new session ID (UUID)
+    try:
+        session_id = generate_uuid()
+
+        # Create the new session
+        new_session = Session_Publisher(
+            id=session_id,
+            user_id=user.get("id"),
+            metadata_id=metadata_id,
+        )
+
+        db.add(new_session)
+        db.commit()  # Commit the new session to the database
+
+        return {
+            "statur": "session id created successfully",
+            "session_id": session_id,
+        }
+
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected  in retrieving session id {str(e)}"
+        )
+
+
+@router.post("/bot_one/{metadata_id}/{session_id}")
+async def generator_bot_one(
+    user: user_dependency,
+    db: db_dependency,
+    metadata_id: int,
+    session_id: str,
+    user_prompt_request: UserPromptRequest,
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    langfuse_callback_handler = LlamaIndexCallbackHandler()
+    langfuse_callback_handler.set_trace_params(
+        user_id=user.get("username"), session_id=session_id
+    )
+
+    # Query to retrieve the titles
+    try:
+        query = (
+            select(Metadata.title)
+            .join(Session_Publisher, Metadata.id == metadata_id)
+            .where(
+                Session_Publisher.user_id == user.get("id"),
+                Session_Publisher.id == session_id,
+            )
+        )
+
+        result = db.execute(query)
+        titles = result.scalars().all()
+        print(titles)
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    if user_prompt_request.streaming:
+        return EventSourceResponse(
+            generate_streaming_completion(
+                user_prompt_request.prompt,
+                session_id,
+            )
+        )
+    else:
+        response, metadata, scores = generate_completion_non_streaming(
+            session_id, user_prompt_request.prompt, titles, type_bot="specific"
+        )
+
+        existing_session = (
+            db.query(Session_Publisher).filter(Session_Publisher.id == session_id).first()
+        )
+        existing_session.updated_at = datetime.now()
+        db.commit()
+        
+        return BotResponse(
+            content=response,
+            metadata=metadata,
+            scores=scores,
+        )
+
+
+@router.get("/bot_one{metadata_id}")
+async def get_all_session_bot_one(
+    user: user_dependency, db: db_dependency, metadata_id: int
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Query the session IDs based on the user ID
+        query = select(Session_Publisher.id, Session_Publisher.updated_at).where(
+            Session_Publisher.user_id == user.get("id"),
+            Session_Publisher.metadata_id == metadata_id,
+        )
+
+        result = db.execute(query)
+        sessions = result.all()
+        
+        session_data = [{"id": session.id, "updated_at": str(session.updated_at)} for session in sessions]
+
+        # Convert list of tuples to a simple list
+        session_sorted_data = sorted(session_data, key=lambda x: datetime.fromisoformat(x['updated_at']), reverse=True)
+
+        return session_sorted_data
+
+    except Exception as e:
+        # Log the error and return JSONResponse for FastAPI
+        print(f"An error occurred while fetching session IDs: {e}")
+        return JSONResponse(status_code=400, content="Error retrieving session IDs")

api/router/bot_specific.py

@@ -0,0 +1,342 @@
+from typing import Annotated, List, Optional
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+from service.dto import UserPromptRequest, BotResponse, BotCreateRequest
+from core.chat.chatstore import ChatStore
+from db.database import get_db
+from db.models import Bot_Meta, Bot, Metadata
+from db.models import Session as SessionModel
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.exc import NoResultFound
+from sqlalchemy import select
+from api.function import (
+    generate_streaming_completion,
+    generate_completion_non_streaming,
+)
+from api.router.user import user_dependency
+from sse_starlette.sse import EventSourceResponse
+from utils.utils import generate_uuid
+from langfuse.llama_index import LlamaIndexCallbackHandler
+from datetime import datetime
+
+
+router = APIRouter(tags=["Bot_Specific"])
+
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+def get_chat_store():
+    return ChatStore()
+
+
+@router.post("/bot")
+async def create_bot_id(
+    user: user_dependency,
+    db: db_dependency,
+    bot_request: BotCreateRequest,
+):
+
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    # Create a new bot entry
+    try:
+        # Create a new bot entry
+        new_bot = Bot(
+            user_id=user.get("id"), bot_name=bot_request.bot_name
+        )  # Assuming user has an 'id' attribute
+
+        db.add(new_bot)
+        db.commit()  # Commit the transaction
+        db.refresh(new_bot)  # Optional: Refresh the instance with the database state
+
+        return {"status": "success", "bot_id": new_bot.id}
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Roll back the transaction in case of an error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+
+@router.post("/meta/{bot_id}")
+async def create_bot_specific(
+    user: user_dependency,
+    db: db_dependency,
+    bot_id: int,
+    metadata_id: List[Optional[int]],  # Use the Pydantic model
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Create BotMeta instances for each metadata_id
+        bot_meta_entries = [
+            Bot_Meta(bot_id=bot_id, metadata_id=mid) for mid in metadata_id
+        ]
+
+        # Insert all entries into the database
+        db.add_all(bot_meta_entries)
+        db.commit()  # Commit the transaction
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    return {"status": "success", "bot_meta": [entry.id for entry in bot_meta_entries]}
+
+
+@router.put("/meta/{bot_id}")
+async def update_bot_specific(
+    user: user_dependency,
+    db: db_dependency,
+    bot_id: int,
+    metadata_id: List[Optional[int]],  # Use the Pydantic model
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Fetch existing Bot_Meta entries related to bot_id
+        existing_entries = db.query(Bot_Meta).filter(Bot_Meta.bot_id == bot_id).all()
+
+        # Delete existing entries
+        for entry in existing_entries:
+            db.delete(entry)
+
+        # Commit the deletion
+        db.commit()
+
+        # Insert the new metadata entries
+        bot_meta_entries = [
+            Bot_Meta(bot_id=bot_id, metadata_id=mid) for mid in metadata_id
+        ]
+        db.add_all(bot_meta_entries)
+        db.commit()
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Rollback in case of any database error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    return {"status": "success", "bot_meta": [entry.id for entry in bot_meta_entries]}
+
+
+@router.delete("/meta/{bot_id}/{metadata_id}")
+async def delete_bot_specific(
+    user: user_dependency,
+    db: db_dependency,
+    bot_id: int,
+    metadata_id: int,  # Changed to int to specify a single metadata_id
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Delete the specific metadata entry for the given bot_id
+        bot_meta_entry = (
+            db.query(Bot_Meta)
+            .filter(Bot_Meta.bot_id == bot_id, Bot_Meta.metadata_id == metadata_id)
+            .first()  # Use first() to get a single entry
+        )
+
+        if not bot_meta_entry:
+            return JSONResponse(status_code=404, content="No entry found to delete.")
+
+        # Delete the found entry
+        db.delete(bot_meta_entry)
+        db.commit()
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Rollback in case of any database error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    return {
+        "status": "success",
+        "deleted_entry_id": bot_meta_entry.id,
+    }
+
+
+@router.delete("/bot_all/{bot_id}")
+async def delete_bot_id(
+    user: user_dependency,
+    db: db_dependency,
+    bot_id: int,
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Fetch the bot to ensure it exists
+        bot_entry = db.query(Bot).filter(Bot.id == bot_id).first()
+        print("bot entry", bot_entry)
+
+        if not bot_entry:
+            return JSONResponse(
+                status_code=404, content=f"Bot with id {bot_id} not found."
+            )
+
+        db.query(SessionModel).filter(SessionModel.bot_id == bot_id).delete(
+            synchronize_session="fetch"
+        )
+        db.query(Bot_Meta).filter(Bot_Meta.bot_id == bot_id).delete(
+            synchronize_session="fetch"
+        )
+        db.delete(bot_entry)
+        db.commit()  # Commit all deletions
+
+        return {
+            "status": "success",
+            "deleted_bot_id": bot_id,
+        }
+
+    except SQLAlchemyError as e:
+        db.rollback()  # Rollback in case of any database error
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+
+@router.post("/session/{bot_id}/new")
+async def create_new_session(user: user_dependency, db: db_dependency, bot_id: int):
+    # Check if user is authenticated
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    print(user.get("id"))
+    user_id = user.get("id")
+    # Ensure the bot belongs to the user
+    bot_query = select(Bot).where(Bot.id == bot_id, Bot.user_id == user_id)
+
+    try:
+        bot = db.execute(bot_query).scalar_one()
+        print(bot)
+
+    except NoResultFound:
+        return JSONResponse(
+            status_code=404, content="Bot not found or unauthorized access."
+        )
+
+    # Generate a new session ID (UUID)
+    try:
+        session_id = generate_uuid()
+
+        # Create the new session
+        new_session = SessionModel(
+            id=session_id,
+            user_id=user.get("id"),
+            bot_id=bot_id,
+        )
+
+        db.add(new_session)
+        db.commit()  # Commit the new session to the database
+
+        return {
+            "session_id": session_id,
+        }
+
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected  in retrieving session id {str(e)}"
+        )
+
+
+@router.get("/bot/all/{bot_id}")
+async def get_all_session_ids(user: user_dependency, db: db_dependency, bot_id: int):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        query = select(SessionModel.id, SessionModel.updated_at).where(
+            SessionModel.user_id == user.get("id"),
+            SessionModel.bot_id == bot_id
+        )
+
+        result = db.execute(query)
+        sessions = result.all()
+        session_data = [{"id": session.id, "updated_at": session.updated_at} for session in sessions]
+
+        return session_data
+
+    except Exception as e:
+        # Log the error and return JSONResponse for FastAPI
+        print(f"An error occurred while fetching session IDs: {e}")
+        return JSONResponse(status_code=400, content="Error retrieving session IDs")
+
+
+@router.post("/bot/{bot_id}/{session_id}")
+async def bot_generator_spesific(
+    user: user_dependency,
+    db: db_dependency,
+    bot_id: int,
+    session_id: str,
+    user_prompt_request: UserPromptRequest,
+):
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    langfuse_callback_handler = LlamaIndexCallbackHandler()
+    langfuse_callback_handler.set_trace_params(
+        user_id=user.get("username"), session_id=session_id
+    )
+
+    # Query to retrieve the titles
+    try:
+        query = (
+            select(Metadata.title)
+            .join(Bot_Meta, Metadata.id == Bot_Meta.metadata_id)
+            .join(SessionModel, Bot_Meta.bot_id == bot_id)
+            .where(
+                SessionModel.user_id == user.get("id"), SessionModel.id == session_id
+            )
+        )
+
+        result = db.execute(query)
+        titles = result.scalars().all()
+        print(titles)
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        )
+
+    if user_prompt_request.streaming:
+        return EventSourceResponse(
+            generate_streaming_completion(
+                user_prompt_request.prompt,
+                session_id,
+            )
+        )
+    else:
+        response, metadata, scores = generate_completion_non_streaming(
+            session_id, user_prompt_request.prompt, titles, type_bot="specific"
+        )
+        
+        existing_session = (
+            db.query(SessionModel).filter(SessionModel.id == session_id).first()
+        )
+        existing_session.updated_at = datetime.now()
+        db.commit()
+
+        return BotResponse(
+            content=response,
+            metadata=metadata,
+            scores=scores,
+        )

api/router/category.py

@@ -0,0 +1,158 @@
+from api.router.user import user_dependency
+from fastapi.responses import JSONResponse
+from fastapi import APIRouter, HTTPException, Depends, Query
+
+from db.models import Category
+from db.database import get_db
+from service.dto import CategoryCreate
+
+from script.vector_db import IndexManager
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError
+
+from typing import Annotated
+
+router = APIRouter(tags=["Category"])
+
+index_manager = IndexManager()
+db_dependency = Annotated[Session, Depends(get_db)]
+
+
+@router.get("/category")
+async def get_all_categories(user: user_dependency, db: db_dependency):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Logic to retrieve all categories
+        categories = db.query(Category).all()
+        if not categories:
+            return JSONResponse(status_code=404, content="No categories found")
+
+        return {
+            "message": "Categories retrieved successfully",
+            "categories": [
+                {"id": cat.id, "category": cat.category} for cat in categories
+            ],
+        }
+
+    except SQLAlchemyError as e:
+        return JSONResponse(
+            status_code=500, content="Database error occurred: " + str(e)
+        )
+
+
+@router.get("/category/{category_id}")
+async def get_categories_by_ids(
+    user: user_dependency,
+    db: db_dependency,
+    category_id: int,
+):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Fetch categories based on the list of provided category_ids
+        category = db.query(Category).filter(Category.id == category_id).first()
+
+        if category is None:
+            return JSONResponse(status_code=404, content="No categories found for the given IDs")
+
+        return {
+            "message": "Categories retrieved successfully",
+            "category": {"id": category.id, "category": category.category},
+        }
+
+    except SQLAlchemyError as e:
+        return JSONResponse(
+            status_code=500, content="Database error occurred: " + str(e)
+        )
+
+@router.post("/category")
+async def create_category(user: user_dependency, db: db_dependency, category: CategoryCreate):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Check if category already exists
+        existing_category = (
+            db.query(Category).filter(Category.category == category.category_name).first()
+        )
+        if existing_category:
+            return JSONResponse(status_code=400, content="Category already exists")
+
+        # Logic to create a new category
+        new_category = Category(category=category)  # Assuming Category is your model
+        db.add(new_category)
+        db.commit()
+        db.refresh(new_category)
+
+        return {
+            "message": "Category created successfully",
+            "category_id": new_category.id,
+        }
+
+    except IntegrityError:
+        db.rollback()
+        return JSONResponse(
+            status_code=400,
+            content="Database integrity error: possibly a duplicate entry.",
+        )
+
+    except SQLAlchemyError as e:
+        db.rollback()
+        return JSONResponse(
+            status_code=500, content="Database error occurred: " + str(e)
+        )
+
+
+@router.put("/category/{category_id}")
+async def update_category(
+    user: user_dependency, db: db_dependency, category_id: int, category: CategoryCreate
+):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Logic to update an existing category
+        existing_category = (
+            db.query(Category).filter(Category.id == category_id).first()
+        )
+        if not existing_category:
+            return JSONResponse(status_code=404, content="Category not found")
+
+        existing_category.category = category.category_name
+        db.commit()
+
+        return {"message": "Category updated successfully"}
+
+    except SQLAlchemyError as e:
+        db.rollback()
+        return JSONResponse(
+            status_code=500, content="Database error occurred: " + str(e)
+        )
+
+
+@router.delete("/category/{category_id}")
+async def delete_category(user: user_dependency, db: db_dependency, category_id: int):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Logic to delete an existing category
+        existing_category = (
+            db.query(Category).filter(Category.id == category_id).first()
+        )
+        if not existing_category:
+            return JSONResponse(status_code=404, content="Category not found")
+
+        db.delete(existing_category)
+        db.commit()
+
+        return {"message": "Category deleted successfully"}
+
+    except SQLAlchemyError as e:
+        db.rollback()
+        return JSONResponse(
+            status_code=500, content="Database error occurred: " + str(e)
+        )

api/router/role.py

@@ -1,20 +1,108 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException, Depends
+from fastapi.responses import JSONResponse
+from api.router.user import user_dependency
+from typing import Annotated
+from sqlalchemy.orm import Session
+from db.database import get_db
+from db.models import Role, User
+from service.dto import RoleCreate, RoleUpdate
 
 
 router = APIRouter(tags=["Roles"])
+db_dependency = Annotated[Session, Depends(get_db)]
+
 
 @router.get("/roles")
-async def get_data_roles():
-    pass
+async def get_data_roles(user: user_dependency, db: db_dependency):
+    # Periksa apakah user valid dan memiliki role_id = 1
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    try:
+        # Query data role
+        roles = db.query(Role).all()
+        
+        # Jika tidak ada role ditemukan, kembalikan 404
+        if not roles:
+            return JSONResponse(status_code=404, content="No roles found")
+        
+        return roles
 
+    except Exception as e:
+        # Menangkap kesalahan untuk debug
+        print(f"Error fetching roles: {str(e)}")
+        return JSONResponse(status_code=500, content="Internal Server Error")
 
+# POST: Add a new role
 @router.post("/roles")
-async def add_data_roles():
-    pass
-@router.put("/roles/{id}")
-async def update_data_roles():
-    pass
+async def add_data_roles(
+    role_data: RoleCreate, user: user_dependency, db: db_dependency
+):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    new_role = Role(name=role_data.name)
+    db.add(new_role)
+    db.commit()
+    db.refresh(new_role)
+
+    return {"message": "Role added successfully", "role": new_role}
+
+
+@router.put("/roles/{role_id}")
+async def update_data_roles(
+    role_id: int, role_data: RoleUpdate, user: user_dependency, db: db_dependency
+):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    role = db.query(Role).filter(Role.id == id).first()
+    if role is None:
+        return JSONResponse(status_code=404, content="Role not found")
+
+    role.name = role_data.name
+    db.commit()
+    db.refresh(role)
+
+    return {"message": "Role updated successfully", "role": role}
+
 
+# DELETE: Remove a role
 @router.delete("/roles/{id}")
-async def remove_data_roles():
-    pass
+async def remove_data_roles(id: int, user: user_dependency, db: db_dependency):
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    role = db.query(Role).filter(Role.id == id).first()
+    if role is None:
+        return JSONResponse(status_code=404, content="Role not found")
+
+    db.delete(role)
+    db.commit()
+
+    return {"message": "Role removed successfully"}
+
+
+@router.put("/role_user/{id}")
+async def update_user_role(
+    user: user_dependency, db: db_dependency, user_id: int, role_data: RoleUpdate
+):
+    # Check if the current user is authenticated and has an admin role (role_id == 1)
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+
+    # Fetch the user to be updated
+    user_to_update = db.query(User).filter(User.id == user_id).first()
+    if user_to_update is None:
+        return JSONResponse(status_code=404, content="User not found")
+
+    # Update the user's role
+    user_to_update.role_id = (
+        role_data.role_id
+    )  # Assuming role_data contains the new role_id
+
+    # Commit the changes to the database
+    db.commit()
+    db.refresh(user_to_update)
+
+    return {"message": "User role updated successfully", "user": user_to_update}

api/router/trial.py

@@ -3,18 +3,17 @@ from fastapi import APIRouter
 
 router = APIRouter(tags=["Trial"])
 
-@router.get("/roles")
+@router.get("/trials")
 async def get_trial_data():
     pass
 
-
-@router.post("/roles")
+@router.post("/trials")
 async def add_trial_data():
     pass
-@router.put("/roles/{id}")
+@router.put("/trials/{id}")
 async def update_trial_data():
     pass
 
-@router.delete("/roles/{id}")
+@router.delete("/trials/{id}")
 async def remove_trial_data():
     pass

api/router/user.py

@@ -1,20 +1,153 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends, status
+from fastapi.security import OAuth2PasswordRequestForm
+from fastapi.responses import JSONResponse
+from db.models import User
+from db.database import get_db
+from api.auth import get_current_user, create_access_token
+from service.dto import CreateUserRequest, UserVerification, Token
+from typing import Annotated
+from passlib.context import CryptContext
+from sqlalchemy.orm import Session
+from datetime import timedelta
 
 
 router = APIRouter(tags=["User"])
 
-@router.post("/login")
-async def get_data_roles():
-    pass
+bcrypt_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+db_dependency = Annotated[Session, Depends(get_db)]
+user_dependency = Annotated[dict, Depends(get_current_user)]
+
+ACCESS_TOKEN_EXPIRE_MINUTES = 43200
+
+
+@router.post("/login", response_model=Token)
+async def login_for_access_token(
+    login_data: Annotated[OAuth2PasswordRequestForm, Depends()],
+    db: Session = Depends(get_db),
+):
+    user = db.query(User).filter(User.username == login_data.username).first()
+
+    if not user or not bcrypt_context.verify(login_data.password, user.password_hash):
+        return JSONResponse(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            content="Incorrect username or password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    try:
+        access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
+        access_token = create_access_token(
+            user.username, user.name, user.id, user.role_id, access_token_expires, user.email
+        )
+
+        return {"access_token": access_token, "token_type": "bearer"}
+
+    except Exception as e:
+        print(e)
+        return JSONResponse(
+            status_code=500, content="An error occuring when login"
+        )
+
+
+@router.get("/login", response_model=dict)
+async def get_user(user: user_dependency):
+    if user is None:
+        return JSONResponse(
+            status_code=401, content="Authentication Failed"
+        )
+    return {
+        "username": user.get("username"),
+        "name" : user.get("name"),
+        "id": user.get("id"),
+        "email": user.get("email"),
+        "role": user.get("role_id"),
+    }
+
+
+@router.get("/users", response_model=list[dict])
+async def get_all_users(user: user_dependency, db: Session = Depends(get_db)):
+    # Check if the current user has an admin role
+    if user.get("role_id") != 1:  # Adjust this check based on how roles are represented
+        return JSONResponse(
+            status_code=401, content="Authentication Failed"
+        )
+
+    # Query the database to retrieve all users
+    users = db.query(
+        User
+    ).all()  # Assuming you have a User model with an SQLAlchemy session
+    return [
+        {
+            "id": user.id,
+            "username": user.username,
+            "name": user.name,
+            "email": user.email,
+            "role": user.role_id,
+        }
+        for user in users
+    ]
 
 
 @router.post("/register")
-async def register_user():
-    pass
+async def register_user(db: db_dependency, create_user_request: CreateUserRequest):
+    existing_user = (
+        db.query(User).filter(User.email == create_user_request.email).first()
+    )
+
+    if existing_user:
+        return JSONResponse(
+            status_code=400, content="Email is already registered"
+        )
+
+    try:
+        password_hash = bcrypt_context.hash(create_user_request.password)
+
+        create_user_model = User(
+            name=create_user_request.name,
+            username=create_user_request.username,
+            email=create_user_request.email,
+            role_id=create_user_request.role_id,
+            password_hash=password_hash,
+        )
+
+        db.add(create_user_model)
+        db.commit()
+        db.refresh(create_user_model)
+
+        return {"message": "User created successfully", "user_id": create_user_model.id}
+    except Exception as e:
+        print(e)
+        return JSONResponse(
+            status_code=500, content="An error occuring when register user"
+        )
+
+
 @router.post("/forgot_password")
 async def forget_password():
     pass
 
+
 @router.post("/change_password")
-async def change_password():
-    pass
+async def change_password(
+    user: user_dependency, db: db_dependency, user_verification: UserVerification
+):
+    if user is None:
+        return JSONResponse(
+            status_code=401, content="Authentication Failed"
+        )
+    user_model = db.query(User).filter(User.id == user.get("id")).first()
+
+    if not bcrypt_context.verify(
+        user_verification.password, user_model.hashed_password
+    ):
+        return JSONResponse(
+            status_code=401, content="Error on password change"
+        )
+
+    user_model.hashed_password = bcrypt_context.hash(user_verification.new_password)
+    db.add(user_model)
+    db.commit()
+    db.refresh(user_model)
+
+    return {"message": "User's password successfully changed", "user_id": user_model.id}

app.py

@@ -1,13 +1,25 @@
 from fastapi.applications import FastAPI
-from api.router import health, topic, user, bot, trial, role
+from api.router import (
+    health,
+    user,
+    trial,
+    role,
+    book,
+    book_collection,
+    category,
+    bot_general,
+    bot_specific,
+    bot_one
+)
 from fastapi.middleware.cors import CORSMiddleware
 from api.events import register_events
 from utils.utils import pipe
-import uvicorn
+
 
 def create_instance() -> FastAPI:
     return FastAPI()
 
+
 def add_middleware(app: FastAPI) -> FastAPI:
     app.add_middleware(
         CORSMiddleware,
@@ -18,15 +30,20 @@ def add_middleware(app: FastAPI) -> FastAPI:
     )
     return app
 
+
 def init_database(app: FastAPI) -> FastAPI:
     return app
 
 
 def register_routers(app: FastAPI) -> FastAPI:
     app.include_router(user.router)
-    app.include_router(topic.router)
-    app.include_router(bot.router)
-    app.include_router(trial.router)
+    app.include_router(category.router)
+    app.include_router(book.router)
+    app.include_router(book_collection.router)
+    app.include_router(bot_general.router)
+    app.include_router(bot_specific.router)
+    app.include_router(bot_one.router)
+    app.include_router(trial.router) 
     app.include_router(role.router)
     app.include_router(health.router)
 
@@ -35,8 +52,13 @@ def register_routers(app: FastAPI) -> FastAPI:
 
 def init_app() -> FastAPI:
     app: FastAPI = pipe(
-        create_instance(), add_middleware, init_database, register_events, register_routers
+        create_instance(),
+        add_middleware,
+        init_database,
+        register_events,
+        register_routers,
     )
     return app
 
+
 app = init_app()

config.py

@@ -9,6 +9,7 @@ class MysqlConfig(BaseSettings):
     DB_USERNAME: str = ""
     DB_PASSWORD: str = ""
     DB_NAME: str = ""
+    DB_URI_SQL_ALCHEMY: str = ""
 
     class Config:
         env_file = ".env"

core/chat/chatstore.py

@@ -1,10 +1,10 @@
 import redis
 import os
 import json
-from fastapi import HTTPException
-from uuid import uuid4
+from fastapi.responses import JSONResponse
 from typing import Optional, List
 from llama_index.storage.chat_store.redis import RedisChatStore
+from pymongo.mongo_client import MongoClient
 from llama_index.core.memory import ChatMemoryBuffer
 from service.dto import ChatMessage
 
@@ -16,73 +16,88 @@ class ChatStore:
             port=10365,
             password=os.environ.get("REDIS_PASSWORD"),
         )
+
+        uri = os.getenv("MONGO_URI")
+        self.client = MongoClient(uri)
+
+    def initialize_memory_bot(self, session_id):
         
-    def generate_uuid(use_hex=False):
-        if use_hex:
-            return str(uuid4().hex)
-        else:
-            return str(uuid4())
-        
-    def initialize_memory_bot(self, session_id=None):
-        if session_id is None:
-            session_id = self.generate_uuid()
-    # chat_store = SimpleChatStore()
         chat_store = RedisChatStore(
-            redis_client=self.redis_client
-        )  # Need to be configured
-
-        memory = ChatMemoryBuffer.from_defaults(
-            token_limit=3000, chat_store=chat_store, chat_store_key=session_id
+            redis_client=self.redis_client, ttl=86400  # Time-to-live set for 1 hour
         )
 
+        db = self.client["bot_database"]
+
+        if (
+            self.redis_client.exists(session_id)
+            or session_id in db.list_collection_names()
+        ):
+            if session_id not in self.redis_client.keys():
+                self.add_chat_history_to_redis(
+                    session_id
+                )  # Add chat history to Redis if not found
+            # Create memory buffer with chat store and session key
+            memory = ChatMemoryBuffer.from_defaults(
+                token_limit=3000, chat_store=chat_store, chat_store_key=session_id
+            )
+        else:
+            # Handle the case where the session doesn't exist
+            memory = ChatMemoryBuffer.from_defaults(
+                token_limit=3000, chat_store=chat_store, chat_store_key=session_id
+            )
+
         return memory
-    
+
     def get_messages(self, session_id: str) -> List[dict]:
         """Get messages for a session_id."""
         items = self.redis_client.lrange(session_id, 0, -1)
         if len(items) == 0:
             return []
-        
+
         # Decode and parse each item into a dictionary
         return [json.loads(m.decode("utf-8")) for m in items]
 
     def delete_last_message(self, session_id: str) -> Optional[ChatMessage]:
         """Delete last message for a session_id."""
         return self.redis_client.rpop(session_id)
-    
-    def delete_messages(self, key: str) -> Optional[List[ChatMessage]]:
-        """Delete messages for a key."""
-        self.redis_client.delete(key)
+
+    def delete_messages(self, session_id: str) -> Optional[List[ChatMessage]]:
+        """Delete messages for a session_id."""
+        self.redis_client.delete(session_id)
+        db = self.client["bot_database"]
+        db.session_id.drop()
         return None
-    
+
     def clean_message(self, session_id: str) -> Optional[ChatMessage]:
         """Delete specific message for a session_id."""
         current_list = self.redis_client.lrange(session_id, 0, -1)
-        
+
         indices_to_delete = []
         for index, item in enumerate(current_list):
             data = json.loads(item)  # Parse JSON string to dict
 
             # Logic to determine if item should be removed
-            if (data.get("role") == "assistant" and data.get("content") is None) or (data.get("role") == "tool"):
+            if (data.get("role") == "assistant" and data.get("content") is None) or (
+                data.get("role") == "tool"
+            ):
                 indices_to_delete.append(index)
 
         # Remove elements by their indices in reverse order
         for index in reversed(indices_to_delete):
-            self.redis_client.lrem(session_id, 1, current_list[index])  # Remove the element from the list in Redis
+            self.redis_client.lrem(
+                session_id, 1, current_list[index]
+            )  # Remove the element from the list in Redis
 
     def get_keys(self) -> List[str]:
         """Get all keys."""
-        try :
+        try:
             print(self.redis_client.keys("*"))
             return [key.decode("utf-8") for key in self.redis_client.keys("*")]
-        
+
         except Exception as e:
-            # Log the error and raise HTTPException for FastAPI
+            # Log the error and return JSONResponse for FastAPI
             print(f"An error occurred in update data.: {e}")
-            raise HTTPException(
-                status_code=400, detail="the error when get keys"
-            )
+            return JSONResponse(status_code=400, content="the error when get keys")
 
     def add_message(self, session_id: str, message: ChatMessage) -> None:
         """Add a message for a session_id."""
@@ -90,4 +105,57 @@ class ChatStore:
         self.redis_client.rpush(session_id, item)
 
     def _message_to_dict(self, message: ChatMessage) -> dict:
-        return message.model_dump()
+        return message.model_dump()
+
+    def add_chat_history_to_redis(self, session_id: str) -> None:
+        """Fetch chat history from MongoDB and add it to Redis."""
+        db = self.client["bot_database"]
+        collection = db[session_id]
+
+        try:
+            chat_history = collection.find()
+            chat_history_list = [
+                {
+                    key: message[key]
+                    for key in message
+                    if key not in ["_id", "timestamp"] and message[key] is not None
+                }
+                for message in chat_history
+                if message is not None
+            ]
+
+            for message in chat_history_list:
+                # Convert MongoDB document to the format you need
+                item = json.dumps(
+                    self._message_to_dict(ChatMessage(**message))
+                )  # Convert message to dict
+                # Push to Redis
+                self.redis_client.rpush(session_id, item)
+            self.redis_client.expire(session_id, time=86400)
+
+        except Exception as e:
+            return JSONResponse(status_code=500, content="Add Database Error")
+
+    def get_all_messages_mongodb(self, session_id):
+        """Get all messages for a session_id from MongoDB."""
+        try:
+            db = self.client["bot_database"]
+            collection = db[session_id]
+
+            # Retrieve all documents from the collection
+            documents = collection.find()
+
+            # Convert the cursor to a list and exclude the _id field
+            documents_list = [
+                {key: doc[key] for key in doc if key !="_id" and doc[key] is not None}
+                for doc in documents
+            ]
+
+            # Print the list of documents without the _id field
+            print(documents_list)  # Optional: If you want to see the output
+
+            return documents_list
+        
+        except Exception as e:
+            print(f"An error occurred while retrieving messages: {e}")
+            return JSONResponse(status_code=500, content=f"An error occurred while retrieving messages: {e}") 

core/chat/engine.py

@@ -1,26 +1,19 @@
-from typing import Optional, List
+from typing import List
 from llama_index.core.vector_stores import (
     MetadataFilter,
     MetadataFilters,
-    FilterCondition,
 )
 
-from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.tools import QueryEngineTool, ToolMetadata
 from llama_index.agent.openai import OpenAIAgent
 from llama_index.llms.openai import OpenAI
-from llama_index.storage.chat_store.redis import RedisChatStore
-from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.query_engine import CitationQueryEngine
 from llama_index.core import Settings
 from core.chat.chatstore import ChatStore
 
-from service.dto import ChatMessage
 from config import GPTBOT_CONFIG
-from core.prompt import SYSTEM_BOT_TEMPLATE
-import redis
-import os
-import json
+from core.prompt import SYSTEM_BOT_TEMPLATE, ADDITIONAL_INFORMATIONS
+from core.parser import join_list
 
 
 class Engine:
@@ -33,48 +26,38 @@ class Engine:
         )
 
         self.chat_store = ChatStore()
-
         Settings.llm = self.llm
 
-    def _build_description_bot(self, title, category):
-        try:
-            prompt = f"Write a detailed description for an OpenAI agent with the title '{title}' and categorized under '{category}'."
-            description = self.llm.complete(prompt)
-
-            return description
-
-        except Exception as e:
-            return f"Error generating description: {str(e)}"
+    def get_citation_engine(self, titles:List, index):
+        filters = [
+            MetadataFilter(
+                key="title",
+                value=title,
+                operator="==",
+            )
+            for title in titles
+        ]
 
-    def get_citation_engine(self, title, category, index):
-        filters = MetadataFilters(
-            filters=[
-                MetadataFilter(key="title", value=title),
-                MetadataFilter(key="category", value=category),
-            ],
-            condition=FilterCondition.AND,
-        )
+        filters = MetadataFilters(filters=filters, condition="or")
 
         # Create the QueryEngineTool with the index and filters
         kwargs = {"similarity_top_k": 5, "filters": filters}
 
         retriever = index.as_retriever(**kwargs)
 
-        citation_engine = CitationQueryEngine(retriever=retriever)
+        # citation_engine = CitationQueryEngine(retriever=retriever)
 
-        return citation_engine
+        return CitationQueryEngine.from_args(index, retriever=retriever)
 
-    def get_chat_engine(
-        self, session_id, index, title=None, category=None, type="general"
-    ):
+    def get_chat_engine(self, session_id, index, titles=None, type_bot="general"):
         # Create the QueryEngineTool based on the type
-        if type == "general":
+        if type_bot == "general":
             # query_engine = index.as_query_engine(similarity_top_k=3)
             citation_engine = CitationQueryEngine.from_args(index, similarity_top_k=5)
             description = "A book containing information about medicine"
         else:
-            citation_engine = self.get_citation_engine(title, category, index)
-            description = self._build_description_bot()
+            citation_engine = self.get_citation_engine(titles, index)
+            description = "A book containing information about medicine"
 
         metadata = ToolMetadata(name="bot-belajar", description=description)
         print(metadata)
@@ -85,12 +68,17 @@ class Engine:
         print(vector_query_engine)
 
         # Initialize the OpenAI agent with the tools
+        
+        if type_bot == "general":
+            system_prompt = SYSTEM_BOT_TEMPLATE.format(additional_information="")
+        else:
+            additional_information = ADDITIONAL_INFORMATIONS.format(titles=join_list(titles))
+            system_prompt = SYSTEM_BOT_TEMPLATE.format(additional_information=additional_information)
         chat_engine = OpenAIAgent.from_tools(
             tools=[vector_query_engine],
             llm=self.llm,
             memory=self.chat_store.initialize_memory_bot(session_id),
-            # memory = self.initialize_memory_bot(session_id),
-            system_prompt=SYSTEM_BOT_TEMPLATE,
+            system_prompt=system_prompt,
         )
 
-        return chat_engine
+        return chat_engine

core/parser.py

@@ -90,3 +90,13 @@ def seperate_to_list(text):
         final_output.extend([part.strip() for part in split_line if part.strip()])
 
     return final_output
+
+def join_list(items):
+    if not items:
+        return ""
+    elif len(items) == 1:
+        return items[0]
+    elif len(items) == 2:
+        return f"{items[0]} and {items[1]}"
+    else:
+        return ", ".join(items[:-1]) + " and " + items[-1]

core/prompt.py

@@ -1,17 +1,21 @@
 SYSTEM_BOT_TEMPLATE = """
-Kamu adalah Medbot yang gunakan tool kamu untuk menjawab pertanyaan tentang kedokteran. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersediaserta selalu cantumkan kutipan dari teks yang anda kutip. Jika tidak ada jawaban melalui alat yang digunakan, carilah informasi lebih lanjut dengan menggunakan alat. Jika setelah itu tidak ada informasi yang ditemukan, katakan bahwa kamu tidak mengetahuinya.
+Kamu adalah Medbot yang selalu menggunakan tools kamu untuk menjawab pertanyaan tentang kedokteran. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersedia. {additional_information} Jika setelah itu tidak ada informasi yang ditemukan, katakan bahwa kamu tidak mengetahuinya dan berikan informasi dari apa yang kamu ketahui kemudian arahkan pengguna untuk bertanya ke dokter yang lebih ahli.
 
 **Instruksi**:
 
- 1. **Jawaban Berdasarkan Tools**: Jika pengguna bertanya tentang topik kedokteran, gunakanlah tools yang tersedia untuk memberikan jawaban. Pastikan jawabanmu relevan dan sesuai dengan informasi dari tools tersebut.
+ 1. **Jawaban Berdasarkan Tools**: Jika pengguna bertanya tentang topik kedokteran, gunakanlah tools yang tersedia untuk memberikan jawaban. Pastikan jawabanmu relevan dan sesuai dengan informasi dari tools tersebut. Jelaskan informasi dengan content dan lengkap. 
 
  2. **Referensi dan Kutipan**: Jangan menghapus sumber kutipan dari teks yang diberikan. Contohnya, jika teksnya adalah "Ilmu kedokteran sangat dibutuhkan [2]", pastikan untuk menyertakan kutipan sumbernya yaitu [2] dalam jawabanmu.
 
  3. **Ketika Tidak Tahu Jawaban**: Jika pertanyaan pengguna tidak dapat dijawab dengan menggunakan tools ini, sampaikan dengan sopan bahwa kamu tidak memiliki jawaban untuk pertanyaan tersebut. Arahkan pengguna untuk mencari informasi lebih lanjut atau bertanya pada ahli di bidang kedokteran.
 
- 4. **Gaya Jawaban**: Berikan jawaban dengan gaya yang ramah dan profesional. Hindari penggunaan poin-poin, dan sampaikan informasi secara naratif agar lebih mudah dipahami. Gunakan kata 'dok' atau 'dokter' untuk merujuk pada dokter, dan hindari kesan monoton dengan menambahkan emotikon jika sesuai.
+ 4. **Gaya Jawaban**: Berikan jawaban dengan gaya yang ramah dan profesional. Sampaikan informasi secara naratif agar lebih mudah dipahami. Boleh menggunakan point point dan uraiannya agar bisa menjelaskan informasi yang kompleks sehingga mudah dipahami. Gunakan kata 'dok' atau 'dokter' untuk merujuk pada dokter, dan hindari kesan monoton dengan menambahkan emotikon jika sesuai seperti 😁, 😊, 🙌, 😉, 😀, 🤔, 😇.
 
- 5. **Penutup**: Akhiri komunikasi dengan kalimat yang friendly, seperti "Semoga informasi ini bermanfaat, dok ✨" atau "Jika ada pertanyaan lain, jangan ragu untuk bertanya ya dok 😊"
+ 5. **Penutup**: Akhiri komunikasi dengan kalimat yang friendly, seperti "Semoga informasi ini bermanfaat, dok ✨" atau "Jika ada pertanyaan lain, jangan ragu untuk bertanya ya dok 😊" dan sebagainya. 
+"""
+
+ADDITIONAL_INFORMATIONS = """
+Kemudian, kamu menjawab pertanyan user dari buku {titles}, jadi jika user bertaya kamu pastikan akan mengacu buku tersebut yang didapatkan dari tools dari yang kamu punya. 
 """
 
 SYSTEM_TOPIC_TEMPLATE = """
@@ -100,7 +104,7 @@ Your task is to extract and organize metadata for the {class_name}. Follow the i
 
 2. **Extract the Key Subtopic (if applicable):**
    - **Goal:** Determine the most relevant supporting element related to the main topic.
-   - **How:** Identify a sub-element or detail that provides additional depth or clarification to the main topic.
+   - **How:** Identify a sub-element or content that provides additional depth or clarification to the main topic.
    - **Tip:** Ensure the subtopic directly supports or elaborates on the main topic. 
 
 3. **Handle Cases Without a Clear Subtopic:**

core/summarization/summarizer.py

@@ -3,7 +3,7 @@ import os
 import base64
 import fitz
 
-from fastapi import HTTPException
+from fastapi.responses import JSONResponse
 from llama_index.core.vector_stores import (
     MetadataFilter,
     MetadataFilters,
@@ -39,7 +39,7 @@ class SummarizeGenerator:
             print(content_table)
             # content_table = fitz.open(topics_image)
         except Exception as e:
-            raise HTTPException(status_code=400, detail=f"Error opening PDF file: {e}")
+            return JSONResponse(status_code=400, content=f"Error opening PDF file: {e}")
 
         # Initialize a list to collect base64 encoded images
         pix_encoded_combined = []
@@ -57,7 +57,7 @@ class SummarizeGenerator:
                 continue  # Skip to the next page if there's an error
 
         if not pix_encoded_combined:
-            raise HTTPException(status_code=404, detail="No images found in the PDF")
+            return JSONResponse(status_code=404, content="No images found in the PDF")
 
         return pix_encoded_combined
 
@@ -102,7 +102,7 @@ class SummarizeGenerator:
             return str(refined_extractor_output), extractor_dics
 
         except Exception as e:
-            raise HTTPException(status_code=500, detail=f"An error occurred: {e}")
+            return JSONResponse(status_code=500, content=f"An error occurred: {e}")
 
     def _extract_image_as_base64(self, page):
         try:
@@ -110,7 +110,7 @@ class SummarizeGenerator:
             pix_bytes = pix.tobytes()
             return base64.b64encode(pix_bytes).decode("utf-8")
         except Exception as e:
-            raise HTTPException(status_code=500, detail=f"Error extracting image: {e}")
+            return JSONResponse(status_code=500, content=f"Error extracting image: {e}")
 
     def index_summarizer_engine(self, topic, subtopic, index):
         filters = MetadataFilters(

db/database.py

@@ -0,0 +1,58 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import OperationalError
+from config import MYSQL_CONFIG
+from fastapi import HTTPException
+import os
+import base64
+
+
+SQLALCHEMY_DATABASE_URL = MYSQL_CONFIG.DB_URI_SQL_ALCHEMY
+
+# Get the base64 encoded certificate from the environment variable
+ca_cert_base64 = os.getenv("CA_CERT_BASE64")
+
+# Decode the base64 content
+if ca_cert_base64:
+    ca_cert_content = base64.b64decode(ca_cert_base64).decode("utf-8")
+
+    # Write the decoded content to a temporary .pem file
+    with open("/tmp/ca.pem", "w") as f:
+        f.write(ca_cert_content)
+
+    ca_cert_path = "/tmp/ca.pem"
+else:
+    raise ValueError("CA_CERT_BASE64 environment variable is not set")
+
+# Use the decoded CA certificate in the SQLAlchemy engine
+engine = create_engine(
+    SQLALCHEMY_DATABASE_URL,
+    connect_args={
+        "ssl": {
+            "sslmode": "REQUIRED",
+            "ca": ca_cert_path,  # Path to the temporary CA certificate
+            # Add other SSL options as needed
+        }
+    },
+)
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    except OperationalError as e:
+        # Log the error and raise HTTPException for FastAPI
+        print(f"An error occurred in get database sql alchemy.: {e}")
+        raise HTTPException(status_code=400, detail="Database connection error")
+    # Check if it's an authentication-related error
+    except Exception as e:
+        # Check if it's an authentication-related error
+        if "401" in str(e):
+            raise HTTPException(status_code=401, detail="Authentication failed")
+        else:
+            # For any other type of exception, raise a generic 400 error
+            print(f"An error occurred: {e}")
+            raise HTTPException(status_code=400, detail="An unexpected error occurred")

db/delete_data.py

@@ -10,7 +10,7 @@ class DeleteDatabase(Repository):
         if "id" not in params:
             raise ValueError("The 'id' parameter is required.")
         query = """
-            DELETE FROM Metadata 
+            DELETE FROM metadata 
             WHERE id = :id
         """
 

db/get_data.py

@@ -1,5 +1,6 @@
 import logging
 from db.repository import Repository, get_db_conn
+from fastapi.responses import JSONResponse
 
 # Setup logging (configure as needed)
 logging.basicConfig(level=logging.INFO)
@@ -23,14 +24,14 @@ class GetDatabase(Repository):
             return results if results else None
         except Exception as e:
             logging.error(f"An error occurred while executing query: {e}")
-            return None
+            return JSONResponse(status_code=500, content=f"An error occurred while executing query: {e}")
 
     async def get_data(self, title):
         """
         Fetch the first result matching the given title from the metadata table.
         """
         query = """
-        SELECT * FROM Metadata  
+        SELECT * FROM metadata  
         WHERE title = %s
         limit 5;
         """
@@ -40,21 +41,22 @@ class GetDatabase(Repository):
             return results
         except Exception as e:
             logging.error(f"An error occurred while get data: {e}")
-            return None
+            return JSONResponse(status_code=500, content=f"An error occurred while get data: {e}")
 
     async def get_all_data(self):
         """
         Fetch all data from the metadata table.
         """
         query = """
-        SELECT * FROM Metadata
+        SELECT * FROM metadata
         """
         results = await self.execute_query(query)
+        print("result", results)
         return results
     
     async def get_data_by_id(self, id):
         query = f"""
-        SELECT * FROM Metadata WHERE id = :id
+        SELECT * FROM metadata WHERE id = :id
         """
         
         param = {"id" : id}
@@ -64,4 +66,4 @@ class GetDatabase(Repository):
             return results[0] if results else None
         except Exception as e:
             print('Error fetching data by ID %s: %s', id, e)
-            return None
+            return JSONResponse(status_code=500, content=f"An error while fething data: {e}")

db/models.py

@@ -0,0 +1,164 @@
+from typing import Literal
+from typing_extensions import Annotated
+import sqlalchemy
+from sqlalchemy.orm import mapped_column
+from sqlalchemy import Integer, String, ForeignKey,func, DateTime, Boolean, LargeBinary
+from sqlalchemy.orm import DeclarativeBase, Mapped
+import uuid
+import datetime
+import pytz
+
+# Set Jakarta timezone
+jakarta_tz = pytz.timezone('Asia/Jakarta')
+
+def get_jakarta_time():
+    return datetime.datetime.now(jakarta_tz)
+
+# Use the timezone-aware function in SQLAlchemy annotations
+timestamp_current = Annotated[
+    datetime.datetime,
+    mapped_column(nullable=False, default=get_jakarta_time)  # Use default instead of server_default
+]
+
+timestamp_update = Annotated[
+    datetime.datetime,
+    mapped_column(nullable=False, default=get_jakarta_time, onupdate=get_jakarta_time)  # onupdate uses the Python function
+]
+
+message_role = Literal["user", "assistant"]
+
+class Base(DeclarativeBase):
+    type_annotation_map = {
+        message_role: sqlalchemy.Enum("user", "assistant", name="message_role"),
+    }
+
+class User(Base):
+    __tablename__ = "user"
+
+    id = mapped_column(Integer, primary_key=True)
+    name = mapped_column(String(100), nullable=False)
+    username = mapped_column(String(100), unique=True, nullable=False)
+    role_id = mapped_column(Integer, ForeignKey("role.id"))
+    email = mapped_column(String(100), unique=True, nullable=False)
+    password_hash = mapped_column(String(100), nullable=False)
+    created_at: Mapped[timestamp_current] 
+    updated_at : Mapped[timestamp_update] 
+
+class Feedback(Base):
+    __tablename__ = "feedback"
+    
+    id = mapped_column(Integer, primary_key=True)
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    rating = mapped_column(Integer)
+    comment = mapped_column(String(1000))
+    created_at : Mapped[timestamp_current] 
+
+class Role(Base):
+    __tablename__ = "role"
+    
+    id = mapped_column(Integer, primary_key=True)
+    role_name = mapped_column(String(200), nullable=False)
+    description = mapped_column(String(200))
+    
+class User_Role(Base):
+    __tablename__ = "user_role"
+    
+    id = mapped_column(Integer, primary_key=True)
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    role_id = mapped_column(Integer, ForeignKey("role.id"))
+    
+class Bot(Base):
+    __tablename__ = "bot"
+    
+    id = mapped_column(Integer, primary_key=True)
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    bot_name = mapped_column(String(200), nullable=False)
+    created_at : Mapped[timestamp_current] 
+    
+class Session(Base):
+    __tablename__ = "session"
+    
+    id = mapped_column(String(36), primary_key=True, index=True, default=lambda: str(uuid.uuid4()))  # Store as string
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    bot_id = mapped_column(Integer, ForeignKey("bot.id"))
+    created_at : Mapped[timestamp_current]
+    updated_at : Mapped[timestamp_update] 
+
+class Message(Base):
+     __tablename__ = "message"
+     
+     id = mapped_column(Integer, primary_key=True)
+     session_id = mapped_column(String(36), ForeignKey("session.id"), nullable=False)  # Store as string
+     role : Mapped[message_role]
+     goal = mapped_column(String(200))
+     created_at : Mapped[timestamp_current] 
+    
+class Category(Base):
+    __tablename__ = "category"
+    
+    id = mapped_column(Integer, primary_key=True)
+    category = mapped_column(String(200))
+    created_at : Mapped[timestamp_current]
+    updated_at : Mapped[timestamp_update]  
+
+class Metadata(Base):
+    __tablename__ = "metadata"
+    
+    id = mapped_column(Integer, primary_key=True)
+    title = mapped_column(String(200))
+    # image_data = mapped_column(LargeBinary, nullable=True)
+    category_id = mapped_column(Integer, ForeignKey("category.id"))
+    author = mapped_column(String(200))
+    year = mapped_column(Integer)
+    publisher = mapped_column(String(100))
+    thumbnail = mapped_column(LargeBinary, nullable=True)
+    created_at : Mapped[timestamp_current] 
+    updated_at : Mapped[timestamp_update] 
+
+     
+class Bot_Meta(Base):
+    __tablename__ = "bot_meta"
+    
+    id = mapped_column(Integer, primary_key=True)
+    bot_id = mapped_column(Integer, ForeignKey("bot.id"))
+    metadata_id = mapped_column(Integer, ForeignKey("metadata.id"))
+    created_at : Mapped[timestamp_current]
+    updated_at : Mapped[timestamp_update] 
+
+class User_Meta(Base):
+    __tablename__ = "user_meta"
+    
+    id = mapped_column(Integer, primary_key=True)
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    metadata_id = mapped_column(Integer, ForeignKey("metadata.id"))
+    created_at : Mapped[timestamp_current]
+    updated_at : Mapped[timestamp_update] 
+    
+class Planning(Base):
+    __tablename__="planning"
+    
+    id = mapped_column(Integer, primary_key=True)
+    trials_id = mapped_column(Integer, ForeignKey("trials.id"))
+    planning_name = mapped_column(String(200), nullable=False)
+    duration = mapped_column(Integer, nullable=False)  # Duration in months
+    start_date = mapped_column(DateTime, nullable=False)  # Start date of the planning
+    end_date = mapped_column(DateTime, nullable=False)  # End date of the planning
+    is_activated = mapped_column(Boolean)
+    created_at : Mapped[timestamp_current]  # Automatically sets the current timestamp
+    
+class Trials(Base):
+    __tablename__ = "trials"
+    
+    id = mapped_column(Integer, primary_key=True)
+    token_used = mapped_column(Integer, nullable=False)  # Adjust length as needed
+    token_planned = mapped_column(Integer, nullable=False)
+
+
+class Session_Publisher(Base):
+    __tablename__ = "session_publisher"
+    
+    id = mapped_column(String(36), primary_key=True, index=True, default=lambda: str(uuid.uuid4()))  # Store as string
+    user_id = mapped_column(Integer, ForeignKey("user.id"))
+    metadata_id = mapped_column(Integer, ForeignKey("metadata.id"))
+    created_at : Mapped[timestamp_current] 
+    updated_at : Mapped[timestamp_update]

db/save_data.py

@@ -3,21 +3,30 @@ import logging
 from dotenv import load_dotenv
 from db.repository import Repository
 
+
 load_dotenv()
 
 
 class InsertDatabase(Repository):
 
     # Example function to insert data asynchronously
-    async def insert_data(self, params):
+    async def insert_data(self, params, category_id):
         # SQL insert query with named placeholders
         query = """
-        INSERT INTO Metadata (title, category, author, year, publisher, createdAt, updatedAt) 
-        VALUES (:title, :category, :author, :year, :publisher, :createdAt, :updatedAt)
+        INSERT INTO metadata (title, category_id, author, year, publisher) 
+        VALUES (:title, :category_id, :author, :year, :publisher)
         """
 
-        reference = self.update_params(params)
+        reference = {
+            "title": params["title"],
+            "category_id": category_id,  # directly assign category_id
+            "author": params["author"],
+            "year": params["year"],
+            "publisher": params["publisher"]
+        }
+        
 
+        print(reference)
         try:
             # Execute the query with the provided values
             await self._exec(query, reference)

db/update_data.py

@@ -7,30 +7,29 @@ logging.basicConfig(level=logging.INFO)
 
 class UpdateDatabase(Repository):
     async def update_record(self, reference):
+        print("update record", reference)
         if "id" not in reference:
             raise ValueError("The 'id' parameter is required.")
         query = """
-            UPDATE Metadata 
+            UPDATE metadata 
             SET title = :title, 
-                category = :category, 
+                category_id = :category_id, 
                 author = :author, 
                 year = :year, 
-                publisher = :publisher, 
-                updatedAt = :updatedAt
+                publisher = :publisher 
             WHERE id = :id
         """
         print(query)
 
-        updated_reference = self.update_params(reference, update=True)
-        print(updated_reference)
+        print(reference)
 
         try:
-            await self._exec(query, updated_reference)
+            await self._exec(query, reference)
             logging.info(
-                f"Record with id {updated_reference['id']} updated successfully."
+                f"Record with id {reference['id']} updated successfully."
             )
         except Exception as e:
             logging.error(
-                f"Error updating  record with id {updated_reference['id']}: {e}"
+                f"Error updating  record with id {reference['id']}: {e}"
             )
             raise