Spaces:
Sleeping
Sleeping
File size: 66,063 Bytes
e4950ac f08b1e6 e4950ac f08b1e6 bf06760 d08917a bf06760 f08b1e6 bf06760 d08917a bf06760 f08b1e6 bf06760 d08917a e4950ac bf06760 e4950ac d08917a e4950ac bf06760 f08b1e6 d08917a f08b1e6 d08917a f08b1e6 d08917a f08b1e6 d08917a f08b1e6 40931da f9884d7 a399790 40931da 47c982e 40931da a399790 bf06760 d60a7be d73b749 d60a7be bf06760 4eefd6c bf06760 40931da a399790 bf06760 40931da 0fe444f 47c982e d08917a bf06760 5af28e2 14ff867 5af28e2 bf06760 d08917a f08b1e6 e4950ac 7d8c955 e4950ac 0894bc7 6d89d1b f08b1e6 d08917a e4950ac bf06760 e4950ac 40931da 38c665d d08917a 40931da d08917a bf06760 d08917a 2d3a6d0 d08917a 2d3a6d0 d08917a e4950ac d08917a b4fb6f3 e4950ac d08917a e4950ac d08917a e4950ac d08917a e4950ac f08b1e6 e4950ac d08917a e4950ac d08917a e4950ac d08917a bf06760 d08917a bf06760 d08917a d90ca8f d08917a d90ca8f d08917a e4950ac d08917a e4950ac d08917a e4950ac d08917a e4950ac f08b1e6 e4950ac d08917a e4950ac f08b1e6 d08917a e4950ac bf06760 d08917a e4950ac f08b1e6 e4950ac d08917a e4950ac d08917a 40931da 1e4fe02 40931da c25aacb c7a1e0f f9884d7 c25aacb f9884d7 40931da c25aacb 40931da c25aacb d08917a c25aacb 4aa81c4 a6a2185 84e9804 a6a2185 f9884d7 a6a2185 f9884d7 a6a2185 f9884d7 a6a2185 f9884d7 a6a2185 f9884d7 a6a2185 8f7f566 f9884d7 f7f6c3a a6a2185 359d52b f9884d7 b2c97f3 8f7f566 f9884d7 d08917a e4950ac d08917a e4950ac d08917a 48800dc d08917a 8bd2bf8 d08917a 426139e 8bd2bf8 bf06760 8e042cd 9d3ec0f 136f30b c1da09a 9d3ec0f c1da09a 9d3ec0f 4e5ef51 9d3ec0f c1da09a bf06760 9d3ec0f bf06760 9d3ec0f d08917a bf06760 d08917a bf06760 e4950ac d08917a e4950ac d08917a e4950ac bf06760 e4950ac d08917a e4950ac d08917a f08b1e6 d08917a f08b1e6 e4950ac bf06760 e4950ac 9a382de e4950ac bf06760 d08917a f08b1e6 d08917a f08b1e6 d08917a f08b1e6 d08917a e4950ac 40931da a399790 f9884d7 a399790 40931da d08917a 40931da a399790 40931da bf06760 bbf123a a399790 40931da bd21414 40931da bd21414 bf06760 03bf030 bd21414 40931da 03bf030 bd21414 a399790 40931da 9b7d284 b654e9e a399790 85e2306 bbf123a 47c982e a399790 1a022d8 47c982e a399790 47c982e 40931da a399790 40931da 28a8111 9205517 47c982e f9884d7 bf06760 b654e9e 40931da b654e9e 40931da b654e9e 67e57d4 e32ac0f b654e9e 0ddd34e b654e9e 40931da 919e661 b654e9e 4c2c953 b654e9e 40931da b654e9e 40931da b654e9e a399790 40931da b654e9e a399790 e4950ac f08b1e6 d08917a f08b1e6 d08917a f08b1e6 e4950ac 2d2aa0c 3aa8a93 2d2aa0c 3aa8a93 2d2aa0c d08917a 2d2aa0c bf06760 d08917a e4950ac d08917a 2d2aa0c e4950ac 2d2aa0c f68e394 bf06760 d08917a 2d2aa0c 529bb03 2d2aa0c d15bbb3 c25aacb d15bbb3 72764b9 d15bbb3 c25aacb 28fd854 c25aacb d08917a bf1b1ca 72f3118 d08917a ee37af5 72f3118 f08b1e6 e4950ac bf06760 e4950ac bf06760 d08917a bf06760 e4950ac 35c9cbd e4950ac bf06760 e4950ac d08917a e4950ac 40931da f6fd3d2 4eefd6c 696f0ae 651bb1a bf06760 4a75fd1 bf06760 4a75fd1 bf06760 be3107c bf06760 6c18a8e bf06760 47c982e bf06760 0a8efe8 bf06760 ca393e4 16e1355 31df548 bf06760 13c569b bf06760 d60a7be bf06760 04e39a2 eadd1e3 04e39a2 7d35d92 3e78c03 bf06760 d60a7be bf06760 b41393c 311cf7d 9f88cd1 4eefd6c 04e39a2 311cf7d ab0a986 7fc690e d60a7be 9a8046f d98661f 31df548 13c569b e32ac0f ab0a986 213b4e1 bf06760 9a8046f bf06760 b53a294 46558f3 b53a294 31df548 0a8efe8 bf06760 551fd6e d98661f 5af28e2 93846e9 9a8046f d60a7be 9de5c2e d98661f 53f422f d98661f 4eefd6c 8430191 d60a7be 5af28e2 551fd6e 4eefd6c 8430191 86c56d3 551fd6e a87b441 9a8046f bf06760 6d89d1b bf06760 b084e14 a87b441 fb92eb1 bf06760 e354dc5 d98661f bf06760 81844dc b53a294 7e70c1a d60a7be bf06760 4eefd6c 31df548 b53a294 d12af57 b53a294 009c1cf 31df548 4eefd6c 9de5c2e 93846e9 4eefd6c 2efbd35 31df548 cfe0c65 16e1355 cfe0c65 31df548 cfe0c65 83af1fb bf06760 83af1fb f45d85e 83af1fb 314075d 83af1fb 651bb1a f87e2cf 651bb1a 7a8980b f87e2cf 7a8980b 83af1fb 651bb1a 83af1fb 651bb1a bf06760 a1957af d207765 3b0ddd1 bf06760 83af1fb 651bb1a bf06760 d0ce3b6 bf06760 b9cafb8 83af1fb bf06760 83af1fb aa04435 bf06760 76601fb 90bb409 bf06760 d08917a bf06760 fb92eb1 58df56a d08917a bf06760 d08917a bf06760 d08917a bf06760 e4950ac bf06760 d08917a bf06760 d08917a 83af1fb bf06760 ae63ea7 2a0111a bf06760 7a8980b bf06760 34a9b15 bf06760 f9884d7 e4950ac d08917a 83af1fb e4950ac a399790 f9884d7 f08b1e6 4a75fd1 bf06760 4a75fd1 bf06760 4a75fd1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 |
from PIL import Image
import base64
from io import BytesIO
import os
import re
import tempfile
import wave
import requests
import gradio as gr
import time
import shutil
import json
import nltk
import mysql.connector
import fnmatch
# audio related code is not included based on Arun's input
# audio package
import speech_recognition as sr
from pydub import AudioSegment
from pydub.playback import play
# SMTP code is not included since HFSpaces doesn't support it
# email library
import smtplib, ssl
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
# langchain
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnableSequence, RunnableLambda
from langchain_openai import ChatOpenAI
from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import FAISS
from langchain_community.utilities import SQLDatabase
from langchain.agents import create_tool_calling_agent, AgentExecutor, Tool
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.tools import StructuredTool
#from langchain.pydantic_v1 import BaseModel, Field
from pydantic import BaseModel, Field
from PyPDF2 import PdfReader
from nltk.tokenize import sent_tokenize
from datetime import datetime
from sqlalchemy import create_engine
from sqlalchemy.sql import text
import openai
# pandas
import pandas as pd
from pandasai.llm.openai import OpenAI
from pandasai import SmartDataframe
from dotenv import load_dotenv
# Load environment variables
load_dotenv()
# langfuse analytics
from langfuse.callback import CallbackHandler
# Inventory API data table
from tabulate import tabulate
#forcefully stop the agent execution
import concurrent.futures
import threading
# mailjet_rest to send email
from mailjet_rest import Client
import base64
#for PDF form filling
from PyPDFForm import FormWrapper
import os
import zipfile
# Get the current working directory
current_folder = os.getcwd()
#Variables Initialization
agent_executor = None
vector_store1 = None
texts1 = None
excel_dataframe = None
file_extension = None
total_rows = ""
docstatus = ""
sample_table = ""
#This is to define the summary of the runtime tool. This summary will be updated in prompt template and description of the new tool
run_time_tool_summary=""
user_name = ""
# Define global variables for managing the thread and current_event
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
current_event = None
stop_event = threading.Event()
# LangFuse API keys and host settings
os.environ["LANGFUSE_PUBLIC_KEY"] = os.getenv("LANGFUSE_PUBLIC_KEY")
os.environ["LANGFUSE_SECRET_KEY"] = os.getenv("LANGFUSE_SECRET_KEY")
os.environ["LANGFUSE_HOST"] = os.getenv("LANGFUSE_HOST")
DB_USER = 'u852023448_redmindgpt'
DB_PASSWORD = 'redmindGpt@123'
DB_HOST = '217.21.88.10'
DB_NAME = 'u852023448_redmindgpt'
from huggingface_hub import create_branch, create_tag
from huggingface_hub import login
from huggingface_hub import HfApi
api= HfApi()
token = os.getenv("HF_TOKEN")
login(token=os.getenv("HF_TOKEN"))
langfuse_handler = CallbackHandler()
langfuse_handler.auth_check() # Optional: Checks if the authentication is successful
nltk.download('punkt')
open_api_key_token = os.getenv("OPEN_AI_API")
os.environ['OPENAI_API_KEY'] = open_api_key_token
pdf_path = "Input/Inbound.pdf"
db_uri = os.getenv("POSTGRESQL_CONNECTION")
# Database setup
db = SQLDatabase.from_uri(db_uri)
user_email = ""
warehouse_name = ""
warehouse_id = ""
# Today's date to be populated in inventory API
inventory_date = datetime.today().strftime('%Y-%m-%d')
apis = [
# fetch warehouse ID
{
"url": "http://193.203.162.39:81/nxt-wms/userWarehouse/fetchWarehouseForUserId?",
"params": {"query": warehouse_name, "userId": 164}
},
# Stock summary based on warehouse id
{
"url": "http://193.203.162.39:81/nxt-wms/transactionHistory/stockSummary?",
"params": {"branchId": 343, "onDate": inventory_date, "warehouseId": warehouse_id}
}
]
# LLM setup
llm = ChatOpenAI(model="gpt-4o-mini", max_tokens=300, temperature=0.1)
llm_chart = OpenAI(is_safe=False)
def get_schema(_):
schema_info = db.get_table_info() # This should be a string of your SQL schema
return schema_info
def generate_sql_query(question):
schema = get_schema(None)
template_query_generation = """
Schema: {schema}
Question: {question}
Provide a SQL query to answer the above question using the exact field names and table names specified in the schema.
SQL Query (Please provide only the SQL statement without explanations or formatting):
"""
prompt_query_generation = ChatPromptTemplate.from_template(template_query_generation)
schema_and_question = RunnableLambda(lambda _: {'schema': schema, 'question': question})
sql_chain = RunnableSequence(
schema_and_question,
prompt_query_generation,
llm.bind(stop=["SQL Query End"]), # Adjust the stop sequence to your need
StrOutputParser()
)
sql_query = sql_chain.invoke({})
sql_query = sql_chain.invoke({}, config={"callbacks": [langfuse_handler]})
return sql_query.strip()
def run_query(query):
# Clean the query by removing markdown symbols and trimming whitespace
clean_query = query.replace("```sql", "").replace("```", "").strip()
print(f"Executing SQL Query: {clean_query}")
try:
result = db.run(clean_query)
return result
except Exception as e:
print(f"Error executing query: {e}")
return None
# Define the database query tool
# The function that uses the above models
# Define the function that will handle the database query
def database_tool(question):
sql_query = generate_sql_query(question)
return run_query(sql_query)
def get_ASN_data(question):
base_url = os.getenv("ASN_API_URL")
print(f"base_url{base_url}")
complete_url = f"{base_url}branchMaster.id=343&transactionUid={question}&userId=164&transactionType=ASN"
try:
response = requests.get(complete_url)
print(f"complete_url{complete_url}")
print(f"response{response}")
data = response.json()
response.raise_for_status()
if 'result' in data and 'content' in data['result'] and data['result']['content']:
content = data['result']['content'][0]
trnHeaderAsn = content['trnHeaderAsn']
party = content['party'][0]
transactionUid = trnHeaderAsn['transactionUid']
customerOrderNo = trnHeaderAsn.get('customerOrderNo', 'N/A')
orderDate = trnHeaderAsn.get('orderDate', 'N/A')
customerInvoiceNo = trnHeaderAsn.get('customerInvoiceNo', 'N/A')
invoiceDate = trnHeaderAsn.get('invoiceDate', 'N/A')
expectedReceivingDate = trnHeaderAsn['expectedReceivingDate']
transactionStatus = trnHeaderAsn['transactionStatus']
shipper_code = party['shipper']['code'] if party['shipper'] else 'N/A'
shipper_name = party['shipper']['name'] if party['shipper'] else 'N/A'
data = [
["Transaction UID", transactionUid],
["Customer Order No", customerOrderNo],
["Order Date", orderDate],
["Customer Invoice No", customerInvoiceNo],
["Invoice Date", invoiceDate],
["Expected Receiving Date", expectedReceivingDate],
["Transaction Status", transactionStatus],
["Shipper Code", shipper_code],
["Shipper Name", shipper_name]
]
return f"The ASN details of {question} is {data}."
else:
return "ASN Details are not found. Please contact system administrator."
except requests.exceptions.HTTPError as http_err:
print(f"HTTP error occurred: {http_err}")
return "Sorry, we encountered an error while processing your request. Please try after some time."
except Exception as err:
print(f"An error occurred: {err}")
return "Sorry, we encountered an error while processing your request. Please try after some time."
def load_and_split_pdf(pdf_path):
reader = PdfReader(pdf_path)
text = ''
for page in reader.pages:
text += page.extract_text()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=50)
texts = text_splitter.split_text(text)
return texts
def create_vector_store(texts):
embeddings = OpenAIEmbeddings()
vector_store = FAISS.from_texts(texts, embeddings)
return vector_store
def query_vector_store(vector_store, query, config=None):
if config:
print("Config passed:", config)
docs = vector_store.similarity_search(query, k=5)
print(f"Vector store return: {docs}")
return docs
def summarize_document(docs):
summarized_docs = []
for doc in docs:
if isinstance(doc, list):
doc_content = ' '.join([d.page_content for d in doc])
else:
doc_content = doc.page_content
sentences = sent_tokenize(doc_content)
if len(sentences) > 5:
summarized_content = ' '.join(sentences[:5])
else:
summarized_content = doc_content
summarized_docs.append(summarized_content)
return '\n\n'.join(summarized_docs)
texts = load_and_split_pdf(pdf_path)
vector_store = create_vector_store(texts)
def document_data_tool_runtime(question):
print(f"Document data runtime tool enter: {question} with {vector_store1}")
query_response = query_vector_store(vector_store1, question, config={"callbacks": [langfuse_handler]})
return query_response
def document_data_tool(question):
print(f"Document data tool enter: {question}")
# query_string = question['tags'][0] if 'tags' in question and question['tags'] else ""
query_response = query_vector_store(vector_store, question, config={"callbacks": [langfuse_handler]})
# summarized_response = summarize_document(query_response)
return query_response
# mailjet API since SMTP is not supported HF spaces
def send_email_with_attachment_mailjet(recipient_email, subject, body, attach_img_base64=None):
api_key = os.getenv("MAILJET_API_KEY")
api_secret = os.getenv("MAILJET_API_SECRET")
# Initialize the Mailjet client
mailjet = Client(auth=(api_key, api_secret), version='v3.1')
# Define the email details with an attachment
data = {
'Messages': [
{
"From": {
"Email": "[email protected]",
"Name": "Redmind Technologies"
},
"To": [
{
"Email": recipient_email,
"Name": ""
}
],
"Subject": subject,
"TextPart": body,
"CustomID": "AppGettingStartedTest",
"Attachments": [
{
"ContentType": "image/png", # Replace with the correct MIME type of your image
"Filename": "inventory_report.png", # Name of the image as it will appear in the email
"Base64Content": attach_img_base64 # Base64-encoded image content
}
]
}
]
}
# Send the email
result = mailjet.send.create(data=data)
# Check if the email was sent successfully
if result.status_code == 200:
print("Email sent successfully with attachment!")
else:
print(f"Failed to send email. Status code: {result.status_code}")
print(result.json())
#smtp lib
def send_email_with_attachment(recipient_email, subject, body, attachment_path):
try:
sender_email = os.getenv("EMAIL_SENDER")
sender_password = os.getenv("EMAIL_PASSWORD")
# Create a multipart message
msg = MIMEMultipart()
msg['From'] = sender_email
msg['To'] = recipient_email
msg['Subject'] = subject
# Attach the body with the msg instance
msg.attach(MIMEText(body, 'plain'))
# Open the file to be sent
attachment = open(attachment_path, "rb")
# print("Attached the image")
# Instance of MIMEBase and named as p
part = MIMEBase('application', 'octet-stream')
# To change the payload into encoded form
part.set_payload((attachment).read())
# Encode into base64
encoders.encode_base64(part)
part.add_header('Content-Disposition', f"attachment; filename= {attachment_path}")
# Attach the instance 'part' to instance 'msg'
msg.attach(part)
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender_email, sender_password)
text = msg.as_string()
server.sendmail(sender_email, recipient_email, text)
server.quit()
except Exception as error:
print(f"An error occurred: {error}")
# return 1
def make_api_request(url, params):
"""Generic function to make API GET requests and return JSON data."""
try:
response = requests.get(url, params=params)
response.raise_for_status() # Raises an HTTPError if the response was an error
return response.json() # Return the parsed JSON data
except requests.exceptions.HTTPError as http_err:
print(f"HTTP error occurred: {http_err}")
except Exception as err:
print(f"An error occurred: {err}")
def inventory_report(question):
# Split the question to extract warehouse name, user question, and optional email
if question.count(":") > 0:
parts = question.split(":", 2)
warehouse_name= parts[0].strip()
user_question = parts[1].strip()
user_email = parts[2].strip() if len(parts) > 2 else None
print(f"Warehouse: {warehouse_name}, Email: {user_email}, Question: {user_question}")
else:
return "warehouse name not found"
data = make_api_request(apis[0]["url"], apis[0]["params"])
print(data)
if data:
print(f"warehouse name: {warehouse_name}")
# Extracting the id for the warehouse with the name "WH"
warehouse_id = next((item['id'] for item in data['result'] if item['wareHouseId'] == warehouse_name), None)
print(f"warehouse_id:{warehouse_id}")
if (warehouse_id):
print("before api id")
# Step 3: Update the placeholder with the actual warehouse_id
for api in apis:
if isinstance(api, dict) and "params" in api:
if "warehouseId" in api["params"]:
api["params"]["warehouseId"] = warehouse_id
print("after api id")
print(apis)
data1 = make_api_request(apis[1]["url"], apis[1]["params"])
print(data1)
if (data1):
headers = ["S.No", "Warehouse Code", "Warehouse Name", "Customer Code", "Customer Name", "Item Code", "Item Name",
"Currency", "EAN", "UOM", "Quantity", "Gross Weight", "Volume", "Total Value"]
table_data = []
# Check if 'content' exists and is a list
if data1['result'].get('content', []):
for index, item in enumerate(data1['result'], start=1):
row = [
index, # Serial number
item['warehouse']['code'],
item['warehouse']['name'],
item['customer']['code'],
item['customer']['name'],
item['skuMaster']['code'],
item['skuMaster']['name'],
item['currency']['code'],
item['eanUpc'],
item['uom']['code'],
item['totalQty'],
item['grossWeight'],
item['volume'],
item['totalValue']
]
table_data.append(row)
else:
print("No data available in 'content'.")
return "There are no inventory details for the warehouse you have given."
# Convert to pandas DataFrame
df = pd.DataFrame(table_data, columns=headers)
print(df)
chart_link = chat_with_llm(df,question)
return chart_link
else:
return "There are no inventory details for the warehouse you have given."
else:
return "Please provide a warehouse name available in the database."
def chat_with_llm(df,question):
sdf = SmartDataframe(df, config={"llm": llm_chart})
llm_response = sdf.chat(question)
return llm_response
def bind_llm(llm, tools,prompt_template):
llm = llm.bind()
agent = create_tool_calling_agent(llm, tools, ChatPromptTemplate.from_template(prompt_template))
agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
return agent_executor
# Define input and output models using Pydantic
class QueryInput(BaseModel):
question: str = Field(
description="The question to be answered by appropriate tool. Please follow the instructions. For API tool, do not send the question as it is. Please send the ASN id.")# Invoke datavisulaization tool by processing the user question and send two inputs to the tool. One input will be the warehouse name and another input to the tool will be the entire user_question itself. Please join those two strings and send them as a single input string with ':' as delimiter")
# config: dict = Field(default={}, description="Optional configuration for the database query.")
# Define the output model for database queries
class QueryOutput(BaseModel):
result: str = Field(...,
description="Display the answer based on the prompts given in each tool. For dataVisualization tool, it sends a image file as output. Please give the image file path only to the gr.Image. For DocumentData tool, Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points.")
# Wrap the function with StructuredTool for better parameter handling
tools = [
StructuredTool(
func=get_ASN_data,
name="APIData",
args_schema=QueryInput,
output_schema=QueryOutput,
description="Tool to get details of ASN api. ASN id will be in the input with the format of first three letters as ASN and it is followed by 11 digit numeral. Pass only the id as input. Do not send the complete user question to the tool. If there are any other queries related to ASN without ASN id, please use the document tool."
),
StructuredTool(
func=document_data_tool,
name="DocumentData",
args_schema=QueryInput,
output_schema=QueryOutput,
description="You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. "
),
StructuredTool(
func=database_tool,
name="DatabaseQuery",
args_schema=QueryInput,
output_schema=QueryOutput,
description="Tool to query the database based on structured input."
),
StructuredTool(
func=inventory_report,
name="dataVisualization",
args_schema=QueryInput,
output_schema=QueryOutput,
description=""" Tool to generate a visual chart output for a particular warehouse based on the provided question.
This tool processes the user question to identify the warehouse name and the specific request. If the user specifies
an email, include the email in the input. The input format should be: 'warehouse name: user question: email (if any)'.
The tool generates the requested chart and sends it to the provided email if specified.
Examples:
1. Question without email, without warehouse: "Analyze item name and quantity in a bar chart in warehouse"
Input to tool: "I want to analyze item name and quantity in a bar chart"
2. Question with email: "Analyze item name and quantity in a bar chart in warehouse Allcargo Logistics and send email to [email protected]"
Input to tool: "Allcargo Logistics: I want to analyze item name and quantity in a bar chart: [email protected]"
"""
)
]
prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval. Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query. Please remove "**" from the response.
For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
For datavisualization, user will ask for inventory report of a particular warehouse. Your job is to return the image path to chat interface and display the image as output.
If the proper response is generated, return the response back. Do not iterate again and again.
{{agent_scratchpad}}
Here is the information you need to process:
Question: {{input}}"""
agent_executor = bind_llm(llm,tools,prompt_template)
def ensure_temp_chart_dir():
temp_chart_dir = os.getenv("IMAGE_MAIN_URL")
if not os.path.exists(temp_chart_dir):
os.makedirs(temp_chart_dir)
def clean_gradio_tmp_dir():
tmp_dir = os.getenv("IMAGE_GRADIO_PATH")
if os.path.exists(tmp_dir):
try:
shutil.rmtree(tmp_dir)
except Exception as e:
print(f"Error cleaning up /tmp/gradio/ directory: {e}")
# Define the interface function
max_iterations = 5
iterations = 0
def handle_query(user_question, chatbot, audio=None):
"""
Function to handle the processing of user input with `AgentExecutor.invoke()`.
"""
global current_event, stop_event
# Clear previous stop event and current_event
stop_event.clear()
if current_event and not current_event.done():
chatbot.append(("","A query is already being processed. Please stop it before starting a new one."))
return gr.update(value=chatbot)
# Start the processing in a new thread
current_event = executor.submit(answer_question_thread, user_question, chatbot)
# Periodically check if current_event is done
while not current_event.done():
if stop_event.is_set():
#current_event.task.cancel() # Attempt to cancel the current_event
current_event.set_result((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
current_event.cancel() # Attempt to cancel the current_event
executor.shutdown(wait=False) # Shutdown the executor
print("Current event cancelled")
print(current_event.cancelled())
chatbot.append((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
return gr.update(value=chatbot)
time.sleep(1) # Wait for 1 second before checking again
if current_event.cancelled():
chatbot.append((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
return gr.update(value=chatbot)
else:
try:
user_question1, response_text1 = current_event.result() # Get the result of the completed current_event
print("output")
print(user_question1)
print(response_text1)
chatbot.append((user_question1, response_text1))
return gr.update(value=chatbot)
except Exception as e:
print(f"Error occurred: {e}")
chatbot.append((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
return gr.update(value=chatbot)
def stop_processing(chatbot):
"""
Stops the current processing if it's running.
"""
global current_event, stop_event
if current_event and not current_event.done():
stop_event.set() # Signal the process to stop
current_event.cancel() # Attempt to cancel the current_event
chatbot.append(("Sorry, we encountered an error while processing your request. Please try after some time.",""))
return gr.update(value=chatbot)
# This function is for agent executor invoke with the option of stop
def answer_question_thread(user_question, chatbot,audio=None):
global iterations
iterations = 0
# Ensure the temporary chart directory exists
# ensure_temp_chart_dir()
# Clean the /tmp/gradio/ directory
# clean_gradio_tmp_dir()
# Handle audio input if provided
"""
if audio is not None:
sample_rate, audio_data = audio
audio_segment = AudioSegment(
audio_data.tobytes(),
frame_rate=sample_rate,
sample_width=audio_data.dtype.itemsize,
channels=1
)
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
audio_segment.export(temp_audio_file.name, format="wav")
temp_audio_file_path = temp_audio_file.name
recognizer = sr.Recognizer()
with sr.AudioFile(temp_audio_file_path) as source:
audio_content = recognizer.record(source)
try:
user_question = recognizer.recognize_google(audio_content)
except sr.UnknownValueError:
user_question = "Sorry, I could not understand the audio."
except sr.RequestError:
user_question = "Could not request results from Google Speech Recognition service."
"""
while iterations < max_iterations:
response = agent_executor.invoke({"input": user_question}, config={"callbacks": [langfuse_handler]}, early_stopping_method="generate")
#create_file_HF()
if isinstance(response, dict):
response_text = response.get("output", "")
else:
response_text = response
if "invalid" not in response_text.lower():
break
iterations += 1
if iterations == max_iterations:
return user_question , "Sorry, I couldn't complete your request" #"The agent could not generate a valid response within the iteration limit."
if os.getenv("IMAGE_PATH") in response_text:
# Open the image file
img = Image.open(os.getenv("IMAGE_PATH"))
# Convert the PIL Image to a base64 encoded string
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
img = f'<img src="data:image/png;base64,{img_str}" style="width:450px; height:400px;">'
response_text = response.get("output", "").split(".")[0] + img
email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
match = re.search(email_pattern, user_question)
if match:
user_email = match.group() # Return the matched email
# email send
if len(user_email) > 0:
# Send email with the chart image attached
send_email_with_attachment_mailjet(
recipient_email=user_email,
subject="Warehouse Inventory Report",
body=response.get("output", "").split(".")[0] + ". This is an auto-generated email containing a chart created using Generative AI.",
# attachment_path=chart_path
attach_img_base64=img_str)
if "send email to" in user_question:
try:
os.remove(img) # Clean up the temporary image file
except Exception as e:
print(f"Error cleaning up image file: {e}")
except Exception as e:
print(f"Error loading image file: {e}")
response_text = "Chart generation failed. Please try again."
return user_question, response_text
else:
return user_question, response_text
# response_text = response_text.replace('\n', ' ').replace(' ', ' ').strip()
# return response_text
# without forceful stop option
def answer_question(user_question, chatbot, audio=None):
global iterations
iterations = 0
# Ensure the temporary chart directory exists
# ensure_temp_chart_dir()
# Clean the /tmp/gradio/ directory
# clean_gradio_tmp_dir()
# Handle audio input if provided
if audio is not None:
sample_rate, audio_data = audio
audio_segment = AudioSegment(
audio_data.tobytes(),
frame_rate=sample_rate,
sample_width=audio_data.dtype.itemsize,
channels=1
)
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
audio_segment.export(temp_audio_file.name, format="wav")
temp_audio_file_path = temp_audio_file.name
recognizer = sr.Recognizer()
with sr.AudioFile(temp_audio_file_path) as source:
audio_content = recognizer.record(source)
try:
user_question = recognizer.recognize_google(audio_content)
except sr.UnknownValueError:
user_question = "Sorry, I could not understand the audio."
except sr.RequestError:
user_question = "Could not request results from Google Speech Recognition service."
while iterations < max_iterations:
response = agent_executor.invoke({"input": user_question}, config={"callbacks": [langfuse_handler]})
if isinstance(response, dict):
response_text = response.get("output", "")
else:
response_text = response
if "invalid" not in response_text.lower():
break
iterations += 1
if iterations == max_iterations:
return "The agent could not generate a valid response within the iteration limit."
if os.getenv("IMAGE_PATH") in response_text:
# Open the image file
img = Image.open(os.getenv("IMAGE_PATH"))
# Convert the PIL Image to a base64 encoded string
buffered = BytesIO()
img.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
img = f'<img src="data:image/png;base64,{img_str}" style="width:450px; height:400px;">'
chatbot.append((user_question, img))
email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
match = re.search(email_pattern, user_question)
if match:
user_email = match.group() # Return the matched email
# email send
if len(user_email) > 0:
# Send email with the chart image attached
send_email_with_attachment_mailjet(
recipient_email=user_email,
subject="Warehouse Inventory Report",
body=response.get("output", "").split(".")[0],
# attachment_path=chart_path
attachment_path=img_str)
# Send email with the chart image attached
"""send_email_with_attachment(
recipient_email=user_email,
subject="Warehouse Inventory Report",
body=response.get("output", "").split(":")[0],
# attachment_path=chart_path
attachment_path=os.getenv("IMAGE_PATH")
)"""
if "send email to" in user_question:
try:
os.remove(img) # Clean up the temporary image file
except Exception as e:
print(f"Error cleaning up image file: {e}")
except Exception as e:
print(f"Error loading image file: {e}")
chatbot.append((user_question, "Chart generation failed. Please try again."))
return gr.update(value=chatbot)
else:
chatbot.append((user_question, response_text))
return gr.update(value=chatbot)
def submit_feedback(feedback, chatbot, request: gr.Request):
gr.Info("Thank you for your feedback.")
#save feedback with user question and response in database
save_feedback(request.username,chatbot[-1][0], chatbot[-1][1], feedback)
feedback_response = "User feedback: " + feedback
return chatbot + [(feedback_response, None)], gr.update(visible=False), gr.update(visible=False)
# Function to connect to MySQL database
def connect_to_db():
return mysql.connector.connect(
host=DB_HOST,
user=DB_USER,
password=DB_PASSWORD,
database=DB_NAME
)
# Function to save feedback to the database
def save_feedback(username, user_question, user_response, feedback):
try:
conn = connect_to_db()
cursor = conn.cursor()
query = "INSERT INTO user_feedback (username, question, response, feedback) VALUES (%s, %s, %s, %s)"
cursor.execute(query, (username, user_question, user_response, feedback))
conn.commit()
except mysql.connector.Error as err:
print(f"Error: {err}")
finally:
if cursor:
cursor.close()
if conn:
conn.close()
def handle_dislike(data: gr.LikeData):
if not data.liked:
print("downvote")
gr.Info("Please enter your feedback.")
return gr.update(visible=True), gr.update(visible=True)
else:
print("upvote")
return gr.update(visible=False), gr.update(visible=False)
# greet with user name on successful login
def update_message(request: gr.Request):
global user_name
user_name = request.username
return f"<h2 style=' font-family: Calibri;'>Welcome, {request.username}</h4>"
# Function to generate a 50-word summary of the newly uploaded doc using OpenAI
def generate_summary(text):
prompt = (
"You are an AI that helps with document analysis. Please provide a concise title and a summary of the following document. "
"The summary should be about 50 words and include key details that can help answer questions accurately:\n\n"
f"{text}\n\nTitle : Summary"
)
# Call the OpenAI API to generate a summary
response = openai.chat.completions.create(
messages=[
{
"role": "user",
"content": prompt,
}
],
model="gpt-4o-mini",
)
# Extract the title and summary from the response
response_content = response.choices[0].message.content
lines = response_content.split("\n")
# Extract title
title_line = lines[0]
title = title_line.split("**Title:**")[-1].strip()
# Extract summary
summary_line = lines[2]
summary = summary_line.split("**Summary:**")[-1].strip()
return title, summary
#function to handle file upload decide whether excel or doc is uploaded and respective tool will be created with appropriate prompts at runtime
def upload_file(filepath):
global vector_store1, file_extension
# Get the file extension
_, file_extension = os.path.splitext(filepath)
if file_extension == ".pdf":
texts1 = load_and_split_pdf(filepath)
vector_store1 = create_vector_store(texts1)
# Generate a 50-word summary from the extracted text
title, summary = generate_summary(texts1)
#return title, summary, file_extension
success_msg = add_to_redmindgpt(title, summary)
elif file_extension == ".xlsx":
title, prompt = process_excel(filepath)
#return title, prompt
success_msg = add_to_redmindgpt(title, prompt)
return success_msg
def generate_example_questions(sheet_name, column_headers):
"""
Generates natural language questions based on column headers.
Args:
sheet_name (str): The name of the Excel sheet.
column_headers (list): List of column headers from the sheet.
Returns:
questions (list): List of generated questions based on the columns.
"""
questions = []
# Check for typical columns and create questions
if 'Product Name' in column_headers or 'Product' in column_headers:
questions.append(f"What is the total sales for a specific product in {sheet_name}?")
if 'Sales Amount' in column_headers or 'Amount' in column_headers:
questions.append(f"What is the total sales amount for a specific region in {sheet_name}?")
if 'Region' in column_headers:
questions.append(f"Which region had the highest sales in {sheet_name}?")
if 'Date' in column_headers:
questions.append(f"What were the total sales during a specific month in {sheet_name}?")
if 'Price' in column_headers:
questions.append(f"What is the price of a specific product in {sheet_name}?")
if any(fnmatch.fnmatch(header, 'Employee*') for header in column_headers):
questions.append(f"What are the details of the distinct broker names?")
return questions
def generate_prompt_from_excel_file(df_dict):
"""
Generates a prompt from an Excel file containing multiple sheets.
Args:
excel_file_path (str): The path to the Excel file.
Returns:
prompt (str): A detailed prompt including sheet names, column headers, sample data,
and example questions for each sheet.
"""
# Initialize prompt with basic structure
prompt = "You have been provided with an Excel file containing data in several sheets.\n"
# Loop through each sheet to extract column headers and sample data
for sheet_name, sheet_df in df_dict.items():
# Extract column headers
column_headers = list(sheet_df.columns)
# Get a sample of the data (first few rows)
sample_data = sheet_df.head(3).to_string(index=False)
# Add sheet details to the prompt
prompt += f"For the sheet '{sheet_name}', the column headers are:"
prompt += f"{', '.join(column_headers)}\n\n"
#prompt += f"Example data from sheet '{sheet_name}':\n"
#prompt += f"{sample_data}\n\n"
# Generate example natural language questions based on columns
example_questions = generate_example_questions(sheet_name, column_headers)
#prompt += "### Example Questions:\n"
#for question in example_questions:
# prompt += f"- {question}\n"
#prompt += "\n"
# Finalize the prompt with function call description
prompt += f"- Query: A natural language question (e.g., List all the employees whose names start with 'A'). The question should be sent as 'What are the employee details with name starts with a'."
prompt += f"""Output : {docstatus}. Here is the sample table:
{sample_table}.
"""
prompt += f"- Query: A natural language question with request to create LOA document (e.g., can you create LOA document for all the employees with broker name XXXX). The question should be sent as 'What are the employee details with broker name XXXX : LOA document'."
prompt += f"""Output: {docstatus}. Here is the sample table:
{sample_table}.
If there is any error, please display the message returned by the function as response. """
return "Excel data", prompt
# Function to handle "Add to RedMindGPT" button click
def add_to_redmindgpt(title, summary):
"""
Adds a document or Excel file to the RedmindGPT system and configures the appropriate runtime tool for handling related queries.
Parameters:
title (str): The title of the document or Excel file.
summary (str): A brief summary of the document or Excel file.
Returns:
str: A message indicating whether the file has been added successfully.
Behavior:
- If the file extension is ".pdf", it sets up a runtime tool for handling document-related queries.
- If the file extension is ".xlsx", it sets up a runtime tool for handling Excel data-related queries.
- Configures the prompt template for the agent executor based on the file type.
- Adds the configured runtime tool to the list of tools used by the agent executor.
"""
global agent_executor, file_extension
if file_extension == ".pdf":
run_time_tool_summary = f"For {title} document related questions, Please refer runtimeDocumentData tool. {summary}. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points."
run_time_tool = StructuredTool(
func=document_data_tool_runtime,
name="runtimeDocumentData",
args_schema=QueryInput,
output_schema=QueryOutput,
description=f"You are an AI assistant trained to help with the questions based on the uploaded document {title}. {summary}. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points."
)
# Add the new tool to the beginning
tools.insert(0, run_time_tool)
prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval. Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
{run_time_tool_summary}
For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query. Please remove "**" from the response.
For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
For datavisualization, user will ask for inventory report of a particular warehouse. Your job is to return the image path to chat interface and display the image as output.
{{agent_scratchpad}}
Here is the information you need to process:
Question: {{input}}"""
agent_executor = bind_llm(llm,tools,prompt_template)
return f"File has been added successfully."
elif file_extension == ".xlsx":
run_time_excel_tool_summary = f"For {title} related questions, Please refer runtimeExcelData tool. {summary}. Display the response only in the format as mentioned in the tool description. "
run_time_excel_tool = StructuredTool(
func=chat_with_excel_data_dataframe,
name="runtimeExcelData",
args_schema=QueryInput,
output_schema=QueryOutput,
description=f"""You are an AI assistant trained to handle Excel data and return meaningful insights. If user query is given with an option of generating the document with the result set dataframe, pass two inputs to the tool. First input is the user query and the second input will be the phrase "create document". display the response only in the below format.
{docstatus}. Here is the sample data:
{sample_table}.
If there is any error, please display the message returned by the function as response. """
)
# Add the new tool to the beginning
tools.insert(0, run_time_excel_tool)
prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval. Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
{run_time_excel_tool_summary}
For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query. Please remove "**" from the response.
For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
For datavisualization, user will ask for inventory report of a particular warehouse. Your job is to return the image path to chat interface and display the image as output.
{{agent_scratchpad}}
Here is the information you need to process:
Question: {{input}}"""
agent_executor = bind_llm(llm,tools,prompt_template)
return f"File has been added successfully."
def process_excel(file):
global excel_dataframe
# Check if the file is None
if file is None:
return "Excel file", "Your excel does not have values. Please upload a different file." # Return an empty dataframe if no file is uploaded
else:
# Read the uploaded Excel file
excel_dataframe = pd.read_excel(file.name, sheet_name=None) # 'file.name' to get the actual file path
#to get title and summary of excel file
title, prompt = generate_prompt_from_excel_file(excel_dataframe)
excel_dataframe = pd.read_excel(file.name)
return title, prompt # Return the success message.
def chat_with_excel_data(question):
global excel_dataframe
response_dataframe = chat_with_llm(excel_dataframe,question)
print(response_dataframe)
return response_dataframe
def chat_with_excel_data_dataframe(question):
isDataFrame = True
print(f"question for excel data frame : {question}")
if "LOA" in question:
#question = question.replace("create document", "").strip()
create_document = True
else:
create_document = False
print(f"create document : {create_document}")
response_dataframe = chat_with_excel_data(question)
if isinstance(response_dataframe, pd.DataFrame) == False:
print("The result is not a DataFrame.")
if ":" in response_dataframe:
isDataFrame = False
names_part = response_dataframe.split(":", 1)[1] # Get everything after the colon and space
# Split the names by commas to create a list
names = names_part.split(",")
# Convert the list of names to a DataFrame
response_dataframe = pd.DataFrame(names, columns=["Result"])
#handle large dataset
response = handle_large_dataset(response_dataframe, create_document,isDataFrame)
return response
#Save the respnse dataframe to an Excel file in hostinger so that the user can download it
#save_file_path = "dataframe_output.xlsx"
#response_dataframe.to_excel(save_file_path, index=False)
#save_file_to_hostinger(save_file_path)
#return response_dataframe.head(10)#, len(response_dataframe)
def save_file_to_hostinger(save_file_path):
from ftplib import FTP
# Step 2: FTP server credentials
ftp_host = 'ftp.redmindtechnologies.com' # Replace with your FTP server address
ftp_user = 'u852023448.redmindGpt' # Replace with your FTP username
ftp_pass = 'RedMind@505' # Replace with your FTP password
remote_file_path = '/RedMindGPT/output.xlsx' # Replace with the desired path on the server
# Create an FTP connection
ftp = FTP(ftp_host)
ftp.login(ftp_user, ftp_pass)
# Open the local file and upload it to the server
with open(save_file_path, 'rb') as file:
ftp.storbinary(f'STOR {remote_file_path}', file)
print(f'File {save_file_path} uploaded to {remote_file_path} on server.')
# Close the FTP connection
ftp.quit()
def handle_large_dataset(df, create_document,isDataFrame):
total_rows = len(df)
#print(df)
print(f"Total rows: {total_rows}")
#docstatus = f"Download the complete dataset <a href='https://huggingface.co/spaces/Redmind/NewageNXTGPT/resolve/main/assets/output.xlsx' download> here.</a>.There are total of {total_rows} rows."
if total_rows < 4000:
# 1. Drop the original first column
if len(df.columns) > 1:
# Skipping the original first column
# List of required columns
required_columns = ['BROKER', 'ACCOUNT NUMBER', 'EMPLOYEE NAME', 'ACCOUNT NAME', 'ACCOUNT ID']
# Filter the DataFrame to include only the required columns
#print(df[required_columns])
#limited_data = df[required_columns]
limited_data11 = df.head(3)
limited_data = limited_data11[required_columns]
limited_data_without_first_column = limited_data.iloc[:, 1:]
else:
limited_data = df.head(20)
limited_data_without_first_column = limited_data
# 2. Add SNo (serial number) as the first column, starting from 1
if isDataFrame :
limited_data_without_first_column.insert(0, 'SNo', range(1, len(limited_data_without_first_column) + 1))
else:
limited_data_without_first_column.insert(0, 'SNo', range(1, len(limited_data) + 1))
# 3. Save the full dataset to a downloadable file
file_path = "output_data.xlsx"
#file_path = os.path.join(current_folder, 'output_data.xlsx')
#Broker Name, Account Number, Employee name,Account Owner,Account ID
df.to_excel(file_path, index=False)
global user_name
# Get today's date and current time
now = datetime.now()
# Format the date and time as 'YYYY-MM-DD HH:MM'
formatted_date_time = now.strftime("%Y-%m-%d %H:%M")
print(formatted_date_time)
directory = user_name + "/" + formatted_date_time
create_file_HF(file_path, directory,False)
dataset_link = get_download_link(directory,file_path)
docstatus = f"""These are the sample data. Download the complete dataset <a href="{dataset_link}" download> here.</a>."""
print("File created in the home path")
#save_file_to_hostinger('output_data.xlsx')
# 4. Create a summary and table of the first 10 rows for display
#columns = list(df.columns)
sample_table = limited_data_without_first_column.to_markdown()
#print(sample_table)
if create_document:
#Logic to generate pdfs with employee name and account number
for index, row in df.iterrows():
# Create a PDF for each row
create_pdf(row['COMPANY'], row['EMPLOYEE NAME'], row['ACCOUNT NUMBER'],directory)
create_document = False
# Directory containing PDF files
pdf_dir = "/home/user/app"
# Output ZIP file name
zip_file_name = "pdf_files.zip"
print(zip_file_name)
zip_files_in_folder(directory, zip_file_name)
create_file_HF(zip_file_name,directory, False)
link = get_download_link(directory,zip_file_name)
print(f"downloadable link: {link}")
docstatus = f"""Please download <a href="{dataset_link}" download>excel</a> and <a href="{link}" download>PDFs</a>."""
print(docstatus)
print(sample_table)
# 5. Return the summary and downloadable link
#return f"""
#There are a total of {total_rows} rows. Please download the complete dataset here: <a href="https://redmindtechnologies.com/RedMindGPT/output.xlsx" download>Download</a>. Here are the first 3 rows:
#{sample_table} """
return sample_table, docstatus
else:
return "Your query returns a large dataset which is not supported in the current version. Please try a different query."
def create_file_HF(file_path,directory,document_created = False):
repo_id = "Redmind/NewageNXTGPT_Repo_trial"
if(document_created):
directory = directory + "/" + "document"
document_created = False
#check if direcotry exists
dir_exists_flag = directory_exists(repo_id, directory, token)
if not dir_exists_flag:
directory = directory + "/" + file_path
else:
directory = directory + "/" + file_path
#create_branch("Redmind/NewageNXTGPT_Repo_trial", repo_type="space", branch="test-branch")
#create_tag("bigcode/the-stack", repo_type="dataset", revision="v0.1-release", tag="v0.1.1", tag_message="Bump release version.")
api.upload_file(path_or_fileobj=file_path, repo_id="Redmind/NewageNXTGPT_Repo_trial",revision = "test-branch", repo_type= "space", path_in_repo=directory)
def create_pdf(cname,ename,account_number, directory):
filled = FormWrapper("Input/LOA_Sample_new.pdf").fill(
{
'company name': cname,
'employee name': ename,
'account number': account_number
},
)
output_file_name = f"{ename}_{cname}.pdf"
document_created = True
with open(output_file_name, "wb+") as output:
output.write(filled.read())
# Get the absolute path
file_path = os.path.abspath(output_file_name)
print(f"The file was created at: {file_path}")
#create_file_HF(output_file_name, directory,document_created)
return f"{output_file_name} is created successfully."
def zip_files_in_folder(directory_output, output_zip):
# Initialize the ZIP file
directory = "./"
with zipfile.ZipFile(output_zip, 'w', zipfile.ZIP_DEFLATED) as zipf:
# List all files in the directory
for file in os.listdir(directory):
file_path = os.path.join(directory, file)
# Check if it's a PDF file and a regular file (not a directory)
if file.endswith(".pdf") and os.path.isfile(file_path):
# Add file to the ZIP archive
zipf.write(file_path, file) # Save with its filename in the ZIP
print(f"ZIP file created: {output_zip}, {output_zip}, {directory_output}")
def directory_exists(repo_id, directory, token):
try:
from datetime import datetime
# List all files in the repository
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=token)
# Check if any file starts with the directory path
return any(file.startswith(directory) for file in files)
except Exception as e:
print(f"Error checking directory existence: {e}")
return False
def get_download_link(file_path,file_name):
# Hugging Face Spaces uses this URL format for repository-hosted files
base_url = "https://huggingface.co/spaces/Redmind/NewageNXTGPT_Repo_trial/resolve/test-branch"
# Replace <your-space-id> with your actual Space name
return f"{base_url}/{file_path}/{file_name}"
css = """
/* Example of custom button styling */
.gr-button {
background-color: #6366f1; /* Change to your desired button color */
color: white;
border-radius: 8px; /* Make the corners rounded */
border: none;
padding: 10px 20px;
font-size: 12px;
cursor: pointer;
}
.gr-button:hover {
background-color: #8a92f7; /* Darker shade on hover */
}
.gr-buttonbig {
background-color: #6366f1; /* Change to your desired button color */
color: white;
border-radius: 8px; /* Make the corners rounded */
border: none;
padding: 10px 20px;
font-size: 14px;
cursor: pointer;
}
.gr-buttonbig:hover {
background-color: #8a92f7; /* Darker shade on hover */
}
/* Customizing the Logout link to be on the right */
.logout-link {
text-align: right;
display: inline-block;
width: 100%;
}
.logout-link a {
color: #4A90E2; /* Link color */
text-decoration: none;
font-size: 16px;
}
.chatbot_gpt {
height: 600px !important; /* Adjust height as needed */
}
.logout-link a:hover {
text-decoration: underline; /* Underline on hover */
}
.message-buttons-right{
display: none !important;
}
body, .gradio-container {
margin: 0;
padding: 0;
}
/* Styling the tab header with a blue background */
.gr-tab-header {
background-color: #4A90E2; /* Blue background for the tab header */
padding: 10px;
border-radius: 8px;
color: white;
font-size: 16px;
}
/* Styling the selected tab text color to be green */
.gr-tab-header .gr-tab-active {
color: green; /* Change selected tab text to green */
}
/* Keep non-selected tab text color white */
.gr-tab-header .gr-tab {
color: white;
}
/* Custom CSS for reducing the size of the video element */
.video-player {
width: 500px; /* Set a custom width for the video */
height: 350px; /* Set a custom height for the video */
margin: 0 auto; /* Center the video horizontally */
}
"""
with gr.Blocks(css=css, theme=gr.themes.Soft()) as demo:
gr.HTML("<CENTER><B><h1 style='font-size:30px; font-family: Calibri;'>RedMindGPT</h1></B></CENTER>")
# Logout link styled as text link in the right corner
gr.Markdown("<div class='logout-link'><a href='/logout'><b>Logout</b></a></div>")
# Unified RedMindGPT Interface with sample ques
with gr.Row():
m = gr.Markdown()
demo.load(update_message, None, m)
# Buttons for sample queries
with gr.Row():
sample_button = gr.Button("What are the details of ASN24091600002", elem_classes="gr-buttonbig")
sample_button1 = gr.Button("What are the active warehouses available", elem_classes="gr-buttonbig")
sample_button2 = gr.Button("Explain Pre-Receiving Yard Management", elem_classes="gr-buttonbig")
sample_button3 = gr.Button("can you generate a doughnut chart with item name and quantities for warehouse WH1000001", elem_classes="gr-buttonbig")
sample_button4 = gr.Button("Analyze item name & quantity for different customers in a stacked bar chart for the warehouse WH1000001 & send email to [email protected]", elem_classes="gr-button")
# Chatbot component
with gr.Row():
chatbot = gr.Chatbot(label="Select any of the questions listed above to experience RedMindGPT in action.", elem_classes="chatbot_gpt")
# Textbox for user questions
with gr.Row():
with gr.Column(scale=1):
message = gr.Textbox(show_label=False, container=False, placeholder="Please enter your question")
with gr.Row():
feedback_textbox = gr.Textbox(visible=False, show_label=False, container=False, placeholder="Please enter your feedback.")
submit_feedback_button = gr.Button("Submit Feedback", visible=False, elem_classes="gr-buttonbig")
with gr.Column(scale=1):
with gr.Row():
button = gr.Button("Submit", elem_id="submit", elem_classes="gr-buttonbig")
stop_button = gr.Button("Stop", elem_classes="gr-buttonbig")
# Rearranged to place Upload Doc and Upload Excel in the same row
with gr.Row():
with gr.Column(scale=1):
# File Upload Section
gr.Markdown("**Add a document or Excel for natural language interaction.**")
with gr.Column(scale=1):
u = gr.UploadButton("Upload a doc/excel", file_count="single", elem_classes="gr-buttonbig")
#excel_file = gr.UploadButton("Upload an excel", file_count="single", elem_classes="gr-buttonbig", file_types=[".xlsx", ".xls"])
with gr.Column(scale=1):
add_button = gr.Button("Add to RedMindGPT", elem_classes="gr-buttonbig", visible=False)
with gr.Row():
title_textbox = gr.Textbox(label="Title", visible=False)
summary_textarea = gr.Textbox(label="Summary", lines=5, visible=False)
output_message = gr.Markdown() # Markdown to display output message
success_message = gr.Markdown() # Placeholder for messages
# Moved function calling lines to the end
stop_button.click(stop_processing, [chatbot], [chatbot])
button.click(handle_query, [message, chatbot], [chatbot])
message.submit(handle_query, [message, chatbot], [chatbot])
message.submit(lambda x: gr.update(value=""), None, [message], queue=False)
button.click(lambda x: gr.update(value=''), [], [message])
chatbot.like(handle_dislike, None, outputs=[feedback_textbox, submit_feedback_button])
submit_feedback_button.click(submit_feedback, [feedback_textbox, chatbot], [chatbot, feedback_textbox, submit_feedback_button])
submit_feedback_button.click(lambda x: gr.update(value=''), [], [feedback_textbox])
sample_button.click(handle_query, [sample_button, chatbot], [chatbot])
sample_button1.click(handle_query, [sample_button1, chatbot], [chatbot])
sample_button2.click(handle_query, [sample_button2, chatbot], [chatbot])
sample_button3.click(handle_query, [sample_button3, chatbot], [chatbot])
sample_button4.click(handle_query, [sample_button4, chatbot], [chatbot])
"""u.upload(upload_file, u, [title_textbox, summary_textarea])
u.upload(lambda _: (gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)), None, [title_textbox, summary_textarea, add_button])
add_button.click(add_to_redmindgpt, [title_textbox, summary_textarea], output_message)
add_button.click(lambda _: (gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)), None, [title_textbox, summary_textarea, add_button])
"""
u.upload(upload_file, u, output_message)
u.upload(lambda _: (gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)), None, [title_textbox, summary_textarea, add_button])
demo.launch(auth=[("lakshmi", "redmind"), ("admin", "redmind"), ("arun", "redmind"), ("NewageGlobal", "Newage123$")], auth_message="RedMindGPT", inline=False)
|