| import os | |
| import pickle | |
| from langchain.document_loaders import UnstructuredURLLoader | |
| from langchain.text_splitter import CharacterTextSplitter | |
| from InstructorEmbedding import INSTRUCTOR | |
| from langchain.vectorstores import FAISS | |
| from langchain.embeddings import HuggingFaceInstructEmbeddings | |
| from langchain.chains import RetrievalQA | |
| from langchain import HuggingFaceHub | |
| from langchain.utilities import GoogleSerperAPIWrapper | |
| import gradio as gr | |
| class Chatbot: | |
| def __init__(self): | |
| os.environ["Hugging_Face_API_KEY"] = "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" | |
| os.environ["HUGGINGFACEHUB_API_TOKEN"] = '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' | |
| os.environ["SERPER_API_KEY"] = "a69857e460dd51585e009a43743711b110b6beee" | |
| self.load_data() | |
| self.load_embeddings() | |
| self.create_qa_model() | |
| def load_data(self): | |
| urls = ['https://www.iimrohtak.ac.in/job.php', | |
| 'https://www.iimrohtak.ac.in/mandatory-disclosure.php', | |
| 'https://www.iimrohtak.ac.in/placement.php', | |
| 'https://www.iimrohtak.ac.in/tender.php', | |
| 'https://www.iimrohtak.ac.in/rti.php', | |
| 'https://www.iimrohtak.ac.in/library.php', | |
| 'https://www.iimrohtak.ac.in/index.php', | |
| 'https://www.iimrohtak.ac.in/index.php', | |
| 'https://www.iimrohtak.ac.in/about.php', | |
| 'https://www.iimrohtak.ac.in/vision.php', | |
| 'https://www.iimrohtak.ac.in/mission.php', | |
| 'https://www.iimrohtak.ac.in/board.php', | |
| 'https://www.iimrohtak.ac.in/director-message.php', | |
| 'https://www.iimrohtak.ac.in/accreditation.php', | |
| 'https://www.iimrohtak.ac.in/logo.php', | |
| 'https://www.iimrohtak.ac.in/nirf.php', | |
| 'https://www.iimrohtak.ac.in/iim-rohtak-values.php', | |
| 'https://www.iimrohtak.ac.in/council-for-strategic-affairs.php', | |
| 'https://www.iimrohtak.ac.in/mandatory-disclosure.php', | |
| 'https://www.iimrohtak.ac.in/ranking.php', | |
| 'https://www.iimrohtak.ac.in/annual-report.php', | |
| 'https://www.iimrohtak.ac.in/event.php', | |
| 'https://www.iimrohtak.ac.in/event-gallery.php', | |
| 'https://www.iimrohtak.ac.in/job.php', | |
| 'https://www.iimrohtak.ac.in/initiative.php', | |
| 'https://www.iimrohtak.ac.in/iimr-in-media.php', | |
| 'https://www.iimrohtak.ac.in/rti.php', | |
| 'https://www.iimrohtak.ac.in/internal-complaint-commitee.php', | |
| 'https://www.iimrohtak.ac.in', | |
| 'https://www.iimrohtak.ac.in/pgp.php', | |
| 'https://www.iimrohtak.ac.in/pgp-first-year.php', | |
| 'https://www.iimrohtak.ac.in/pgp-second-year.php', | |
| 'https://www.iimrohtak.ac.in/objective-methodology.php', | |
| 'https://www.iimrohtak.ac.in/programme-fee.php', | |
| 'https://www.iimrohtak.ac.in/academic-calendar.php', | |
| 'https://www.iimrohtak.ac.in/pgp-contact.php', | |
| 'https://www.iimrohtak.ac.in/placement.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/prospectus/16891449485032.pdf', | |
| 'https://www.iimrohtak.ac.in/dpm.php', | |
| 'https://www.iimrohtak.ac.in/dpm.php', | |
| 'https://www.iimrohtak.ac.in/dpm-admission.php', | |
| 'https://www.iimrohtak.ac.in/areas-of-specialisation.php', | |
| 'https://www.iimrohtak.ac.in/financial-assistance.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/prospectus/16903487969776.pdf', | |
| 'https://www.iimrohtak.ac.in/faqs-for-dpm.php', | |
| 'https://www.iimrohtak.ac.in/dpm-student.php', | |
| 'https://www.iimrohtak.ac.in/publication.php', | |
| 'https://www.iimrohtak.ac.in/dpm-contact.php', | |
| 'https://www.iimrohtak.ac.in/ipm.php', | |
| 'https://www.iimrohtak.ac.in/ipm.php', | |
| 'https://www.iimrohtak.ac.in/faculty-speaks.php', | |
| 'https://www.iimrohtak.ac.in/ipm-curriculum.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/prospectus/16881248909525.pdf', | |
| 'https://www.iimrohtak.ac.in/ipm-admission.php', | |
| 'https://www.iimrohtak.ac.in/ipm-contact.php', | |
| 'https://www.iimrohtak.ac.in/ipm-important-date.php', | |
| 'https://www.iimrohtak.ac.in/ipl.php', | |
| 'https://www.iimrohtak.ac.in/ipl-admission.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/programme-fee/16881248477847.pdf', | |
| 'https://www.iimrohtak.ac.in/ipl-faculty.php', | |
| 'https://www.iimrohtak.ac.in/ipl-contactus.php', | |
| 'https://www.iimrohtak.ac.in/ipl-advisory.php', | |
| 'https://www.iimrohtak.ac.in/ipl-curriculum.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/prospectus/16881265522765.pdf', | |
| 'https://www.iimrohtak.ac.in/fpm-about.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/prospectus/16881252023208.pdf', | |
| 'https://www.iimrohtak.ac.in/fpm-about.php', | |
| 'https://www.iimrohtak.ac.in/fpm-admission.php', | |
| 'https://www.iimrohtak.ac.in/fpm-course-work.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/lor/16884755042121.pdf', | |
| 'https://www.iimrohtak.ac.in/fpm-contact.php', | |
| 'https://www.iimrohtak.ac.in/enquiry.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-about.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-about.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-director-message.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-advisory.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-program-curriculum.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-faculty.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-admission.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-contact.php', | |
| 'https://www.iimrohtak.ac.in/enquiry.php', | |
| 'https://www.iimrohtak.ac.in/epgdsm-blog.php', | |
| 'https://www.iimrohtak.ac.in/one-year-post-graduate-programme-in-management-for-executives.php', | |
| 'https://www.iimrohtak.ac.in/executive-training.php', | |
| 'https://www.iimrohtak.ac.in/faculty-by-area.php', | |
| 'https://www.iimrohtak.ac.in/director.php', | |
| 'https://www.iimrohtak.ac.in/dean.php', | |
| 'https://www.iimrohtak.ac.in/dean-ree.php', | |
| 'https://www.iimrohtak.ac.in/faculty-by-name.php', | |
| 'https://www.iimrohtak.ac.in/faculty-by-area.php', | |
| 'https://www.iimrohtak.ac.in/visiting-faculty.php', | |
| 'https://www.iimrohtak.ac.in/adjunct-faculty.php', | |
| 'https://www.iimrohtak.ac.in/publication.php', | |
| 'https://www.iimrohtak.ac.in/publication.php', | |
| 'https://www.iimrohtak.ac.in/papers-presented-in-conferences.php', | |
| 'https://www.iimrohtak.ac.in/broad-research-areas.php', | |
| 'https://www.iimrohtak.ac.in/cases-other-publication.php', | |
| 'https://www.iimrohtak.ac.in/emerging-economies-cases-journal.php', | |
| 'https://www.iimrohtak.ac.in/newspaper-articles-interviews.php', | |
| 'https://www.iimrohtak.ac.in/assets/images/Research Policy.pdf', | |
| 'https://www.iimrohtak.ac.in/partnership.php', | |
| 'https://www.iimrohtak.ac.in/partnership.php', | |
| 'https://www.iimrohtak.ac.in/institutional-relations-committee.php', | |
| 'https://www.iimrohtak.ac.in/international-partnerships.php', | |
| 'https://www.iimrohtak.ac.in/student-exchange.php', | |
| 'https://www.iimrohtak.ac.in/collaborations-contact.php', | |
| 'https://www.iimrohtak.ac.in/membership.php', | |
| 'https://www.iimrohtak.ac.in/contact.php', | |
| 'https://www.iimrohtak.ac.in/about.php', | |
| 'https://www.iimrohtak.ac.in/faculty-by-area.php', | |
| 'https://www.iimrohtak.ac.in/faculty-by-name.php', | |
| 'https://www.iimrohtak.ac.in/iimr-in-media.php', | |
| 'https://www.iimrohtak.ac.in/publication.php', | |
| 'https://www.iimrohtak.ac.in/library.php', | |
| 'https://www.iimrohtak.ac.in/placement.php', | |
| 'https://www.iimrohtak.ac.in/index.php', | |
| 'https://www.iimrohtak.ac.in/job.php', | |
| 'https://www.iimrohtak.ac.in/tender.php', | |
| 'https://www.iimrohtak.ac.in/library.php', | |
| 'https://www.iimrohtak.ac.in/event.php', | |
| 'https://www.iimrohtak.ac.in/iimr-in-media.php', | |
| 'https://www.iimrohtak.ac.in/it-resource-centre.php', | |
| 'https://www.iimrohtak.ac.in/placement.php', | |
| 'https://www.iimrohtak.ac.in/rti.php', | |
| 'https://www.iimrohtak.ac.in/contact.php', | |
| 'https://www.iimrohtak.ac.in/privacy-policy.php', | |
| 'https://www.iimrohtak.ac.in/panel/assets/images/admission-policy/admissionpolicy2023-25.pdf', | |
| 'https://www.iimrohtak.ac.in/annual-report.php', | |
| 'https://www.iimrohtak.ac.in/students-clubs-and-committees.php', | |
| 'https://www.iimrohtak.ac.in/disclaimer.php', | |
| 'https://www.iimrohtak.ac.in/event-gallery.php', | |
| 'https://www.iimrohtak.ac.in/enquiry.php'] | |
| loaders = UnstructuredURLLoader(urls=urls) | |
| self.data = loaders.load() | |
| def split_documents(self): | |
| text_splitter = CharacterTextSplitter(separator='\n', chunk_size=500, chunk_overlap=20) | |
| self.docs = text_splitter.split_documents(self.data) | |
| def create_embeddings(self): | |
| instructor_embeddings = HuggingFaceInstructEmbeddings(model_name="sembeddings/model_gpt_trained") | |
| db_instructEmbedd = FAISS.from_documents(self.docs, instructor_embeddings) | |
| self.retriever = db_instructEmbedd.as_retriever(search_kwargs={"k": 3}) | |
| with open("db_instructEmbedd.pkl", "wb") as f: | |
| pickle.dump(db_instructEmbedd, f) | |
| def load_embeddings(self): | |
| with open("db_instructEmbedd.pkl", "rb") as f: | |
| self.retriever = pickle.load(f) | |
| self.retriever = self.retriever.as_retriever(search_kwargs={"k": 3}) | |
| def create_qa_model(self): | |
| llm = HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature": 0.1}) | |
| self.qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=self.retriever, return_source_documents=True) | |
| def search_google(self, query): | |
| search = GoogleSerperAPIWrapper() | |
| search.run(query) | |
| def chat(self, query): | |
| self.search_google(query) | |
| result = self.qa({'query': query}) | |
| return result['result'] | |
| chatbot = Chatbot() | |
| def chatbot_response(query): | |
| response = chatbot.chat(query) | |
| return response | |
| iface = gr.Interface(fn=chatbot_response, inputs="text", outputs="text", title="Chatbot Trained on Indian Exam Articles") | |
| iface.launch() | |