| from datasets import load_dataset | |
| from datasets import Dataset | |
| from sentence_transformers import SentenceTransformer | |
| import faiss | |
| import time | |
| #import torch | |
| import pandas as pd | |
| from transformers import AutoTokenizer, GenerationConfig #, AutoModelForCausalLM | |
| #from transformers import AutoModelForCausalLM, AutoModel | |
| from transformers import TextIteratorStreamer | |
| from threading import Thread | |
| from ctransformers import AutoModelForCausalLM, AutoConfig, Config #, AutoTokenizer | |
| from huggingface_hub import Repository, upload_file | |
| import os | |
| HF_TOKEN = os.getenv('HF_Token') | |
| #Log_Path="./Logfolder" | |
| logfile = 'DiabetesChatLog.txt' | |
| historylog = [{ | |
| "Prompt": '', | |
| "Output": '' | |
| }] | |
| data = load_dataset("Namitg02/Test", split='train', streaming=False) | |
| #Returns a list of dictionaries, each representing a row in the dataset. | |
| length = len(data) | |
| embedding_model = SentenceTransformer("all-MiniLM-L6-v2") | |
| embedding_dim = embedding_model.get_sentence_embedding_dimension() | |
| # Returns dimensions of embedidng | |
| index = faiss.IndexFlatL2(embedding_dim) | |
| data.add_faiss_index("embeddings", custom_index=index) | |
| # adds an index column for the embeddings | |
| print("check1") | |
| #question = "How can I reverse Diabetes?" | |
| SYS_PROMPT = """You are an assistant for answering questions. | |
| You are given the extracted parts of documents and a question. Provide a conversational answer. | |
| If you don't know the answer, just say "I do not know." Don't make up an answer.""" | |
| # Provides context of how to answer the question | |
| llm_model = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF" | |
| # TheBloke/Llama-2-7B-Chat-GGML , TinyLlama/TinyLlama-1.1B-Chat-v1.0 , microsoft/Phi-3-mini-4k-instruct, health360/Healix-1.1B-V1-Chat-dDPO | |
| # TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF and tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf not working, TinyLlama/TinyLlama-1.1B-Chat-v0.6, andrijdavid/TinyLlama-1.1B-Chat-v1.0-GGUF" | |
| tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0") | |
| #initiate model and tokenizer | |
| generation_config = AutoConfig.from_pretrained( | |
| "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", | |
| max_new_tokens= 300, | |
| # do_sample=True, | |
| # stream = streamer, | |
| top_p=0.95, | |
| temperature=0.4, | |
| stream = True | |
| # eos_token_id=terminators | |
| ) | |
| # send additional parameters to model for generation | |
| model = AutoModelForCausalLM.from_pretrained(llm_model, model_file = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", model_type="llama", gpu_layers=0, config = generation_config) | |
| def search(query: str, k: int = 2 ): | |
| """a function that embeds a new query and returns the most probable results""" | |
| embedded_query = embedding_model.encode(query) # create embedding of a new query | |
| scores, retrieved_examples = data.get_nearest_examples( # retrieve results | |
| "embeddings", embedded_query, # compare our new embedded query with the dataset embeddings | |
| k=k # get only top k results | |
| ) | |
| return scores, retrieved_examples | |
| # returns scores (List[float]): the retrieval scores from either FAISS (IndexFlatL2 by default) and examples (dict) format | |
| # called by talk function that passes prompt | |
| #print(scores, retrieved_examples) | |
| def format_prompt(prompt,retrieved_documents,k): | |
| """using the retrieved documents we will prompt the model to generate our responses""" | |
| PROMPT = f"Question:{prompt}\nContext:" | |
| for idx in range(k) : | |
| PROMPT+= f"{retrieved_documents['0'][idx]}\n" | |
| return PROMPT | |
| # Called by talk function to add retrieved documents to the prompt. Keeps adding text of retrieved documents to string taht are retreived | |
| def talk(prompt, history): | |
| k = 2 # number of retrieved documents | |
| scores , retrieved_documents = search(prompt, k) # get retrival scores and examples in dictionary format based on the prompt passed | |
| print(retrieved_documents.keys()) | |
| print("check4") | |
| formatted_prompt = format_prompt(prompt,retrieved_documents,k) # create a new prompt using the retrieved documents | |
| print("check5") | |
| print(retrieved_documents['0']) | |
| print(formatted_prompt) | |
| formatted_prompt = formatted_prompt[:600] # to avoid memory issue | |
| print(formatted_prompt) | |
| messages = [{"role":"system","content":SYS_PROMPT},{"role":"user","content":formatted_prompt}] | |
| # binding the system context and new prompt for LLM | |
| # the chat template structure should be based on text generation model format | |
| print("check6") | |
| streamer = TextIteratorStreamer( | |
| tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True | |
| ) | |
| # stores print-ready text in a queue, to be used by a downstream application as an iterator. removes special tokens in generated text. | |
| # timeout for text queue. tokenizer for decoding tokens | |
| # called by generate_kwargs | |
| terminators = [ | |
| tokenizer.eos_token_id, # End-of-Sequence Token that indicates where the model should consider the text sequence to be complete | |
| tokenizer.convert_tokens_to_ids("<|eot_id|>") # Converts a token strings in a single/ sequence of integer id using the vocabulary | |
| ] | |
| # indicates the end of a sequence | |
| input_ids = tokenizer.apply_chat_template( | |
| messages, | |
| add_generation_prompt=True, | |
| return_tensors="pt" | |
| ) | |
| # preparing tokens for model input | |
| # add_generation_prompt argument tells the template to add tokens that indicate the start of a bot response | |
| # print(input_ids) | |
| # print("check7") | |
| # print(input_ids.dtype) | |
| # generate_kwargs = dict( | |
| # tokens= input_ids) #, | |
| # streamer=streamer, | |
| # do_sample=True, | |
| # eos_token_id=terminators, | |
| # ) | |
| # outputs = model.generate( | |
| # ) | |
| # print(outputs) | |
| # calling the model to generate response based on message/ input | |
| # do_sample if set to True uses strategies to select the next token from the probability distribution over the entire vocabulary | |
| # temperature controls randomness. more renadomness with higher temperature | |
| # only the tokens comprising the top_p probability mass are considered for responses | |
| # This output is a data structure containing all the information returned by generate(), but that can also be used as tuple or dictionary. | |
| # | |
| # print("check10") | |
| # t = Thread(target=model.generate, kwargs=generate_kwargs) | |
| # to process multiple instances | |
| # t.start() | |
| # print("check11") | |
| # start a thread | |
| outputs = [] | |
| print(messages) | |
| print(*messages) | |
| # input_ids = tokenizer(*messages) | |
| print(model.generate(tensor([[ 1, 529, 29989, 5205, 29989]]))) | |
| start = time.time() | |
| NUM_TOKENS=0 | |
| print('-'*4+'Start Generation'+'-'*4) | |
| for token in model.generate(input_ids): | |
| print(model.detokenize(input_ids), end='', flush=True) | |
| NUM_TOKENS+=1 | |
| time_generate = time.time() - start | |
| print('\n') | |
| print('-'*4+'End Generation'+'-'*4) | |
| print(f'Num of generated tokens: {NUM_TOKENS}') | |
| print(f'Time for complete generation: {time_generate}s') | |
| print(f'Tokens per secound: {NUM_TOKENS/time_generate}') | |
| print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms') | |
| #outputtokens = model.generate(input_ids) | |
| print("check9") | |
| #print(outputtokens) | |
| #outputs = model.detokenize(outputtokens, decode = True) | |
| #print(outputs) | |
| # for token in model.generate(input_ids): | |
| # print(model.detokenize(token)) | |
| # outputs.append(model.detokenize(token)) | |
| # output = model.detokenize(token) | |
| # print(outputs) | |
| # yield "".join(outputs) | |
| # print("check12") | |
| pd.options.display.max_colwidth = 800 | |
| print("check13") | |
| # outputstring = ''.join(outputs) | |
| # global historylog | |
| # historynew = { | |
| # "Prompt": prompt, | |
| # "Output": outputstring | |
| # } | |
| # historylog.append(historynew) | |
| # return historylog | |
| # print(historylog) | |
| TITLE = "AI Copilot for Diabetes Patients" | |
| DESCRIPTION = "I provide answers to concerns related to Diabetes" | |
| import gradio as gr | |
| # Design chatbot | |
| demo = gr.ChatInterface( | |
| fn=talk, | |
| chatbot=gr.Chatbot( | |
| show_label=True, | |
| show_share_button=True, | |
| show_copy_button=True, | |
| likeable=True, | |
| layout="bubble", | |
| bubble_full_width=False, | |
| ), | |
| theme="Soft", | |
| examples=[["what is Diabetes? "]], | |
| title=TITLE, | |
| description=DESCRIPTION, | |
| ) | |
| # launch chatbot and calls the talk function which in turn calls other functions | |
| print("check14") | |
| #print(historylog) | |
| #memory_panda = pd.DataFrame(historylog) | |
| #Logfile = Dataset.from_pandas(memory_panda) | |
| #Logfile.push_to_hub("Namitg02/Logfile",token = HF_TOKEN) | |
| demo.launch() |