Spaces:
Runtime error
Runtime error
arjunanand13
commited on
Commit
•
6b3f892
1
Parent(s):
1e9d34c
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import transformers
|
3 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM
|
4 |
+
import accelerate
|
5 |
+
import einops
|
6 |
+
import langchain
|
7 |
+
import xformers
|
8 |
+
import os
|
9 |
+
import bitsandbytes
|
10 |
+
import sentence_transformers
|
11 |
+
import huggingface_hub
|
12 |
+
import torch
|
13 |
+
from torch import cuda, bfloat16
|
14 |
+
from transformers import StoppingCriteria, StoppingCriteriaList
|
15 |
+
from langchain.llms import HuggingFacePipeline
|
16 |
+
from langchain.document_loaders import TextLoader, DirectoryLoader
|
17 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
18 |
+
from langchain.embeddings import HuggingFaceEmbeddings
|
19 |
+
from langchain.vectorstores import FAISS
|
20 |
+
from langchain.chains import ConversationalRetrievalChain
|
21 |
+
from huggingface_hub import InferenceClient
|
22 |
+
|
23 |
+
# Login to Hugging Face using a token
|
24 |
+
# huggingface_hub.login(HF_TOKEN)
|
25 |
+
|
26 |
+
|
27 |
+
"""CPU"""
|
28 |
+
|
29 |
+
# model_config = transformers.AutoConfig.from_pretrained(
|
30 |
+
# model_id,
|
31 |
+
# token=HF_TOKEN,
|
32 |
+
# # use_auth_token=hf_auth
|
33 |
+
# )
|
34 |
+
# model = transformers.AutoModelForCausalLM.from_pretrained(
|
35 |
+
# model_id,
|
36 |
+
# trust_remote_code=True,
|
37 |
+
# config=model_config,
|
38 |
+
# # quantization_config=bnb_config,
|
39 |
+
# token=HF_TOKEN,
|
40 |
+
# # use_auth_token=hf_auth
|
41 |
+
# )
|
42 |
+
# model.eval()
|
43 |
+
# tokenizer = transformers.AutoTokenizer.from_pretrained(
|
44 |
+
# model_id,
|
45 |
+
# token=HF_TOKEN,
|
46 |
+
# # use_auth_token=hf_auth
|
47 |
+
# )
|
48 |
+
# generate_text = transformers.pipeline(
|
49 |
+
# model=self.model, tokenizer=self.tokenizer,
|
50 |
+
# return_full_text=True,
|
51 |
+
# task='text-generation',
|
52 |
+
# temperature=0.01,
|
53 |
+
# max_new_tokens=512
|
54 |
+
# )
|
55 |
+
|
56 |
+
|
57 |
+
|
58 |
+
# define custom stopping criteria object
|
59 |
+
class StopOnTokens(StoppingCriteria):
|
60 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
61 |
+
for stop_ids in stop_token_ids:
|
62 |
+
if torch.eq(input_ids[0][-len(stop_ids):], stop_ids).all():
|
63 |
+
return True
|
64 |
+
return False
|
65 |
+
|
66 |
+
stopping_criteria = StoppingCriteriaList([StopOnTokens()])
|
67 |
+
|
68 |
+
|
69 |
+
|
70 |
+
loader = DirectoryLoader('data2/text/range/0-5000', loader_cls=TextLoader)
|
71 |
+
documents = loader.load()
|
72 |
+
print('len of documents are',len(documents))
|
73 |
+
|
74 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=5000, chunk_overlap=250)
|
75 |
+
all_splits = text_splitter.split_documents(documents)
|
76 |
+
|
77 |
+
length_of_all_splits = len(all_splits)
|
78 |
+
print("Length of all_splits:", length_of_all_splits)
|
79 |
+
|
80 |
+
print (all_splits[0])
|
81 |
+
print("#######################################")
|
82 |
+
print (all_splits[1])
|
83 |
+
print("#######################################")
|
84 |
+
print (all_splits[2])
|
85 |
+
print("#######################################")
|
86 |
+
print (all_splits[3])
|
87 |
+
print("#######################################")
|
88 |
+
print (all_splits[4])
|
89 |
+
|
90 |
+
"""
|
91 |
+
Loading of the LLama3 model
|
92 |
+
"""
|
93 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", None)
|
94 |
+
model_id = 'meta-llama/Meta-Llama-3-8B'
|
95 |
+
device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
|
96 |
+
|
97 |
+
|
98 |
+
"""set quantization configuration to load large model with less GPU memory
|
99 |
+
this requires the `bitsandbytes` library"""
|
100 |
+
bnb_config = transformers.BitsAndBytesConfig(
|
101 |
+
load_in_4bit=True,
|
102 |
+
bnb_4bit_quant_type='nf4',
|
103 |
+
bnb_4bit_use_double_quant=True,
|
104 |
+
bnb_4bit_compute_dtype=bfloat16
|
105 |
+
)
|
106 |
+
|
107 |
+
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct",token=HF_TOKEN)
|
108 |
+
model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto",token=HF_TOKEN,quantization_config=bnb_config) # to("cuda:0")
|
109 |
+
terminators = [
|
110 |
+
tokenizer.eos_token_id,
|
111 |
+
tokenizer.convert_tokens_to_ids("<|eot_id|>")
|
112 |
+
]
|
113 |
+
|
114 |
+
"""
|
115 |
+
Setting up the stop list to define stopping criteria.
|
116 |
+
"""
|
117 |
+
|
118 |
+
stop_list = ['\nHuman:', '\n```\n']
|
119 |
+
|
120 |
+
stop_token_ids = [tokenizer(x)['input_ids'] for x in stop_list]
|
121 |
+
stop_token_ids = [torch.LongTensor(x).to(device) for x in stop_token_ids]
|
122 |
+
|
123 |
+
generate_text = transformers.pipeline(
|
124 |
+
model=model,
|
125 |
+
tokenizer=tokenizer,
|
126 |
+
return_full_text=True, # langchain expects the full text
|
127 |
+
task='text-generation',
|
128 |
+
# we pass model parameters here too
|
129 |
+
stopping_criteria=stopping_criteria, # without this model rambles during chat
|
130 |
+
temperature=0.1, # 'randomness' of outputs, 0.0 is the min and 1.0 the max
|
131 |
+
max_new_tokens=512, # max number of tokens to generate in the output
|
132 |
+
repetition_penalty=1.1 # without this output begins repeating
|
133 |
+
)
|
134 |
+
|
135 |
+
llm = HuggingFacePipeline(pipeline=generate_text)
|
136 |
+
|
137 |
+
|
138 |
+
model_name = "sentence-transformers/all-mpnet-base-v2"
|
139 |
+
model_kwargs = {"device": "cuda"}
|
140 |
+
|
141 |
+
embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs=model_kwargs)
|
142 |
+
|
143 |
+
# storing embeddings in the vector store
|
144 |
+
vectorstore = FAISS.from_documents(all_splits, embeddings)
|
145 |
+
|
146 |
+
chain = ConversationalRetrievalChain.from_llm(llm, vectorstore.as_retriever(), return_source_documents=True)
|
147 |
+
|
148 |
+
chat_history = []
|
149 |
+
|
150 |
+
def format_prompt(query):
|
151 |
+
# Construct a clear and structured prompt to guide the LLM's response
|
152 |
+
prompt = f"""
|
153 |
+
You are a knowledgeable assistant with access to a comprehensive database.
|
154 |
+
I need you to answer my question and provide related information in a specific format.
|
155 |
+
Here's what I need:
|
156 |
+
1. A brief, general response to my question based on related answers retrieved.
|
157 |
+
2. A JSON-formatted output containing:
|
158 |
+
- "question": The original question.
|
159 |
+
- "answer": The detailed answer.
|
160 |
+
- "related_questions": A list of related questions and their answers, each as a dictionary with the keys:
|
161 |
+
- "question": The related question.
|
162 |
+
- "answer": The related answer.
|
163 |
+
Here's my question:
|
164 |
+
{query}
|
165 |
+
Include a brief final answer without additional comments, sign-offs, or extra phrases. Be direct and to the point.
|
166 |
+
"""
|
167 |
+
return prompt
|
168 |
+
|
169 |
+
|
170 |
+
def qa_infer(query):
|
171 |
+
formatted_prompt = format_prompt(query)
|
172 |
+
result = chain({"question": formatted_prompt, "chat_history": chat_history})
|
173 |
+
return result['answer']
|
174 |
+
|
175 |
+
# query = "What` is the best TS pin configuration for BQ24040 in normal battery charge mode"
|
176 |
+
# qa_infer(query)
|
177 |
+
|
178 |
+
EXAMPLES = [" How to use IPU1_0 instead of A15_0 to process NDK in TDA2x-EVM",
|
179 |
+
"Can BQ25896 support I2C interface?",
|
180 |
+
"Does TDA2 vout support bt656 8-bit mode?"]
|
181 |
+
|
182 |
+
demo = gr.Interface(fn=qa_infer, inputs="text",allow_flagging='never', examples=EXAMPLES,
|
183 |
+
cache_examples=False,outputs="text")
|
184 |
+
|
185 |
+
# launch the app!
|
186 |
+
#demo.launch(enable_queue = True,share=True)
|
187 |
+
#demo.queue(default_enabled=True).launch(debug=True,share=True)
|
188 |
+
demo.launch()
|