DevashishBhake commited on
Commit
df4e7d2
·
1 Parent(s): cf03eb1

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +49 -0
app.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from ctransformers import AutoModelForCausalLM
3
+
4
+ USER_NAME = "User"
5
+ BOT_NAME = "Falcon"
6
+ DEFAULT_INSTRUCTIONS = f"""The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Falcon, and a human user, called User. In the following interactions, User and Falcon will converse in natural language, and Falcon will answer User's questions. Falcon was built to be respectful, polite and inclusive. Falcon was built by the Technology Innovation Institute in Abu Dhabi. Falcon will never decline to answer a question, and always attempts to give an answer that User would be satisfied with. It knows a lot, and always tells the truth. The conversation begins."""
7
+ STOP_STR = f"\n{USER_NAME}:"
8
+ STOP_SUSPECT_LIST = [":", "\n", "User"]
9
+ temperature = 0.8
10
+ top_p = 0.9
11
+ llm = AutoModelForCausalLM.from_pretrained("TheBloke/falcon-40b-instruct-GGML", model_file="falcon40b-instruct.ggmlv3.q2_K.bin",
12
+ model_type="falcon", threads=8)
13
+ def format_chat_prompt(message: str, instructions: str) -> str:
14
+ instructions = instructions.strip(" ").strip("\n")
15
+ prompt = instructions
16
+ prompt = f"{prompt}\n{USER_NAME}: {message}\n{BOT_NAME}:"
17
+ return prompt
18
+
19
+ def run_chat(message: str):
20
+ prompt = format_chat_prompt(message, DEFAULT_INSTRUCTIONS)
21
+ stream = llm(
22
+ prompt,
23
+ max_new_tokens=1024,
24
+ stop=[STOP_STR, "<|endoftext|>", USER_NAME],
25
+ temperature=temperature,
26
+ top_p=top_p,
27
+ stream=True
28
+ )
29
+ acc_text = ""
30
+ for idx, response in enumerate(stream):
31
+ text_token = response
32
+
33
+ if text_token in STOP_SUSPECT_LIST:
34
+ acc_text += text_token
35
+ continue
36
+
37
+ if idx == 0 and text_token.startswith(" "):
38
+ text_token = text_token[1:]
39
+
40
+ acc_text += text_token
41
+ return acc_text
42
+
43
+ demo = gr.Interface(
44
+ fn=run_chat,
45
+ inputs=gr.inputs.Textbox(label="Message"),
46
+ outputs=gr.outputs.Textbox(label="Generated Text"),
47
+ )
48
+
49
+ demo.launch()