Spaces:
Runtime error
Runtime error
Update main.py
Browse files
main.py
CHANGED
@@ -1,25 +1,58 @@
|
|
1 |
from ctransformers import AutoModelForCausalLM
|
2 |
-
|
3 |
-
|
|
|
|
|
|
|
4 |
|
5 |
-
#Model loading
|
6 |
llm = AutoModelForCausalLM.from_pretrained("pt_merge_model_v3.Q4_K_M.gguf",
|
7 |
model_type='llama',
|
8 |
max_new_tokens = 512,
|
9 |
threads = 3,
|
10 |
)
|
11 |
-
|
12 |
-
|
13 |
-
#Pydantic object
|
14 |
-
class validation(BaseModel):
|
15 |
-
prompt: str
|
16 |
-
|
17 |
-
#Fast API
|
18 |
-
app = FastAPI()
|
19 |
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'
|
24 |
-
prompt = f"<s>[INST]<<SYS>>\n + {system_prompt} + <</SYS>>\n{item.prompt.strip()}[/INST]"
|
25 |
return llm(prompt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
from ctransformers import AutoModelForCausalLM
|
2 |
+
import gradio as gr
|
3 |
+
|
4 |
+
greety = """
|
5 |
+
A special thanks to the Gathnex team members who made a significant contribution to this project.
|
6 |
+
"""
|
7 |
|
|
|
8 |
llm = AutoModelForCausalLM.from_pretrained("pt_merge_model_v3.Q4_K_M.gguf",
|
9 |
model_type='llama',
|
10 |
max_new_tokens = 512,
|
11 |
threads = 3,
|
12 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13 |
|
14 |
+
def stream(prompt, UL):
|
15 |
+
system_prompt = 'You are a helpful AI assistant'
|
16 |
+
prompt = f"<s>[INST]<<SYS>>\n + {system_prompt} + <</SYS>>\n{prompt.strip()}[/INST]"
|
|
|
|
|
17 |
return llm(prompt)
|
18 |
+
|
19 |
+
css = """
|
20 |
+
h1 {
|
21 |
+
text-align: center;
|
22 |
+
}
|
23 |
+
#duplicate-button {
|
24 |
+
margin: auto;
|
25 |
+
color: white;
|
26 |
+
background: #1565c0;
|
27 |
+
border-radius: 100vh;
|
28 |
+
}
|
29 |
+
.contain {
|
30 |
+
max-width: 900px;
|
31 |
+
margin: auto;
|
32 |
+
padding-top: 1.5rem;
|
33 |
+
}
|
34 |
+
"""
|
35 |
+
|
36 |
+
chat_interface = gr.ChatInterface(
|
37 |
+
fn=stream,
|
38 |
+
#additional_inputs_accordion_name = "Credentials",
|
39 |
+
#additional_inputs=[
|
40 |
+
# gr.Textbox(label="OpenAI Key", lines=1),
|
41 |
+
# gr.Textbox(label="Linkedin Access Token", lines=1),
|
42 |
+
#],
|
43 |
+
stop_btn=None,
|
44 |
+
examples=[
|
45 |
+
["explain Large language model"],
|
46 |
+
["what is quantum computing"]
|
47 |
+
],
|
48 |
+
)
|
49 |
+
|
50 |
+
with gr.Blocks(css=css) as demo:
|
51 |
+
gr.HTML("<h1><center>RoyalGPT Free LLM Deployment Space<h1><center>")
|
52 |
+
gr.HTML("<h3><center><a href='https://medium.com/@gathnex'>RoyalGPT</a>💬<h3><center>")
|
53 |
+
gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
|
54 |
+
chat_interface.render()
|
55 |
+
gr.Markdown(greety)
|
56 |
+
|
57 |
+
if __name__ == "__main__":
|
58 |
+
demo.queue(max_size=10).launch()
|