sha1779 commited on
Commit
99eb4fa
·
verified ·
1 Parent(s): 2baa56d

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +49 -16
main.py CHANGED
@@ -1,25 +1,58 @@
1
  from ctransformers import AutoModelForCausalLM
2
- from fastapi import FastAPI, Form
3
- from pydantic import BaseModel
 
 
 
4
 
5
- #Model loading
6
  llm = AutoModelForCausalLM.from_pretrained("pt_merge_model_v3.Q4_K_M.gguf",
7
  model_type='llama',
8
  max_new_tokens = 512,
9
  threads = 3,
10
  )
11
-
12
-
13
- #Pydantic object
14
- class validation(BaseModel):
15
- prompt: str
16
-
17
- #Fast API
18
- app = FastAPI()
19
 
20
- #Zephyr completion
21
- @app.post("/llama_on_cpu")
22
- async def stream(item: validation):
23
- system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'
24
- prompt = f"<s>[INST]<<SYS>>\n + {system_prompt} + <</SYS>>\n{item.prompt.strip()}[/INST]"
25
  return llm(prompt)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from ctransformers import AutoModelForCausalLM
2
+ import gradio as gr
3
+
4
+ greety = """
5
+ A special thanks to the Gathnex team members who made a significant contribution to this project.
6
+ """
7
 
 
8
  llm = AutoModelForCausalLM.from_pretrained("pt_merge_model_v3.Q4_K_M.gguf",
9
  model_type='llama',
10
  max_new_tokens = 512,
11
  threads = 3,
12
  )
 
 
 
 
 
 
 
 
13
 
14
+ def stream(prompt, UL):
15
+ system_prompt = 'You are a helpful AI assistant'
16
+ prompt = f"<s>[INST]<<SYS>>\n + {system_prompt} + <</SYS>>\n{prompt.strip()}[/INST]"
 
 
17
  return llm(prompt)
18
+
19
+ css = """
20
+ h1 {
21
+ text-align: center;
22
+ }
23
+ #duplicate-button {
24
+ margin: auto;
25
+ color: white;
26
+ background: #1565c0;
27
+ border-radius: 100vh;
28
+ }
29
+ .contain {
30
+ max-width: 900px;
31
+ margin: auto;
32
+ padding-top: 1.5rem;
33
+ }
34
+ """
35
+
36
+ chat_interface = gr.ChatInterface(
37
+ fn=stream,
38
+ #additional_inputs_accordion_name = "Credentials",
39
+ #additional_inputs=[
40
+ # gr.Textbox(label="OpenAI Key", lines=1),
41
+ # gr.Textbox(label="Linkedin Access Token", lines=1),
42
+ #],
43
+ stop_btn=None,
44
+ examples=[
45
+ ["explain Large language model"],
46
+ ["what is quantum computing"]
47
+ ],
48
+ )
49
+
50
+ with gr.Blocks(css=css) as demo:
51
+ gr.HTML("<h1><center>RoyalGPT Free LLM Deployment Space<h1><center>")
52
+ gr.HTML("<h3><center><a href='https://medium.com/@gathnex'>RoyalGPT</a>💬<h3><center>")
53
+ gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
54
+ chat_interface.render()
55
+ gr.Markdown(greety)
56
+
57
+ if __name__ == "__main__":
58
+ demo.queue(max_size=10).launch()