T145 commited on
Commit
f7b7d51
1 Parent(s): 60e6fbb

Try using llama_cpp with a ZEUS quant

Browse files
Files changed (2) hide show
  1. app.py +24 -26
  2. requirements.txt +2 -1
app.py CHANGED
@@ -1,10 +1,12 @@
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
3
 
4
- """
5
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
- """
7
- client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 
8
 
9
 
10
  def respond(
@@ -27,7 +29,7 @@ def respond(
27
 
28
  response = ""
29
 
30
- for message in client.chat_completion(
31
  messages,
32
  max_tokens=max_tokens,
33
  stream=True,
@@ -40,25 +42,21 @@ def respond(
40
  yield response
41
 
42
 
43
- """
44
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
45
- """
46
- demo = gr.ChatInterface(
47
- respond,
48
- additional_inputs=[
49
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
50
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
51
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
52
- gr.Slider(
53
- minimum=0.1,
54
- maximum=1.0,
55
- value=0.95,
56
- step=0.05,
57
- label="Top-p (nucleus sampling)",
58
- ),
59
- ],
60
- )
61
-
62
-
63
  if __name__ == "__main__":
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  demo.launch()
 
1
  import gradio as gr
2
+ from llama_cpp import Llama
3
 
4
+ LLM = Llama.from_pretrained(
5
+ repo_id="mradermacher/ZEUS-8B-V2-i1-GGUF",
6
+ filename="*Q4_K_M.gguf",
7
+ chat_format="llama-2",
8
+ verbose=False
9
+ )
10
 
11
 
12
  def respond(
 
29
 
30
  response = ""
31
 
32
+ for message in llm.create_chat_completion(
33
  messages,
34
  max_tokens=max_tokens,
35
  stream=True,
 
42
  yield response
43
 
44
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  if __name__ == "__main__":
46
+ demo = gr.ChatInterface(
47
+ respond,
48
+ additional_inputs=[
49
+ gr.Textbox(value="You are a friendly assistant.", label="System message"),
50
+ gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
51
+ gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
52
+ gr.Slider(
53
+ minimum=0.1,
54
+ maximum=1.0,
55
+ value=0.95,
56
+ step=0.05,
57
+ label="Top-p (nucleus sampling)",
58
+ ),
59
+ ],
60
+ )
61
+
62
  demo.launch()
requirements.txt CHANGED
@@ -1 +1,2 @@
1
- huggingface_hub==0.25.2
 
 
1
+ huggingface_hub==0.25.2
2
+ llama_cpp