File size: 4,818 Bytes
da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 1ad806d da90805 4e3e7a7 1ad806d da90805 1ad806d da90805 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
# Ref: https://huggingface.co/spaces/ysharma/Chat_with_Meta_llama3_8b
import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
from threading import Thread
DESCRIPTION = '''
<div>
<h1 style="text-align: center;">Sarashina2.2-3B Instruct</h1>
<p>非公式Sarashina2.2-3B Instructだよ。 <a href="https://huggingface.co/sbintuitions/sarashina2.2-3b-instruct-v0.1"><b>sbintuitions/sarashina2.2-3b-instruct-v0.1</b></a>.</p>
</div>
'''
LICENSE = """
<p/>
---
Built with Sarashina
"""
PLACEHOLDER = """
<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
<h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">非公式 Sarashina2.2-3B Instruct Test</h1>
<p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">なんでもきいてね</p>
</div>
"""
css = """
h1 {
text-align: center;
display: block;
}
#duplicate-button {
margin: auto;
color: white;
background: #1565c0;
border-radius: 100vh;
}
"""
# Load the tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("sbintuitions/sarashina2.2-3b-instruct-v0.1", use_fast=False)
model = AutoModelForCausalLM.from_pretrained("sbintuitions/sarashina2.2-3b-instruct-v0.1", torch_dtype=torch.bfloat16)
model=model.to("cuda:0")
@spaces.GPU()
def chat_llama3_8b(message: str,
history: list,
temperature: float,
max_new_tokens: int
) -> str:
"""
Generate a streaming response using the llama3-8b model.
Args:
message (str): The input message.
history (list): The conversation history used by ChatInterface.
temperature (float): The temperature for generating the response.
max_new_tokens (int): The maximum number of new tokens to generate.
Returns:
str: The generated response.
"""
conversation = []
for user, assistant in history:
conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True,return_tensors="pt")
streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict(
input_ids= input_ids.to(model.device),
streamer=streamer,
max_new_tokens=max_new_tokens,
do_sample=True,
temperature=temperature,
top_p=0.95,
repetition_penalty=1.1,
)
# This will enforce greedy generation (do_sample=False) when the temperature is passed 0, avoiding the crash.
if temperature == 0:
generate_kwargs['do_sample'] = False
t = Thread(target=model.generate, kwargs=generate_kwargs)
t.start()
outputs = []
for text in streamer:
outputs.append(text)
yield "".join(outputs)
# Gradio block
chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface', scale=1)
with gr.Blocks(fill_height=True, css=css) as demo:
gr.Markdown(DESCRIPTION)
gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
gr.ChatInterface(
fn=chat_llama3_8b,
chatbot=chatbot,
fill_height=True,
additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
additional_inputs=[
gr.Slider(minimum=0,
maximum=1,
step=0.1,
value=0.5,
label="Temperature",
render=False),
gr.Slider(minimum=128,
maximum=4096,
step=1,
value=512,
label="Max new tokens",
render=False ),
],
examples=[
['小学生にもわかるように相対性理論を教えてください。'],
['宇宙の起源を知るための方法をステップ・バイ・ステップで教えてください。'],
['1から100までの素数を求めるスクリプトをPythonで書いてください。'],
['友達の陽葵にあげる誕生日プレゼントを考えてください。ただし、陽葵は中学生で、私は同じクラスの男性であることを考慮してください。'],
['ペンギンがジャングルの王様であることを正当化するように説明してください。']
],
cache_examples=False,
)
gr.Markdown(LICENSE)
if __name__ == "__main__":
demo.launch()
|