|
|
import gradio as gr |
|
|
from huggingface_hub import InferenceClient |
|
|
|
|
|
""" |
|
|
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference |
|
|
""" |
|
|
client = InferenceClient("Grandediw/lora_model") |
|
|
|
|
|
def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p): |
|
|
messages = [{"role": "system", "content": system_message}] |
|
|
|
|
|
for val in history: |
|
|
if val[0]: |
|
|
messages.append({"role": "user", "content": val[0]}) |
|
|
if val[1]: |
|
|
messages.append({"role": "assistant", "content": val[1]}) |
|
|
|
|
|
messages.append({"role": "user", "content": message}) |
|
|
|
|
|
response = "" |
|
|
|
|
|
for message in client.chat_completion( |
|
|
messages, |
|
|
max_tokens=max_tokens, |
|
|
stream=True, |
|
|
temperature=temperature, |
|
|
top_p=top_p, |
|
|
): |
|
|
token = message.choices[0].delta.content |
|
|
response += token |
|
|
yield response |
|
|
|
|
|
""" |
|
|
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface |
|
|
""" |
|
|
with gr.Blocks(title="Enhanced LORA Chat Interface") as demo: |
|
|
gr.Markdown( |
|
|
""" |
|
|
# LORA Chat Assistant |
|
|
Welcome! This is a demo of a LORA-based Chat Assistant. |
|
|
Start by entering your prompt in the chat box below. |
|
|
""" |
|
|
) |
|
|
|
|
|
with gr.Row(): |
|
|
|
|
|
with gr.Column(): |
|
|
chat = gr.ChatInterface( |
|
|
fn=respond, |
|
|
additional_inputs=[], |
|
|
height=500 |
|
|
) |
|
|
|
|
|
|
|
|
with gr.Column(): |
|
|
gr.Markdown("### Configuration") |
|
|
system_message = gr.Textbox( |
|
|
value="You are a friendly Chatbot.", |
|
|
label="Initial Behavior (System Message)", |
|
|
lines=3, |
|
|
placeholder="Describe how the assistant should behave..." |
|
|
) |
|
|
|
|
|
with gr.Accordion("Advanced Settings", open=False): |
|
|
max_tokens = gr.Slider( |
|
|
minimum=1, maximum=2048, value=512, step=1, |
|
|
label="Max new tokens", |
|
|
info="Controls the maximum number of tokens in the response." |
|
|
) |
|
|
temperature = gr.Slider( |
|
|
minimum=0.1, maximum=4.0, value=0.7, step=0.1, |
|
|
label="Temperature", |
|
|
info="Higher values produce more random outputs." |
|
|
) |
|
|
top_p = gr.Slider( |
|
|
minimum=0.1, maximum=1.0, value=0.95, step=0.05, |
|
|
label="Top-p (nucleus sampling)", |
|
|
info="Limits the tokens considered to the top portion by cumulative probability." |
|
|
) |
|
|
|
|
|
|
|
|
chat.configure( |
|
|
additional_inputs=[system_message, max_tokens, temperature, top_p] |
|
|
) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
demo.launch() |
|
|
|
