import gradio as gr
from huggingface_hub import InferenceClient
import random
# Define the model to be used
model = "google/gemma-7b" #"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO"
client = InferenceClient(model)
def format_prompt_mixtral(message, history):
prompt = ""
if history:
for user_prompt, bot_response in history:
prompt += f"[INST] {user_prompt} [/INST]"
prompt += f" {bot_response} "
prompt += f"[INST] {message} [/INST]"
return prompt
def chat_inf(system_prompt, prompt, history, seed, temp, tokens, top_p, rep_p):
if system_prompt:
system_prompt = f'{system_prompt}, '
generate_kwargs = dict(
temperature=temp,
max_new_tokens=tokens,
top_p=top_p,
repetition_penalty=rep_p,
do_sample=True,
seed=seed,
)
formatted_prompt = format_prompt_mixtral(f"{system_prompt}{prompt}", history)
stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
output = ""
for response in stream:
output += response.token.text
yield [(prompt, output)]
history.append((prompt, output))
yield history
def clear_fn():
return None, None, None
rand_val = random.randint(1, 1111111111111111)
def check_rand(inp, val):
if inp:
return gr.Slider(label="Seed", minimum=1, maximum=1111111111111111, value=random.randint(1, 1111111111111111))
else:
return gr.Slider(label="Seed", minimum=1, maximum=1111111111111111, value=int(val))
with gr.Blocks() as app:
gr.HTML("""