import gradio as gr from huggingface_hub import InferenceClient import torch from transformers import AutoModelForCausalLM, AutoTokenizer """ For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference """ # client = InferenceClient("HuggingFaceH4/zephyr-7b-beta") model_id = "GoToCompany/llama3-8b-cpt-sahabatai-v1-instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id) # def respond( # message, # history: list[tuple[str, str]], # system_message, # max_tokens, # temperature, # top_p, # ): # messages = [{"role": "system", "content": system_message}] # for val in history: # if val[0]: # messages.append({"role": "user", "content": val[0]}) # if val[1]: # messages.append({"role": "assistant", "content": val[1]}) # messages.append({"role": "user", "content": message}) # response = "" # for message in client.chat_completion( # messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # token = message.choices[0].delta.content # response += token # yield response # """ # For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface # """ # demo = gr.ChatInterface( # respond, # additional_inputs=[ # gr.Textbox(value="You are a friendly Chatbot.", label="System message"), # gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), # gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), # gr.Slider( # minimum=0.1, # maximum=1.0, # value=0.95, # step=0.05, # label="Top-p (nucleus sampling)", # ), # ], # ) # if __name__ == "__main__": # demo.launch() # Function to generate text def generate_text(prompt, max_length=100): inputs = tokenizer(prompt, return_tensors="pt") outputs = model.generate( **inputs, max_length=max_length, num_return_sequences=1, no_repeat_ngram_size=2, do_sample=True, top_p=0.95, temperature=0.7 ) return tokenizer.decode(outputs[0], skip_special_tokens=True) # Gradio frontend def gradio_interface(prompt, max_length): if not prompt.strip(): return "Please enter a prompt." try: output = generate_text(prompt, max_length=max_length) return output except Exception as e: return f"An error occurred: {str(e)}" # Define Gradio components with gr.Blocks() as demo: gr.Markdown("# LLaMA3 8B CPT Sahabatai Instruct") gr.Markdown("Generate text using the **LLaMA3 8B CPT Sahabatai Instruct** model.") with gr.Row(): with gr.Column(): prompt_input = gr.Textbox( label="Enter your prompt", placeholder="Type something...", lines=3, ) max_length_slider = gr.Slider( label="Max Length", minimum=10, maximum=200, value=100, step=10, ) generate_button = gr.Button("Generate") with gr.Column(): output_text = gr.Textbox( label="Generated Text", lines=10, interactive=False, ) # Link the button to the function generate_button.click( fn=gradio_interface, inputs=[prompt_input, max_length_slider], outputs=output_text, ) # Launch the Gradio app if __name__ == "__main__": demo.launch()