File size: 1,195 Bytes
93396ac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 |
import gradio as gr
from transformers import AutoTokenizer, FastLanguageModel
# Load the model and tokenizer
model, tokenizer = FastLanguageModel.from_pretrained(
model_name="lora_model", # Replace with your trained model name
max_seq_length=512,
dtype="float16",
load_in_4bit=True,
)
FastLanguageModel.for_inference(model)
# Define the inference function
def generate_response(user_input):
# Prepare the input for the model
labeled_prompt = (
"Please provide the response with the following labels:\n"
f"User Input: {user_input}\n"
"Response:"
)
inputs = tokenizer(
[labeled_prompt],
return_tensors="pt",
padding=True,
truncation=True,
max_length=512,
).to("cuda")
response = model.generate(input_ids=inputs.input_ids, attention_mask=inputs.attention_mask, max_new_tokens=100, pad_token_id=tokenizer.eos_token_id)
return tokenizer.decode(response[0], skip_special_tokens=True)
# Create a Gradio interface
iface = gr.Interface(fn=generate_response, inputs="text", outputs="text", title="Chatbot Interface", description="Enter your message below:")
# Launch the app
iface.launch()
|