Spaces:
Running
Running
File size: 1,160 Bytes
d3218e7 5617cf0 654c025 d3218e7 5617cf0 d3218e7 654c025 d3218e7 654c025 d3218e7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import os
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
import gradio as gr
# Set up the model and tokenizer
MODEL_ID = "microsoft/Phi-3.5-mini-instruct"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
os.environ['HF_TOKEN']=os.environ.get('HF_TOKEN')
os.environ['HUGGINGFACEHUB_API_TOKEN']=os.environ.get('HF_TOKEN')
# Load the model with quantization
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
device_map="auto"
)
# Define the function for the Gradio interface
def chat_with_phi(message):
conversation = [{"role": "user", "content": message}]
pipe = pipeline(
"text-generation",
model=model,
tokenizer=tokenizer,
)
response = pipe(conversation)
return response[0]['generated_text']
# Set up the Gradio interface
app = gr.Interface(
fn=chat_with_phi,
inputs=gr.Textbox(label="Type your message:"),
outputs=gr.Textbox(label="Phi 3.5 Responds:"),
title="Phi 3.5 Text Chat",
description="Chat with Phi 3.5 model. Ask anything!",
theme="huggingface"
)
# Launch the app
app.launch(debug=True)
|