Spaces:
Sleeping
Sleeping
File size: 942 Bytes
15b386c 4c21848 15b386c f9cae74 15b386c f9cae74 05976e6 15b386c f9cae74 15b386c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
from transformers import AutoTokenizer
from petals import AutoDistributedModelForCausalLM
import streamlit as st
# Choose any model available at https://health.petals.dev
model_name = "petals-team/StableBeluga2" # This one is fine-tuned Llama 2 (70B)
# Connect to a distributed network hosting model layers
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoDistributedModelForCausalLM.from_pretrained(model_name)
system_prompt = "### System:\nYou are Stable Beluga, an AI that is very precise and creative. Be as creative and accurate you can.\n\n"
conv=system_prompt
message = st.chat_input('Message')
if message:
prompt = f"### User: {message}\n\n### Assistant:\n"
conv+=prompt
print(conv)
# Run the model as if it were on your computer
inputs = tokenizer(conv, return_tensors="pt")["input_ids"]
outputs = model.generate(inputs, max_new_tokens=256)
st.write(tokenizer.decode(outputs[0])[3:-4]) |