Update app.py

app.py

@@ -4,6 +4,10 @@ from huggingface_hub import InferenceClient
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
 client = InferenceClient("BenBranyon/zephyr-sumbot-all-songs")
+model_id = "BenBranyon/zephyr-sumbot-all-songs"
+model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer.use_default_system_prompt = False
 
 
 def respond(