Update app.py

app.py

@@ -34,7 +34,7 @@ tokenizer = AutoTokenizer.from_pretrained(model)
 # max_tokens is for the maximum length for generation.
 
 # Input the model name or path. Can be GPTQ or AWQ models.
-llm = LLM(model=model)
+llm = LLM(model=model, quantization="gptq", kv_cache_dtype="fp8_e5m2")
 
 @spaces.GPU
 def generate(message, history, system, max_tokens, temperature, top_p, top_k, penalty):