awq fix

qa_engine/qa_engine.py

@@ -66,7 +66,7 @@ class TransformersPipelineModel(LLM):
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         if "AWQ" in model_id:
             model = AutoAWQForCausalLM.from_quantized(model_id, fuse_layers=True,
-                                          trust_remote_code=False, safetensors=True).model
+                                          trust_remote_code=False, safetensors=True, torch_dtype=torch.bfloat16).model
         else:
             model = AutoModelForCausalLM.from_pretrained(
                 model_id,
@@ -80,7 +80,7 @@ class TransformersPipelineModel(LLM):
             'text-generation',
             model=model,
             tokenizer=tokenizer,
-            torch_dtype=torch.float16,
+            torch_dtype=torch.bfloat16,
             device_map='auto',
             eos_token_id=tokenizer.eos_token_id,
             pad_token_id=tokenizer.eos_token_id,