Update app.py

app.py

@@ -26,9 +26,9 @@ HF_TOKEN = os.environ.get("Inference_Calls", None)
 # from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration, TextIteratorStreamer
 # processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
 # model = LlavaNextForConditionalGeneration.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf", torch_dtype=torch.float16, low_cpu_mem_usage=True)
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+"""from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN)
+model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
@@ -37,7 +37,10 @@ model = AutoModelForCausalLM.from_pretrained(
 terminators = [
     tokenizer.eos_token_id,
     tokenizer.convert_tokens_to_ids("<|eot_id|>")
-    ]
+]"""
+from huggingface_hub import InferenceClient
+model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+client = InferenceClient(model_id, api_key="HF_TOKEN")
 
 embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
 
@@ -46,12 +49,10 @@ ASR_BATCH_SIZE = 8
 ASR_CHUNK_LENGTH_S = 30
 TEMP_FILE_LIMIT_MB = 1024 #2048
 
-from huggingface_hub import InferenceClient
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 # client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 device = 0 if torch.cuda.is_available() else "cpu"
 
@@ -84,7 +85,7 @@ def respond(
             messages.append({"role": "assistant", "content": val[1]})
 
     messages.append({"role": "user", "content": message})
-
+"""
     input_ids = tokenizer.apply_chat_template(
         messages,
         add_generation_prompt=True,
@@ -128,7 +129,7 @@ def respond(
 
         response += token
         yield response
-"""
+
 
 @spaces.GPU
 def transcribe(asr_inputs, task):