secret set on HF

app.py

@@ -1,11 +1,11 @@
-import gradio as gr
+import os
 from huggingface_hub import InferenceClient
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
+HF_API_KEY = os.getenv("HF_API_KEY")  # Retrieve API key from environment variable
+
 models = ["HuggingFaceH4/zephyr-7b-beta", "microsoft/Phi-4-mini-instruct"]
-client = InferenceClient(model=models[1])
+client = InferenceClient(model=models[1], token=HF_API_KEY)  # Pass API key to client
+
 
 
 def respond(
@@ -16,7 +16,7 @@ def respond(
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
+         = [{"role": "system", "content": system_message}]
 
     for val in history:
         if val[0]:

model_local.py

@@ -0,0 +1,19 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+model_name = "microsoft/Phi-4-mini-instruct"
+
+# Load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+
+# Generate a response
+def chat_with_model(prompt, max_new_tokens=100):
+    inputs = tokenizer(prompt, return_tensors="pt").to("cpu")  # Move to GPU if available
+    output = model.generate(**inputs, max_new_tokens=max_new_tokens)
+    return tokenizer.decode(output[0], skip_special_tokens=True)
+
+# Test the model
+user_input = "Explain quantum computing in simple terms."
+response = chat_with_model(user_input)
+print("Chatbot Response:", response)