File size: 1,450 Bytes
8d8525b c138cbc 335f4a7 f79962c 8d8525b c138cbc 8d8525b c138cbc f79962c c138cbc 0e6877a 335f4a7 8d8525b 2e1b7f6 8d8525b 2e1b7f6 8d8525b 2e1b7f6 8d8525b 335f4a7 f79962c 35666dc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
from flask import Flask, request, Response, jsonify
from llama_cpp import Llama
app = Flask(__name__)
# Load the model
print("🔄 Loading model...")
llm = Llama.from_pretrained(
repo_id="bartowski/google_gemma-3-1b-it-GGUF",
filename="google_gemma-3-1b-it-IQ4_XS.gguf",
n_ctx=2048
)
print("✅ Model loaded!")
@app.route("/")
def home():
print("📢 Serving index.html")
return render_template("index.html")
def generate_response(user_input):
"""Generator function to stream model output"""
try:
response = llm.create_chat_completion(
messages=[{"role": "user", "content": user_input}],
stream=True # Enable streaming
)
for chunk in response:
if "choices" in chunk and len(chunk["choices"]) > 0:
token = chunk["choices"][0]["delta"].get("content", "")
if token:
print(f"📝 Token: {token}", flush=True) # Debugging
yield token
except Exception as e:
print(f"❌ Error generating response: {e}")
yield "[Error occurred]"
@app.route("/chat", methods=["POST"])
def chat():
user_input = request.json.get("message", "")
if not user_input:
return jsonify({"error": "Empty input"}), 400
return Response(generate_response(user_input), content_type="text/plain")
if __name__ == "__main__":
app.run(host="0.0.0.0", port=7860, debug=True)
|