File size: 951 Bytes
afd4afc 3f672d1 afd4afc 3f672d1 afd4afc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
from flask import Flask, request, jsonify
app = Flask(__name__)
hf_hub_download("TheBloke/phi-2-GGUF", "phi-2.Q8_0.gguf", local_dir="./")
phi = Llama(model_path="./phi-2.Q8_0.gguf", n_ctx=2048, n_gpu_layers=999)
app.route("/spaces/MrOvkill/phastfi/", methods=["GET"])
def index():
return "<html><body><h1>Use API</h1><p>Use /completion as POST with a prompt in a JSON query.</p></body></html>"
@app.route("/spaces/MrOvkill/phastfi/generate", methods=["POST"])
def completion():
prompt = request.json["prompt"]
res = phi(
prompt,
temperature=0.33,
top_p=0.95,
top_k=42,
max_tokens=1024,
num_completions=2,
)
return jsonify({
"responses": [
res["choices"][0]["text"],
res["choices"][1]["text"]
]
})
if __name__ == "__main__":
app.run(host="0.0.0.0", port=7860) |