Spaces:
Runtime error
Runtime error
File size: 807 Bytes
008045b 4f608eb 7ec63cb 008045b 8f2a0d5 008045b 8f2a0d5 08921b4 8f2a0d5 008045b 8f2a0d5 fc6816d 008045b 08921b4 008045b 08921b4 008045b 08921b4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
from flask import Flask, request, jsonify
from langchain_community.llms import LlamaCpp
app = Flask(__name__)
n_gpu_layers = 0
n_batch = 1024
llm = LlamaCpp(
model_path="Phi-3-mini-4k-instruct-q4.gguf", # path to GGUF file
temperature=0.1,
n_gpu_layers=n_gpu_layers,
n_batch=n_batch,
verbose=True,
n_ctx=2048
)
@app.route('/', methods=['POST'])
def get_skills():
cv_body = request.json.get('cv_body')
# Simple inference example
output = llm(
f"<|user|>\n{cv_body}<|end|>\n<|assistant|>Can you list the skills mentioned in the CV?<|end|>",
max_tokens=256, # Generate up to 256 tokens
stop=["<|end|>"],
echo=True, # Whether to echo the prompt
)
return jsonify({'skills': output})
if __name__ == '__main__':
app.run() |