main.py

from flask import Flask, request, jsonify
from langchain.llms import LlamaCpp
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.prompts import PromptTemplate
from langchain.schema.output_parser import StrOutputParser

app = Flask(__name__)

@app.route('/', methods=['POST'])
def get_skills():
    n_gpu_layers = 0
    n_batch = 1024

    callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])

    llm = LlamaCpp(
        model_path="phi-2.Q4_K_M.gguf",
        temperature=0.1,
        n_gpu_layers=n_gpu_layers,
        n_batch=n_batch,
        callback_manager=callback_manager,
        verbose=True,
        n_ctx=2048
    )

    cv_body = request.json.get('cv_body')

    template = """Instruct:
    Take a deep breath to deep understand, and don't this cv vc = {cv_body} . to answer this question and instructions </s> {question}
    \nOutput:"""

    prompt = PromptTemplate(template=template, input_variables=["question","text"])
    chain = prompt | llm | StrOutputParser()

    ans = chain.invoke({"question": "What are his best skills? write in points","text":cv_body},
                       config={
                           # "callbacks": [ConsoleCallbackHandler()]
                       })

    return jsonify({'skills': ans})

if __name__ == '__main__':
    app.run()