|
|
import os |
|
|
|
|
|
from langchain.llms import CTransformers |
|
|
from fastapi import FastAPI |
|
|
from pydantic import BaseModel |
|
|
|
|
|
file_name = "zephyr-7b-beta.Q4_K_S.gguf" |
|
|
config = { |
|
|
"max_new_token": 1024, |
|
|
"repetition_penalty": 1.1, |
|
|
"temperature": 0.5, |
|
|
"top_k": 50, |
|
|
"top_p": 0.9, |
|
|
"stream": True, |
|
|
"threads": int(os.cpu_count() / 2), |
|
|
} |
|
|
llm = CTransformers( |
|
|
model=file_name, |
|
|
model_type="mistral", |
|
|
lib="avx2", |
|
|
**config |
|
|
) |
|
|
print(llm) |
|
|
|
|
|
|
|
|
class validation(BaseModel): |
|
|
prompt: str |
|
|
|
|
|
|
|
|
|
|
|
app = FastAPI() |
|
|
|
|
|
@app.post("/llm_on_cpu") |
|
|
async def stream(item: validation): |
|
|
system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' |
|
|
E_INST = "</s>" |
|
|
user, assistant = "<|user|>", "<|assistant|>" |
|
|
prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt}{E_INST}\n{assistant}\n" |
|
|
|
|
|
return llm(prompt) |
|
|
|
