|
|
from llama_cpp.server.app import create_app, Settings |
|
|
from fastapi.responses import HTMLResponse |
|
|
from fastapi.middleware.cors import CORSMiddleware |
|
|
from fastapi.responses import RedirectResponse |
|
|
import os |
|
|
|
|
|
model_path = "/home/user/model/gguf-model.gguf" |
|
|
|
|
|
app = create_app( |
|
|
Settings( |
|
|
n_threads=4, |
|
|
model=model_path, |
|
|
embedding=True, |
|
|
n_gpu_layers=33 |
|
|
) |
|
|
) |
|
|
app.add_middleware( |
|
|
CORSMiddleware, |
|
|
allow_origins=["*"], |
|
|
allow_credentials=True, |
|
|
allow_methods=["*"], |
|
|
allow_headers=["*"], |
|
|
) |
|
|
|
|
|
@app.get("/") |
|
|
async def redirect_root_to_docs(): |
|
|
return RedirectResponse("/docs") |
|
|
|
|
|
if __name__ == "__main__": |
|
|
import uvicorn |
|
|
uvicorn.run(app, host="0.0.0.0", port=7860) |
|
|
|
