Duplicate from matthoffner/ggml-ctransformers-fastapi

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,23 @@
+FROM python:latest
+
+ENV PYTHONUNBUFFERED 1
+
+EXPOSE 8000
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+
+
+COPY --chown=user . $HOME/app
+
+RUN ls -al
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,19 @@
+---
+title: ggml-ctransformers-fastapi
+emoji: 🐳🤗⚡️
+sdk: docker
+app_port: 8000
+duplicated_from: matthoffner/ggml-ctransformers-fastapi
+---
+
+# ggml-ctransformers-fastapi
+
+## <a href="https://github.com/ggerganov/ggml" target="_blank">ggml</a>
+## <a href="https://github.com/marella/ctransformers" target="_blank">ctransformers</a>
+## [FastAPI Docs](https://matthoffner-ggml-ctransformers-fastapi.hf.space/docs)
+
+### Updates
+
+* Added /v1/chat/completions
+* [Start using ctransformers](https://github.com/marella/ctransformers)
+* [Added starcoder example](https://github.com/ggerganov/ggml/tree/master/examples/starcoder)

main.py

@@ -0,0 +1,45 @@
+import fastapi
+import json
+import markdown
+import uvicorn
+from fastapi.responses import HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from sse_starlette.sse import EventSourceResponse
+from ctransformers import AutoModelForCausalLM
+from pydantic import BaseModel
+
+llm = AutoModelForCausalLM.from_pretrained("TheBloke/starcoder-beta-GGML",
+                                           model_file="starcoder-beta.ggmlv3.q4_0.bin",
+                                           model_type="starcoder")
+app = fastapi.FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/")
+async def index():
+    with open("README.md", "r", encoding="utf-8") as readme_file:
+        md_template_string = readme_file.read()
+    html_content = markdown.markdown(md_template_string)
+    return HTMLResponse(content=html_content, status_code=200)
+
+class ChatCompletionRequest(BaseModel):
+    prompt: str
+
+@app.post("/v1/chat/completions")
+async def chat(request: ChatCompletionRequest, response_mode=None):
+    tokens = llm.tokenize(request.prompt)
+    async def server_sent_events(chat_chunks, llm):
+        for token in llm.generate(chat_chunks):
+            yield dict(data=llm.detokenize(token))
+        yield dict(data="[DONE]")
+
+    return EventSourceResponse(server_sent_events(tokens, llm))
+
+if __name__ == "__main__":
+  uvicorn.run(app, host="0.0.0.0", port=8000)
+

requirements.txt

@@ -0,0 +1,11 @@
+uvicorn
+markdown
+fastapi
+loguru
+torch
+numpy
+transformers
+ctransformers
+accelerate
+langchain
+sse_starlette