10th

Dockerfile.backup

@@ -0,0 +1,61 @@
+# Use an NVIDIA CUDA base image
+ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
+FROM nvidia/cuda:${CUDA_IMAGE}
+
+
+ENV HOST 0.0.0.0
+
+# Set the working directory in the container to /app
+#WORKDIR /app
+
+RUN mkdir -p /app/cache && chmod -R 777 /app/cache
+
+ENV HF_HOME=/app/cache
+
+# Install Python and pip
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y git build-essential \
+    python3 python3-pip gcc wget \
+    ocl-icd-opencl-dev opencl-headers clinfo \
+    libclblast-dev libopenblas-dev \
+    && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+ENV CUDA_DOCKER_ARCH=all
+ENV LLAMA_CUBLAS=1
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Install required packages from requirements.txt
+COPY ./requirements.txt /app/requirements.txt
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+
+# Expose the ports for FastAPI and Streamlit
+EXPOSE 8000
+EXPOSE 8501
+
+# Copy and give execute permissions to the start script
+COPY start_server.sh /app/start_server.sh
+RUN chmod +x /app/start_server.sh
+
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+WORKDIR /home/user/app
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+	SYSTEM=spaces
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . /home/user/app
+
+# Run the start script
+#CMD ["/app/start_server.sh"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

app.py

@@ -0,0 +1,91 @@
+import os
+#import gradio as gr
+import copy
+import time
+import llama_cpp
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download  
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+import nest_asyncio
+import uvicorn
+
+app = FastAPI()
+MODEL_NAME = "TheBloke/CodeLlama-7B-GGUF"
+PDF_PATH = "/opt/docs"
+CLASSIFIER_MODEL_NAME = "roberta-large-mnli"
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=['*'],
+    allow_headers=['*'],
+)
+
+llm = Llama(
+    model_path=hf_hub_download(
+        repo_id=os.environ.get("REPO_ID", MODEL_NAME),
+        filename=os.environ.get("MODEL_FILE", "codellama-7b.Q8_0.gguf"),
+    ),
+    n_ctx=2048,
+    n_gpu_layers=50, # change n_gpu_layers if you have more or less VRAM 
+) 
+
+history = []
+
+system_message = """
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
+"""
+
+@app.post("/predict")
+def generate_text(message, history):
+    temp = ""
+    input_prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "
+    for interaction in history:
+        input_prompt = input_prompt + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " </s><s> [INST] "
+
+    input_prompt = input_prompt + str(message) + " [/INST] "
+
+    output = llm(
+        input_prompt,
+        temperature=0.15,
+        top_p=0.1,
+        top_k=40, 
+        repeat_penalty=1.1,
+        max_tokens=1024,
+        stop=[
+            "<|prompter|>",
+            "<|endoftext|>",
+            "<|endoftext|> \n",
+            "ASSISTANT:",
+            "USER:",
+            "SYSTEM:",
+        ],
+        stream=True,
+    )
+    for out in output:
+        stream = copy.deepcopy(out)
+        temp += stream["choices"][0]["text"]
+        yield temp
+
+    history = ["init", input_prompt]
+
+
+# demo = gr.ChatInterface(
+#     generate_text,
+#     title="llama-cpp-python on GPU",
+#     description="Running LLM with https://github.com/abetlen/llama-cpp-python",
+#     examples=["tell me everything about llamas"],
+#     cache_examples=True,
+#     retry_btn=None,
+#     undo_btn="Delete Previous",
+#     clear_btn="Clear",
+# )
+# demo.queue(concurrency_count=1, max_size=5)
+# demo.launch()
+    
+nest_asyncio.apply()
+uvicorn.run(app, port=8000)