File size: 1,574 Bytes
76fb5c4 572e6c5 2ae36bd 572e6c5 106db30 2ae36bd 106db30 892047e 106db30 6ebdc50 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
ARG UBUNTU_VERSION=22.04
# This needs to generally match the container host's environment.
ARG CUDA_VERSION=11.7.1
# Target the CUDA build image
ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
# Target the CUDA runtime image
ARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
FROM ${BASE_CUDA_DEV_CONTAINER} as build
# Unless otherwise specified, we make a fat build.
ARG CUDA_DOCKER_ARCH=all
RUN apt-get update && \
apt-get install -y build-essential git
# Install Python3 and pip
RUN apt-get install -y python3 python3-pip
WORKDIR /app
COPY . .
# Set nvcc architecture
ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
# Enable cuBLAS
ENV LLAMA_CUBLAS=1
FROM ${BASE_CUDA_RUN_CONTAINER} as runtime
# Install build and runtime dependencies
RUN apt-get update && \
apt-get install -y \
libopenblas-dev \
ninja-build \
build-essential \
pkg-config \
curl
# Install Python3 and pip for the runtime container as well
RUN apt-get install -y python3 python3-pip
RUN pip3 install -U pip setuptools wheel && \
pip3 install --verbose llama-cpp-python[server]
# Download model
RUN mkdir model && \
curl -L https://huggingface.co/matthoffner/Magicoder-S-DS-6.7B-GGUF/resolve/main/Magicoder-S-DS-6.7B_Q4_K_M.gguf -o model/gguf-model.gguf
COPY ./main.py ./
# Set environment variable for the host
ENV HOST=0.0.0.0
ENV PORT=7860
# Expose a port for the server
EXPOSE ${PORT}
# Run the server start script
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
|