11

Dockerfile

@@ -1,33 +1,46 @@
-ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
+# Use an NVIDIA CUDA base image
+ARG CUDA_IMAGE="12.1.1-cudnn8-devel-ubuntu22.04"
 FROM nvidia/cuda:${CUDA_IMAGE}
 
-# We need to set the host to 0.0.0.0 to allow outside access
+
 ENV HOST 0.0.0.0
 
-RUN apt-get update && apt-get upgrade -y \
-    && apt-get install -y git build-essential \
-    python3 python3-pip gcc wget \
-    ocl-icd-opencl-dev opencl-headers clinfo \
-    libclblast-dev libopenblas-dev \
-    && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+# Set the working directory in the container to /app
+#WORKDIR /app
+
+RUN mkdir -p /app/cache && chmod -R 777 /app/cache
+
+ENV HF_HOME=/app/cache
 
-COPY . .
+# Install Python and pip
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  build-essential \
+  python3.9 \
+  python3-pip \
+  git \
+  ffmpeg \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# setting build related env vars
-ENV CUDA_DOCKER_ARCH=all
-ENV LLAMA_CUBLAS=1
+# Copy the current directory contents into the container at /app
+COPY . /app
 
-# Install depencencies
-RUN python3 -m pip install --upgrade pip pytest cmake \
-    scikit-build setuptools fastapi uvicorn sse-starlette \
-    pydantic-settings starlette-context gradio huggingface_hub hf_transfer
+# Install required packages from requirements.txt
+COPY ./requirements.txt /app/requirements.txt
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
 
-# Install llama-cpp-python (build with cuda)
-RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
+# Expose the ports for FastAPI and Streamlit
+EXPOSE 8000
+EXPOSE 8501
+
+# Copy and give execute permissions to the start script
+COPY start_server.sh /app/start_server.sh
+RUN chmod +x /app/start_server.sh
 
 RUN useradd -m -u 1000 user
 # Switch to the "user" user
 USER user
+
+WORKDIR /home/user/app
 # Set home to the user's home directory
 ENV HOME=/home/user \
 	PATH=/home/user/.local/bin:$PATH \
@@ -39,9 +52,9 @@ ENV HOME=/home/user \
 	GRADIO_THEME=huggingface \
 	SYSTEM=spaces
 
-WORKDIR $HOME/app
-
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
-COPY --chown=user . $HOME/app
+COPY --chown=user . /home/user/app
 
-CMD ["python3", "app.py"]
+# Run the start script
+#CMD ["/app/start_server.sh"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.backup

@@ -1,61 +0,0 @@
-# Use an NVIDIA CUDA base image
-ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
-FROM nvidia/cuda:${CUDA_IMAGE}
-
-
-ENV HOST 0.0.0.0
-
-# Set the working directory in the container to /app
-#WORKDIR /app
-
-RUN mkdir -p /app/cache && chmod -R 777 /app/cache
-
-ENV HF_HOME=/app/cache
-
-# Install Python and pip
-RUN apt-get update && apt-get upgrade -y \
-    && apt-get install -y git build-essential \
-    python3 python3-pip gcc wget \
-    ocl-icd-opencl-dev opencl-headers clinfo \
-    libclblast-dev libopenblas-dev \
-    && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
-
-ENV CUDA_DOCKER_ARCH=all
-ENV LLAMA_CUBLAS=1
-
-# Copy the current directory contents into the container at /app
-COPY . /app
-
-# Install required packages from requirements.txt
-COPY ./requirements.txt /app/requirements.txt
-RUN pip3 install --no-cache-dir -r /app/requirements.txt
-
-# Expose the ports for FastAPI and Streamlit
-EXPOSE 8000
-EXPOSE 8501
-
-# Copy and give execute permissions to the start script
-COPY start_server.sh /app/start_server.sh
-RUN chmod +x /app/start_server.sh
-
-RUN useradd -m -u 1000 user
-# Switch to the "user" user
-USER user
-WORKDIR /home/user/app
-# Set home to the user's home directory
-ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH \
-    PYTHONPATH=$HOME/app \
-	PYTHONUNBUFFERED=1 \
-	GRADIO_ALLOW_FLAGGING=never \
-	GRADIO_NUM_PORTS=1 \
-	GRADIO_SERVER_NAME=0.0.0.0 \
-	GRADIO_THEME=huggingface \
-	SYSTEM=spaces
-
-# Copy the current directory contents into the container at $HOME/app setting the owner to the user
-COPY --chown=user . /home/user/app
-
-# Run the start script
-#CMD ["/app/start_server.sh"]
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

app.py

@@ -1,91 +0,0 @@
-import os
-#import gradio as gr
-import copy
-import time
-import llama_cpp
-from llama_cpp import Llama
-from huggingface_hub import hf_hub_download  
-from fastapi import FastAPI, Request
-from fastapi.middleware.cors import CORSMiddleware
-import nest_asyncio
-import uvicorn
-
-app = FastAPI()
-MODEL_NAME = "TheBloke/CodeLlama-7B-GGUF"
-PDF_PATH = "/opt/docs"
-CLASSIFIER_MODEL_NAME = "roberta-large-mnli"
-
-# Add CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=['*'],
-    allow_credentials=True,
-    allow_methods=['*'],
-    allow_headers=['*'],
-)
-
-llm = Llama(
-    model_path=hf_hub_download(
-        repo_id=os.environ.get("REPO_ID", MODEL_NAME),
-        filename=os.environ.get("MODEL_FILE", "codellama-7b.Q8_0.gguf"),
-    ),
-    n_ctx=2048,
-    n_gpu_layers=50, # change n_gpu_layers if you have more or less VRAM 
-) 
-
-history = []
-
-system_message = """
-You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
-If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
-"""
-
-@app.post("/predict")
-def generate_text(message, history):
-    temp = ""
-    input_prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "
-    for interaction in history:
-        input_prompt = input_prompt + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " </s><s> [INST] "
-
-    input_prompt = input_prompt + str(message) + " [/INST] "
-
-    output = llm(
-        input_prompt,
-        temperature=0.15,
-        top_p=0.1,
-        top_k=40, 
-        repeat_penalty=1.1,
-        max_tokens=1024,
-        stop=[
-            "<|prompter|>",
-            "<|endoftext|>",
-            "<|endoftext|> \n",
-            "ASSISTANT:",
-            "USER:",
-            "SYSTEM:",
-        ],
-        stream=True,
-    )
-    for out in output:
-        stream = copy.deepcopy(out)
-        temp += stream["choices"][0]["text"]
-        yield temp
-
-    history = ["init", input_prompt]
-
-
-# demo = gr.ChatInterface(
-#     generate_text,
-#     title="llama-cpp-python on GPU",
-#     description="Running LLM with https://github.com/abetlen/llama-cpp-python",
-#     examples=["tell me everything about llamas"],
-#     cache_examples=True,
-#     retry_btn=None,
-#     undo_btn="Delete Previous",
-#     clear_btn="Clear",
-# )
-# demo.queue(concurrency_count=1, max_size=5)
-# demo.launch()
-    
-nest_asyncio.apply()
-uvicorn.run(app, port=8000)

main.py

@@ -14,7 +14,7 @@ from langchain.prompts import PromptTemplate
 from langchain.chains import RetrievalQA
 from langchain.schema.runnable import RunnableBranch
 from langchain_core.runnables import RunnableLambda
-
+import torch
 
 # Logger configuration
 logging.basicConfig(level=logging.INFO,
@@ -22,6 +22,11 @@ logging.basicConfig(level=logging.INFO,
                     datefmt='%Y-%m-%d %H:%M:%S')
 logger = logging.getLogger(__name__)
 
+
+import os
+os.system("nvidia-smi")
+print("TORCH_CUDA", torch.cuda.is_available())
+
 # Add path to sys
 # sys.path.insert(0,'/opt/accelerate')
 # sys.path.insert(0,'/opt/uvicorn')