Server Deployment

Dockerfile

@@ -8,4 +8,4 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY . .
 
-CMD ["gunicorn", "-b", "0.0.0.0:7860","main:app"]
+CMD ["gunicorn", "-b", "0.0.0.0:7860","model:app"]

main.py

@@ -1,5 +1,6 @@
 from flask import Flask, request, jsonify
 from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 
 # Initialize the Llama model with chat format set to "llama-2"
 llm = Llama(model_path="E:/langchain-chat-gui-main/langchain-chat-gui-main/llama-2-7b-chat.Q2_K.gguf", chat_format="llama-2")

model.py

@@ -0,0 +1,21 @@
+from huggingface_hub import hf_hub_download
+
+# Define the repository ID (username/repository_name)
+repo_id = 'TheBloke/Llama-2-7B-Chat-GGUF'
+
+# Define the filename you want to download
+filename = 'llama-2-7b-chat.Q2_K.gguf'
+
+# Define the cache directory (optional)
+# If not provided, the default cache directory will be used
+cache_dir = './path_to_cache_directory'
+
+# Download the file
+file_path = hf_hub_download(
+    repo_id=repo_id,
+    filename=filename,
+    cache_dir=cache_dir
+)
+
+# The file_path variable now contains the local path to the downloaded file
+print(f"File downloaded to: {file_path}")

requirements.txt

@@ -10,4 +10,4 @@ MarkupSafe==2.1.5
 numpy==1.26.4
 typing_extensions==4.11.0
 Werkzeug==3.0.3
-
+huggingface_hub