3

Dockerfile

@@ -1,7 +1,5 @@
 FROM python:3.9
 
-RUN wget -q zephyr-7b-beta.Q4_K_S.gguf https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/resolve/main/zephyr-7b-beta.Q4_K_S.gguf
-
 COPY requirements.txt ./requirements.txt
 
 RUN python -m pip install -U pip && \

main.py

@@ -1,13 +1,20 @@
+import os
+
 from ctransformers import AutoModelForCausalLM
 from fastapi import FastAPI
 from pydantic import BaseModel
 import requests
-#
-# llm = AutoModelForCausalLM.from_pretrained("zephyr-7b-beta.Q4_K_S.gguf",
-# model_type='mistral',
-# max_new_tokens = 1096,
-# threads = 3,
-# )
+from huggingface_hub import hf_hub_download
+
+file_name = "zephyr-7b-beta.Q4_K_S.gguf"
+if not os.path.exists(file_name):
+    hf_hub_download("TheBloke/zephyr-7B-beta-GGUF", filename=file_name, local_dir=f"./")
+
+llm = AutoModelForCausalLM.from_pretrained(file_name,
+    model_type='mistral',
+    max_new_tokens = 1096,
+    threads = 3,
+)
 
 #Pydantic object
 class validation(BaseModel):

requirements.txt

@@ -5,4 +5,4 @@ uvicorn
 requests
 python-dotenv
 ctransformers
-requests
+huggingface-hub