5th

Dockerfile

@@ -56,4 +56,5 @@ ENV HOME=/home/user \
 COPY --chown=user . /home/user/app
 
 # Run the start script
-CMD ["/app/start_server.sh"]
+#CMD ["/app/start_server.sh"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
 
 # Initialize FastAPI app
 app = FastAPI()
-NGROK_TOKEN = "2aQUM6MDkhjcPEBbIFTiu4cZBBr_sMMei8h5yejFbxFeMFuQ"  # Replace with your NGROK token
+#NGROK_TOKEN = "2aQUM6MDkhjcPEBbIFTiu4cZBBr_sMMei8h5yejFbxFeMFuQ"  # Replace with your NGROK token
 #MODEL_NAME = "/opt/Llama-2-13B-chat-GPTQ"
 #MODEL_NAME = "MediaTek-Research/Breeze-7B-Instruct-64k-v0.1"
 MODEL_NAME = "codellama/CodeLlama-7b-Instruct-hf"
@@ -215,7 +215,7 @@ def setup_chain():
 setup_chain()
 ###############
 
-if __name__ == "__main__":
+#if __name__ == "__main__":
 
     # if NGROK_TOKEN is not None:
     #     ngrok.set_auth_token(NGROK_TOKEN)
@@ -227,6 +227,6 @@ if __name__ == "__main__":
     # print("You can use {}/predict to get the assistant result.".format(public_url))
     # logger.info("You can use {}/predict to get the assistant result.".format(public_url))
 
-    nest_asyncio.apply()
-    uvicorn.run(app, port=8000)
+    #nest_asyncio.apply()
+    #uvicorn.run(app, port=8000)