| MODEL_PATH = "" | |
| # if MODEL_PATH is "", default llama.cpp/gptq models | |
| # will be downloaded to: ./models | |
| # Example ggml path: | |
| #MODEL_PATH = "models/llama2_7b_chat.bin" | |
| # options: llama.cpp, gptq, transformers | |
| BACKEND_TYPE = "llama.cpp" | |
| #BACKEND_TYPE = "llama2.cu" | |
| # only for transformers bitsandbytes 8 bit | |
| LOAD_IN_8BIT = False | |
| MAX_MAX_NEW_TOKENS = 2048 | |
| DEFAULT_MAX_NEW_TOKENS = 1024 | |
| MAX_INPUT_TOKEN_LENGTH = 4000 | |
| DEFAULT_SYSTEM_PROMPT = "" | |