app.py

%cd /content
!apt-get -y install -qq aria2

!git clone -b v1.3 https://github.com/camenduru/text-generation-webui
%cd /content/text-generation-webui
!pip install -r requirements.txt
!pip install -U gradio==3.28.3

!mkdir /content/text-generation-webui/repositories
%cd /content/text-generation-webui/repositories
!git clone -b v1.2 https://github.com/camenduru/GPTQ-for-LLaMa.git
%cd GPTQ-for-LLaMa
!python setup_cuda.py install

output_dir="/content/text-generation-webui/models/Llama-2-7b-Chat-GPTQ"

wget -O "${output_dir}/config.json" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/raw/main/config.json"
wget -O "${output_dir}/generation_config.json" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/raw/main/generation_config.json"
wget -O "${output_dir}/special_tokens_map.json" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/raw/main/special_tokens_map.json"
wget -O "${output_dir}/tokenizer.model" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/resolve/main/tokenizer.model"
wget -O "${output_dir}/tokenizer_config.json" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/raw/main/tokenizer_config.json"
wget -O "${output_dir}/gptq_model-4bit-128g.safetensors" "https://huggingface.co/4bit/Llama-2-7b-Chat-GPTQ/resolve/main/gptq_model-4bit-128g.safetensors"

%cd /content/text-generation-webui
!python server.py --share --chat --wbits 4 --groupsize 128 --model_type llama