ready for QLoRA

llama-factory/config/mgtv_template.yaml

@@ -0,0 +1,42 @@
+### model
+model_name_or_path: ORG_NAME/MODEL_NAME
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+
+### dataset
+dataset: alpaca_mgtv_p2
+template: CHAT_TEMPLATE
+cutoff_len: 8192
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/MODEL_NAME
+logging_steps: 5
+save_steps: 35
+plot_loss: true
+# overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 6.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.01
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 35
+
+report_to: wandb
+run_name: MODEL_NAME_lora_sft

llama-factory/config/mgtv_template_4bit.yaml

@@ -0,0 +1,43 @@
+### model
+model_name_or_path: ORG_NAME/MODEL_NAME
+quantization_bit: 4
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+
+### dataset
+dataset: alpaca_mgtv_p2
+template: CHAT_TEMPLATE
+cutoff_len: 8192
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/MODEL_NAME
+logging_steps: 5
+save_steps: 35
+plot_loss: true
+# overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 6.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.01
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 35
+
+report_to: wandb
+run_name: MODEL_NAME_lora_sft

llm_toolkit/setup_lf.py

@@ -0,0 +1,60 @@
+import os, json, sys, yaml
+from dotenv import find_dotenv, load_dotenv
+
+found_dotenv = find_dotenv(".env")
+
+if len(found_dotenv) == 0:
+    found_dotenv = find_dotenv(".env.example")
+print(f"loading env vars from: {found_dotenv}")
+load_dotenv(found_dotenv, override=False)
+
+path = os.path.dirname(found_dotenv)
+print(f"Adding {path} to sys.path")
+sys.path.append(path)
+
+from llm_toolkit.llm_utils import *
+from llm_toolkit.logical_reasoning_utils import *
+
+org_name = os.getenv("ORG_NAME")
+model_name = os.getenv("MODEL_NAME")
+chat_template = os.getenv("CHAT_TEMPLATE")
+filename = os.getenv("YAML")
+data_path = os.getenv("DATA_PATH")
+
+print(org_name, model_name, chat_template, filename, data_path)
+
+if not filename:
+    print("Error: Environment variable YAML not set")
+    sys.exit(1)
+
+if not os.path.exists(filename):
+    print(f"Error: File {filename} not found")
+    sys.exit(1)
+
+file = open(filename)
+yaml_content = file.read()
+file.close()
+
+keys = ["ORG_NAME", "MODEL_NAME", "CHAT_TEMPLATE"]
+for key in keys:
+    yaml_content = yaml_content.replace(key, os.getenv(key))
+
+# print(f"YAML content:\n{yaml_content}")
+parts = filename.split("/")
+parts[-1] = "models"
+parts.append(f"{os.getenv('MODEL_NAME')}.yaml")
+filename = "/".join(parts)
+print(f"Writing to {filename}")
+
+# Create the parent directory if it doesn't exist
+os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+file = open(filename, "w")
+file.write(yaml_content)
+file.close()
+
+y = yaml.safe_load(open(filename))
+print(f"{filename}:\n", json.dumps(y, indent=2))
+
+dataset = load_alpaca_data(data_path, using_p1=False)
+print_row_details(dataset, [0, -1])

scripts/tune-lf_v2.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/../llama-factory/
+echo Current Directory:
+pwd
+
+export ORG_NAME=$1
+export MODEL_NAME=$2
+export CHAT_TEMPLATE=$3
+export DATA_PATH=../datasets/mgtv
+export YAML=config/mgtv_template_4bit.yaml
+
+export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True"
+
+python ../llm_toolkit/setup_lf.py
+llamafactory-cli train config/models/$MODEL_NAME.yaml

scripts/tune-mgtv-4bit.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+# pip install --upgrade transformers
+
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_4bit.csv
+
+$BASEDIR/scripts/tune-lf_v2.sh  Qwen Qwen2-72B-Instruct qwen
+
+$BASEDIR/scripts/tune-lf_v2.sh  shenzhi-wang Llama3.1-70B-Chinese-Chat llama3
+

scripts/tune-mgtv.sh

@@ -1 +1 @@
-tune-mgtv-gemma.sh
+tune-mgtv-4bit.sh