config for llm-tuner

README.md

@@ -1,3 +1,30 @@
 ---
 license: apache-2.0
+language:
+- en
+pipeline_tag: text2text-generation
+tags:
+- alpaca
+- llama
+- chat
+- gpt4
 ---
+
+This repository comes with LoRA checkpoint to make LLaMA into a chatbot like language model. The checkpoint is the output of instruction following fine-tuning process with the following settings on 8xA100(40G) DGX system.
+- Training script: borrowed from the official [Alpaca-LoRA](https://github.com/tloen/alpaca-lora) implementation
+- Training script:
+```shell
+python finetune.py \
+    --base_model='decapoda-research/llama-7b-hf' \
+    --data_path='alpaca_data_gpt4.json' \
+    --num_epochs=10 \
+    --cutoff_len=512 \
+    --group_by_length \
+    --output_dir='./gpt4-alpaca-lora-7b' \
+    --lora_target_modules='[q_proj,k_proj,v_proj,o_proj]' \
+    --lora_r=16 \
+    --batch_size=... \
+    --micro_batch_size=...
+```
+
+You can find how the training went from W&B report [here](https://wandb.ai/chansung18/gpt4_alpaca_lora/runs/nl1xi6ru?workspace=user-chansung18).

adapter-config.json

@@ -0,0 +1,20 @@
+{
+  "base_model_name_or_path": "decapoda-research/llama-7b-hf",
+  "bias": "none",
+  "enable_lora": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "merge_weights": false,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj"
+  ]
+}

adapter_config.json

@@ -0,0 +1,21 @@
+{
+  "base_model_name_or_path": "decapoda-research/llama-7b-hf",
+  "bias": "none",
+  "enable_lora": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "merge_weights": false,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3117d8d2972e4733004f36da6a6cbce3238ae9bb5001da2d253b2bb00a103987
+size 67201357

config.json

@@ -0,0 +1,6 @@
+{
+  "_name_or_path": "chansung/gpt4-alpaca-lora-7b",
+  "adapter_type": "lora",
+  "adapter_fusion": "dynamic",
+  "task_type": "text2text-generation"
+}

fusion-config.json

@@ -0,0 +1,8 @@
+{
+  "fusion": "dynamic",
+  "fusion_params": {
+    "temperature": 1.0,
+    "trainable": true,
+    "initial_weights": null
+  }
+}

llm-tuner-config.json

@@ -0,0 +1,21 @@
+{
+  "data_path": "path/to/your/data/file",
+  "num_epochs": 3,
+  "cutoff_len": 512,
+  "group_by_length": true,
+  "output_dir": "path/to/your/output/directory",
+  "batch_size": 8,
+  "micro_batch_size": 1,
+  "lr": 5e-5,
+  "warmup_steps": 100,
+  "weight_decay": 0.01,
+  "adam_epsilon": 1e-8,
+  "max_grad_norm": 1.0,
+  "logging_steps": 50,
+  "save_steps": 500,
+  "eval_steps": 500,
+  "overwrite_output_dir": true,
+  "do_train": true,
+  "do_eval": true,
+  "do_predict": true
+}