File size: 5,943 Bytes
71a6dfa |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "JvMRbVLEJlZT"
},
"outputs": [],
"source": [
"#@title 🤗 AutoTrain LLM\n",
"#@markdown In order to use this colab\n",
"#@markdown - upload train.csv to a folder named `data/`\n",
"#@markdown - train.csv must contain a `text` column\n",
"#@markdown - choose a project name if you wish\n",
"#@markdown - change model if you wish, you can use most of the text-generation models from Hugging Face Hub\n",
"#@markdown - add huggingface information (token and repo_id) if you wish to push trained model to huggingface hub\n",
"#@markdown - update hyperparameters if you wish\n",
"#@markdown - click `Runtime > Run all` or run each cell individually\n",
"\n",
"import os\n",
"!pip install -U autotrain-advanced > install_logs.txt\n",
"!autotrain setup --colab > setup_logs.txt"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "A2-_lkBS1WKA"
},
"outputs": [],
"source": [
"#@markdown ---\n",
"#@markdown #### Project Config\n",
"#@markdown Note: if you are using a restricted/private model, you need to enter your Hugging Face token in the next step.\n",
"project_name = 'my_autotrain_llm' # @param {type:\"string\"}\n",
"model_name = 'abhishek/llama-2-7b-hf-small-shards' # @param {type:\"string\"}\n",
"\n",
"#@markdown ---\n",
"#@markdown #### Push to Hub?\n",
"#@markdown Use these only if you want to push your trained model to a private repo in your Hugging Face Account\n",
"#@markdown If you dont use these, the model will be saved in Google Colab and you are required to download it manually.\n",
"#@markdown Please enter your Hugging Face write token. The trained model will be saved to your Hugging Face account.\n",
"#@markdown You can find your token here: https://huggingface.co/settings/tokens\n",
"push_to_hub = False # @param [\"False\", \"True\"] {type:\"raw\"}\n",
"hf_token = \"hf_XXX\" #@param {type:\"string\"}\n",
"repo_id = \"username/repo_name\" #@param {type:\"string\"}\n",
"\n",
"#@markdown ---\n",
"#@markdown #### Hyperparameters\n",
"learning_rate = 2e-4 # @param {type:\"number\"}\n",
"num_epochs = 1 #@param {type:\"number\"}\n",
"batch_size = 7 # @param {type:\"slider\", min:1, max:32, step:1}\n",
"block_size = 1024 # @param {type:\"number\"}\n",
"trainer = \"sft\" # @param [\"default\", \"sft\"] {type:\"raw\"}\n",
"warmup_ratio = 0.1 # @param {type:\"number\"}\n",
"weight_decay = 0.01 # @param {type:\"number\"}\n",
"gradient_accumulation = 4 # @param {type:\"number\"}\n",
"use_fp16 = True # @param [\"False\", \"True\"] {type:\"raw\"}\n",
"use_peft = True # @param [\"False\", \"True\"] {type:\"raw\"}\n",
"use_int4 = True # @param [\"False\", \"True\"] {type:\"raw\"}\n",
"lora_r = 16 #@param {type:\"number\"}\n",
"lora_alpha = 32 #@param {type:\"number\"}\n",
"lora_dropout = 0.05 #@param {type:\"number\"}\n",
"\n",
"os.environ[\"PROJECT_NAME\"] = project_name\n",
"os.environ[\"MODEL_NAME\"] = model_name\n",
"os.environ[\"PUSH_TO_HUB\"] = str(push_to_hub)\n",
"os.environ[\"HF_TOKEN\"] = hf_token\n",
"os.environ[\"REPO_ID\"] = repo_id\n",
"os.environ[\"LEARNING_RATE\"] = str(learning_rate)\n",
"os.environ[\"NUM_EPOCHS\"] = str(num_epochs)\n",
"os.environ[\"BATCH_SIZE\"] = str(batch_size)\n",
"os.environ[\"BLOCK_SIZE\"] = str(block_size)\n",
"os.environ[\"WARMUP_RATIO\"] = str(warmup_ratio)\n",
"os.environ[\"WEIGHT_DECAY\"] = str(weight_decay)\n",
"os.environ[\"GRADIENT_ACCUMULATION\"] = str(gradient_accumulation)\n",
"os.environ[\"USE_FP16\"] = str(use_fp16)\n",
"os.environ[\"USE_PEFT\"] = str(use_peft)\n",
"os.environ[\"USE_INT4\"] = str(use_int4)\n",
"os.environ[\"LORA_R\"] = str(lora_r)\n",
"os.environ[\"LORA_ALPHA\"] = str(lora_alpha)\n",
"os.environ[\"LORA_DROPOUT\"] = str(lora_dropout)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "g3cd_ED_yXXt"
},
"outputs": [],
"source": [
"!autotrain llm \\\n",
"--train \\\n",
"--model ${MODEL_NAME} \\\n",
"--project-name ${PROJECT_NAME} \\\n",
"--data-path data/ \\\n",
"--text-column text \\\n",
"--lr ${LEARNING_RATE} \\\n",
"--batch-size ${BATCH_SIZE} \\\n",
"--epochs ${NUM_EPOCHS} \\\n",
"--block-size ${BLOCK_SIZE} \\\n",
"--warmup-ratio ${WARMUP_RATIO} \\\n",
"--lora-r ${LORA_R} \\\n",
"--lora-alpha ${LORA_ALPHA} \\\n",
"--lora-dropout ${LORA_DROPOUT} \\\n",
"--weight-decay ${WEIGHT_DECAY} \\\n",
"--gradient-accumulation ${GRADIENT_ACCUMULATION} \\\n",
"$( [[ \"$USE_FP16\" == \"True\" ]] && echo \"--fp16\" ) \\\n",
"$( [[ \"$USE_PEFT\" == \"True\" ]] && echo \"--use-peft\" ) \\\n",
"$( [[ \"$USE_INT4\" == \"True\" ]] && echo \"--use-int4\" ) \\\n",
"$( [[ \"$PUSH_TO_HUB\" == \"True\" ]] && echo \"--push-to-hub --token ${HF_TOKEN} --repo-id ${REPO_ID}\" )"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.10.11"
}
},
"nbformat": 4,
"nbformat_minor": 0
} |