File size: 7,913 Bytes
d48047d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "gdf-gWhrxVLc"
},
"source": [
"(Loading can take a long time. Please be patient.)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "RPA4EuwJHudN"
},
"outputs": [],
"source": [
"# @title Installing Requirements\n",
"%%capture\n",
"!pip install gradio\n",
"# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n",
"!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
"\n",
"# We have to check which Torch version for Xformers (2.3 -> 0.0.27)\n",
"from torch import __version__; from packaging.version import Version as V\n",
"xformers = \"xformers==0.0.27\" if V(__version__) < V(\"2.4.0\") else \"xformers\"\n",
"!pip install --no-deps {xformers} trl peft accelerate bitsandbytes triton\n",
"\n",
"from unsloth import FastLanguageModel\n",
"import torch\n",
"max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n",
"dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n",
"load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.\n",
"\n",
"# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n",
"fourbit_models = [\n",
" #\"unsloth/Meta-Llama-3.1-8B-bnb-4bit\", # Llama-3.1 15 trillion tokens model 2x faster!\n",
" \"Solshine/mergekit-passthrough-aogjzpb\",\n",
" #\"unsloth/Meta-Llama-3.1-70B-bnb-4bit\",\n",
" #\"unsloth/Meta-Llama-3.1-405B-bnb-4bit\", # We also uploaded 4bit for 405b!\n",
" #\"unsloth/Mistral-Nemo-Base-2407-bnb-4bit\", # New Mistral 12b 2x faster!\n",
" #\"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\",\n",
" #\"unsloth/mistral-7b-v0.3-bnb-4bit\", # Mistral v3 2x faster!\n",
" #\"unsloth/mistral-7b-instruct-v0.3-bnb-4bit\",\n",
" #\"unsloth/Phi-3.5-mini-instruct\", # Phi-3.5 2x faster!\n",
" #\"unsloth/Phi-3-medium-4k-instruct\",\n",
" #\"unsloth/gemma-2-9b-bnb-4bit\",\n",
" #\"unsloth/gemma-2-27b-bnb-4bit\", # Gemma 2x faster!\n",
"] # More models at https://huggingface.co/unsloth\n",
"\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name = \"Solshine/mergekit-passthrough-aogjzpb\",\n",
" max_seq_length = max_seq_length,\n",
" dtype = dtype,\n",
" load_in_4bit = load_in_4bit,\n",
" token = \"XXXXX\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n",
"\n",
")\n",
"\n",
"alpaca_prompt = \"\"\"Below is an instruction that describes a task. Reason through the query inside <thinking> tags, and then provide your final response inside <output> tags. If you detect that you made a mistake in your reasoning at any point, correct yourself inside <reflection> tags. Write a response that appropriately completes the request.\n",
"\n",
"### Instruction:\n",
"{}\n",
"\n",
"### Response:\n",
"{}\"\"\"\n",
"\n",
"EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN\n",
"def formatting_prompts_func(examples):\n",
" instructions = examples[\"prompt\"]\n",
" outputs = examples[\"response\"]\n",
" texts = []\n",
" for instruction, output in zip(instructions, outputs):\n",
" # Must add EOS_TOKEN, otherwise your generation will go on forever!\n",
" text = alpaca_prompt.format(instruction, output) + EOS_TOKEN\n",
" texts.append(text)\n",
" return {\"text\": texts}"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "Rh4hKp8AglIY"
},
"outputs": [],
"source": [
"# Ensure model is prepared for inference\n",
"model = FastLanguageModel.for_inference(model)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 610
},
"id": "u9SeBZ6WoHD0",
"outputId": "8950b091-4b4d-4353-a6aa-ed0699cd7955"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
"Running on public URL: https://88a900ce9e5af38193.gradio.live\n",
"\n",
"This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
]
},
{
"data": {
"text/html": [
"<div><iframe src=\"https://88a900ce9e5af38193.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# @title Create Gradio User Inferface\n",
"import gradio as gr\n",
"\n",
"MASTER_PROMPT = \"\"\"You are an assistant who answers all questions by first thinking internally using thinking tags, then reflects on your thinking, then answers, and then reflects on your answer offering any needed corrections. You are concise and accurate.\"\"\"\n",
"\n",
"# Add this function for the chatbot\n",
"def chatbot(message, history):\n",
" # Combine history and master prompt and new message\n",
" full_prompt = MASTER_PROMPT + \"\\n\\n\" + \"\\n\".join([f\"Human: {h[0]}\\nAI: {h[1]}\" for h in history])\n",
" full_prompt += f\"\\nHuman: {message}\\nAI:\"\n",
"\n",
" inputs = tokenizer([alpaca_prompt.format(full_prompt, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
"\n",
" output = model.generate(**inputs, max_new_tokens=1000, temperature=0.7)\n",
" response = tokenizer.decode(output[0], skip_special_tokens=True)\n",
"\n",
" # Extract the AI's response\n",
" ai_response = response.split(\"AI:\")[-1].strip()\n",
"\n",
" return ai_response\n",
"\n",
"# Create the Gradio interface\n",
"iface = gr.ChatInterface(\n",
" chatbot,\n",
" title=\"AI Assistant\",\n",
" description=\"Ask questions, get reflections.\",\n",
" theme=\"soft\",\n",
" examples=[\n",
" \"How do I make fertilizer for tomatoes in Oregon?\",\n",
" \"What is Korean Natural Farming?\",\n",
" \"Can you explain the philosophy behind quantum physics?\",\n",
" \"Please explain quantum entanglement in simple terms.\",\n",
" ],\n",
")\n",
"\n",
"# Launch the interface\n",
"iface.launch(share=True, debug=True)\n"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
|