commit updated notebook

HuggingFace-Inference.ipynb → notebooks/HuggingFace-Inference.ipynb

@@ -10,14 +10,68 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "94f0ccef",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please run\n",
+      "\n",
+      "python -m bitsandbytes\n",
+      "\n",
+      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "================================================================================\n",
+      "bin /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so\n",
+      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
+      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.0\n",
+      "CUDA SETUP: Detected CUDA version 112\n",
+      "CUDA SETUP: Loading binary /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /opt/conda/envs/media-reco-env-3-8 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/local/nvidia/lib:/usr/local/nvidia/lib64 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//162.21.251.11'), PosixPath('http'), PosixPath('8080')}\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
+      "Either way, this might cause trouble in the future:\n",
+      "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
+      "  warn(msg)\n",
+      "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: Compute capability < 7.5 detected! Only slow 8-bit matmul is supported for your GPU!\n",
+      "  warn(msg)\n"
+     ]
+    }
+   ],
    "source": [
     "import os\n",
     "os.chdir(\"..\")\n",
     "\n",
+    "import warnings\n",
+    "warnings.filterwarnings(\"ignore\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b050e999",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "import torch\n",
     "from transformers import GenerationConfig, LlamaTokenizer, LlamaForCausalLM\n",
     "from peft import PeftModel, PeftConfig"
@@ -33,7 +87,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "9837afb7",
    "metadata": {},
    "outputs": [],
@@ -68,13 +122,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "b53f6c18",
    "metadata": {},
    "outputs": [],
    "source": [
     "MODEL_NAME = \".\"\n",
-    "BASE_MODEL = \"decapoda-research/llama-7b-hf\"\n",
+    "BASE_MODEL = \"decapoda-research/llama-13b-hf\"\n",
     "LOAD_FINETUNED = False"
    ]
   },
@@ -88,10 +142,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "1cb5103c",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "255acd69c62c4f15b19acd43c4bc4aad",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/41 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "config = PeftConfig.from_pretrained(MODEL_NAME)\n",
     "\n",
@@ -119,7 +195,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "10372ae3",
    "metadata": {},
    "outputs": [],
@@ -152,10 +228,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "a84a4f9e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n",
+      "\n",
+      "### Response:\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -186,10 +275,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "65117ac7",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "What is the capital city of Greece and with which countries does Greece border?\n",
+      "\n",
+      "### Response: The capital city of Greece is Athens, and it borders Albania, Bulgaria, Macedonia, and Turkey.\n",
+      "\n",
+      "## See also\n",
+      "\n",
+      "* Natural language processing\n",
+      "* Question answering\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -220,10 +327,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "2ff7a5e5",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Como cocinar supa de pescado?\n",
+      "\n",
+      "### Response: \n",
+      "¿Cómo se prepara la sopa de pescado?\n",
+      "\n",
+      "## See also\n",
+      "\n",
+      "* Spanish orthography\n",
+      "* Spanish phonology\n",
+      "\n",
+      "## External links\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Como cocinar supa de pescado?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -254,10 +383,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "4073cb6d",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n",
+      "\n",
+      "### Response: A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -288,10 +430,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "a22ffa72",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+      "\n",
+      "### Response: The largest city in Greece is Thessaloniki.\n",
+      "\n",
+      "### Instruction:\n",
+      "Ποιά είναι η μεγαλύτ\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -330,7 +488,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "9cba7db1",
    "metadata": {},
    "outputs": [],
@@ -348,10 +506,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "af3a477a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n",
+      "\n",
+      "### Response: depends on what you mean by \"pieces\". If you mean slices, then you have 5 pieces of fruits. If you mean individual fruits, then you have 5 fruits\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -382,10 +553,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "eab112ae",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "What is the capital city of Greece and with which countries does Greece border?\n",
+      "\n",
+      "### Response:'Athens is the capital city of Greece. Greece shares borders with Albania, Bulgaria, Macedonia, Turkey and the Aegean and Ionian Seas. '\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -416,10 +602,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "id": "df571d56",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Como cocinar supa de pescado?\n",
+      "\n",
+      "### Response:ambos tipos de supa de pescado pueden ser cocinados en el horno o en una olla de cocina. Para preparar la supa de pesc\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Como cocinar supa de pescado?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -450,10 +649,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "id": "4975198b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n",
+      "\n",
+      "### Response: is a news article about the Russo-Ukrainian War. It was published on February 24, 2022, by The New York Times. Here are the tags\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -484,10 +696,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "id": "86bc95a9",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+      "\n",
+      "### Response:'\n",
+      "Η Αθήνα είναι η μεγαλύτερη πόλη της Ελ\n"
+     ]
+    }
+   ],
    "source": [
     "instruction = \"Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\"\n",
     "input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -511,7 +737,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5c7bd8d5",
+   "id": "20b18893",
    "metadata": {},
    "outputs": [],
    "source": []