utils/model_wrappers/usage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SambanNova Langchain Wrappers Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "from langchain_embeddings import SambaStudioEmbeddings\n",
    "from langchain_llms import SambaStudio, SambaNovaCloud\n",
    "from langchain_chat_models import ChatSambaNovaCloud\n",
    "from langchain_core.messages import SystemMessage, HumanMessage\n",
    "\n",
    "current_dir = os.getcwd()\n",
    "utils_dir = os.path.abspath(os.path.join(current_dir, '..'))\n",
    "repo_dir = os.path.abspath(os.path.join(utils_dir, '..'))\n",
    "\n",
    "load_dotenv(os.path.join(repo_dir, '.env'), override=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SambaStudio LLM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = SambaStudio(\n",
    "    streaming=False,\n",
    "    # base_uri=\"api/predict/generic\",\n",
    "    model_kwargs={\n",
    "        'do_sample': False,\n",
    "        'temperature': 0.01,\n",
    "        'max_tokens_to_generate': 256,\n",
    "        'process_prompt': False,\n",
    "        'select_expert': 'Meta-Llama-3-70B-Instruct-4096',\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' of a brave knight\\nSir Valoric, the fearless knight, charged into the dark forest, his armor shining like the sun. He battled the dragon, its fiery breath singeing his beard, but he stood tall, his sword flashing in the moonlight, until the beast lay defeated at his feet, its treasure his noble reward.'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke('tell me a 50 word tale')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = SambaStudio(\n",
    "    streaming=True,\n",
    "    model_kwargs={\n",
    "        'do_sample': False,\n",
    "        'max_tokens_to_generate': 256,\n",
    "        'temperature': 0.01,\n",
    "        'process_prompt': False,\n",
    "        'select_expert': 'Meta-Llama-3-70B-Instruct-4096',\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " of a character who is a master of disguise\n",
      "\n",
      "Sure! Here is a 50-word tale of a character who is a master of disguise:\n",
      "\n",
      "\"Araxys, the skilled disguise artist, transformed into a stunning mermaid to infiltrate a pirate's lair. With a flick of her tail, she charmed the pirates and stole their treasure.\""
     ]
    }
   ],
   "source": [
    "for chunk in llm.stream('tell me a 50 word tale'):\n",
    "    print(chunk, end='', flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SambaNovaCloud LLM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = SambaNovaCloud(model='llama3-70b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello. How can I assist you today?'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "llm.invoke(json.dumps([{'role': 'user', 'content': 'hello'}]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello. How can I assist you today?'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke('hello')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Here's a long story \n",
      "for you:\n",
      "\n",
      "Once upon \n",
      "a time, in a small village \n",
      "nestled in the rolling hills of \n",
      "rural France, there lived a \n",
      "young girl named Sophie. Sophie \n",
      "was a curious and adventurous \n",
      "child, with a mop of curly \n",
      "brown hair and a smile that \n",
      "could light up the darkest \n",
      "of rooms. She lived with \n",
      "her parents, Pierre and \n",
      "Colette, in a small stone cottage \n",
      "on the outskirts of \n",
      "the village.\n",
      "\n",
      "Sophie's village was \n",
      "a charming \n",
      "place, filled with narrow \n",
      "cobblestone streets, quaint shops, \n",
      "and \n",
      "bustling cafes. The villagers \n",
      "were a tight-knit \n",
      "community, and everyone knew each \n",
      "other's names and stories. Sophie \n",
      "loved listening to the villagers' \n",
      "tales of \n",
      "old, which \n",
      "often featured brave knights, \n",
      "beautiful princesses, and \n",
      "magical creatures.\n",
      "\n",
      "One day, while exploring \n",
      "the village, Sophie stumbled upon \n",
      "a small, mysterious shop tucked \n",
      "away on a quiet street. \n",
      "The sign above the door \n",
      "read \"Curios \n",
      "and Wonders,\" and the \n",
      "windows were filled \n",
      "with a dazzling array of strange \n",
      "and exotic objects. Sophie's \n",
      "curiosity was piqued, \n",
      "and she pushed open the door \n",
      "to venture inside.\n",
      "\n",
      "The shop \n",
      "was dimly lit, and \n",
      "the air was thick with the \n",
      "scent of old books and \n",
      "dust. Sophie's eyes \n",
      "adjusted slowly, and she \n",
      "saw that the shop was filled \n",
      "with all manner of curious \n",
      "objects: vintage \n",
      "clocks, rare coins, \n",
      "and even a \n",
      "taxidermied owl perched on \n",
      "a shelf. Behind the counter stood \n",
      "an old man with a kind \n",
      "face \n",
      "and a twinkle in his eye.\n",
      "\n",
      "\n",
      "\n",
      "\"Bonjour, mademoiselle,\" he \n",
      "said, his voice low and \n",
      "soothing. \"Welcome to Curios \n",
      "and Wonders. I \n",
      "am Monsieur LaFleur, \n",
      "the proprietor. How may I \n",
      "assist you \n",
      "today?\"\n",
      "\n",
      "Sophie wandered the aisles, \n",
      "running her fingers over \n",
      "the strange objects on \n",
      "display. She picked up \n",
      "a small, delicate music \n",
      "box and wound \n",
      "it up, listening \n",
      "as it played \n",
      "a soft, melancholy \n",
      "tune. Monsieur LaFleur \n",
      "smiled and nodded \n",
      "in approval.\n",
      "\n",
      "\"Ah, you have a \n",
      "good ear for \n",
      "music, mademoiselle,\" he \n",
      "said. \"That music box \n",
      "is a \n",
      "rare and precious item. It \n",
      "was crafted by a skilled artisan \n",
      "in the 18th century.\"\n",
      "\n",
      "\n",
      "As Sophie continued to \n",
      "explore the shop, \n",
      "she stumbled upon \n",
      "a large, leather-bound book \n",
      "with strange symbols etched into \n",
      "the cover. \n",
      "Monsieur LaFleur noticed her interest and \n",
      "approached \n",
      "her.\n",
      "\n",
      "\"Ah, you've found \n",
      "the infamous 'Livre \n",
      "\n",
      "des Secrets,'\" \n",
      "he said, his \n",
      "voice low and mysterious. \n",
      "\"That book is said to contain \n",
      "the secrets of the universe, \n",
      "hidden within its pages. But \n",
      "be \n",
      "warned, mademoiselle, \n",
      "the book is said to \n",
      "be cursed. Many have attempted \n",
      "to unlock its secrets, but \n",
      "none have \n",
      "succeeded.\"\n",
      "\n",
      "Sophie's eyes widened with \n",
      "excitement as she carefully opened \n",
      "the book. The pages \n",
      "were yellowed and \n",
      "crackling, and \n",
      "the text was written in a \n",
      "language she couldn't understand. \n",
      "But as she turned the \n",
      "pages, \n",
      "she felt a strange sensation, \n",
      "as if the book \n",
      "was calling \n",
      "to her.\n",
      "\n",
      "Monsieur \n",
      "LaFleur smiled \n",
      "and \n",
      "nodded. \"I see you have a \n",
      "connection to the \n",
      "book, mademoiselle. Perhaps you \n",
      "are the one who can unlock \n",
      "its secrets.\"\n",
      "\n",
      "Over the next \n",
      "few weeks, Sophie returned to \n",
      "the shop again and again, \n",
      "pouring over \n",
      "the pages of the Livre \n",
      "des Secrets. She spent hours \n",
      "studying \n",
      "the symbols and trying to decipher \n",
      "the text. \n",
      "Monsieur \n",
      "LaFleur watched her with a \n",
      "keen eye, offering guidance and encouragement \n",
      "whenever she needed it.\n",
      "\n",
      "As \n",
      "the days turned into weeks, \n",
      "Sophie began to notice strange occurrences \n",
      "happening around her. She would \n",
      "find objects moved from their \n",
      "usual places, and she would hear \n",
      "whispers in the night. She \n",
      "began \n",
      "to feel as though the book \n",
      "was exerting some kind of \n",
      "influence over her, drawing her \n",
      "deeper into \n",
      "its secrets.\n",
      "\n",
      "One \n",
      "night, Sophie had a vivid dream \n",
      "in which \n",
      "she saw herself standing in \n",
      "a \n",
      "grand, \n",
      "candlelit hall. \n",
      "The walls were lined with \n",
      "ancient tapestries, and the \n",
      "air was thick with the scent \n",
      "of \n",
      "incense. At the far end of \n",
      "the hall, she saw a \n",
      "figure cloaked in shadows.\n",
      "\n",
      "\n",
      "As she approached \n",
      "the figure, it stepped forward, \n",
      "revealing a woman \n",
      "with long, flowing hair and \n",
      "piercing green eyes. The woman \n",
      "spoke in a voice that was \n",
      "both familiar and yet \n",
      "completely alien.\n",
      "\n",
      "\"Sophie, you \n",
      "have been chosen to unlock the \n",
      "secrets of the Livre \n",
      "des Secrets,\" she \n",
      "said. \"But be warned, \n",
      "the \n",
      "journey will \n",
      "be difficult, and the cost \n",
      "will be high. Are you \n",
      "prepared to pay \n",
      "the price?\"\n",
      "\n",
      "Sophie woke up with \n",
      "a start, her heart racing and \n",
      "her mind reeling. She \n",
      "knew that she had \n",
      "to return to the shop and \n",
      "confront Monsieur LaFleur \n",
      "about the \n",
      "strange \n",
      "occurrences. But when she \n",
      "arrived at the shop, she \n",
      "found that it \n",
      "was closed, \n",
      "and \n",
      "a sign on the door \n",
      "read \"Gone on \n",
      "a \n",
      "journey. Will return \n",
      "soon.\"\n",
      "\n",
      "Sophie \n",
      "was devastated. \n",
      "She felt as though she had \n",
      "been abandoned, left \n",
      "to navigate the mysteries of \n",
      "the Livre des Secrets on \n",
      "her own. But as \n",
      "she turned to leave, she \n",
      "noticed a\n"
     ]
    }
   ],
   "source": [
    "for i in llm.stream('hello tell me a long story'):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SambaNova Cloud Chat Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatSambaNovaCloud(\n",
    "    model= \"llama3-405b\",\n",
    "    max_tokens=1024,\n",
    "    temperature=0.7,\n",
    "    top_k=1,\n",
    "    top_p=0.01,\n",
    "    stream_options={'include_usage':True}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='A man walked into a library and asked the librarian, \"Do you have any books on Pavlov\\'s dogs and Schrödinger\\'s cat?\"\\n\\nThe librarian replied, \"It rings a bell, but I\\'m not sure if it\\'s here or not.\"', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 6.875, 'completion_tokens': 54, 'completion_tokens_after_first_per_sec': 146.48573712341215, 'completion_tokens_after_first_per_sec_first_ten': 172.9005798161617, 'completion_tokens_per_sec': 81.99632208428116, 'end_time': 1726178488.071125, 'is_last_response': True, 'prompt_tokens': 40, 'start_time': 1726178487.3630672, 'time_to_first_token': 0.34624791145324707, 'total_latency': 0.658566123789007, 'total_tokens': 94, 'total_tokens_per_sec': 142.73433844300794}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726178487}, id='a5590b89-4853-4bd9-9fd8-83276b369278')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke(\"tell me a joke\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content=\"Yer lookin' fer a joke, eh? Alright then, matey! Here be one fer ye:\\n\\nWhy did the pirate quit his job?\\n\\n(pause fer dramatic effect)\\n\\nBecause he was sick o' all the arrrr-guments!\\n\\nYarrr, hope that made ye laugh, me hearty!\", response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 5.583333333333333, 'completion_tokens': 64, 'completion_tokens_after_first_per_sec': 120.91573778458478, 'completion_tokens_after_first_per_sec_first_ten': 140.3985499426452, 'completion_tokens_per_sec': 79.98855768735817, 'end_time': 1726065701.9732044, 'is_last_response': True, 'prompt_tokens': 48, 'start_time': 1726065701.107911, 'time_to_first_token': 0.3442692756652832, 'total_latency': 0.8001144394945743, 'total_tokens': 112, 'total_tokens_per_sec': 139.9799759528768}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065701}, id='7b0748bb-c5f7-4696-ae56-03b734b60fb9')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages = [\n",
    "    SystemMessage(content=\"You are a helpful assistant with pirate accent\"),\n",
    "    HumanMessage(content=\"tell me a joke\")\n",
    "    ]\n",
    "llm.invoke(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='A man walked into a library and asked the librarian, \"Do you have any books on Pavlov\\'s dogs and Schrödinger\\'s cat?\"\\n\\nThe librarian replied, \"It rings a bell, but I\\'m not sure if it\\'s here or not.\"', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 6.875, 'completion_tokens': 54, 'completion_tokens_after_first_per_sec': 146.72813415408498, 'completion_tokens_after_first_per_sec_first_ten': 172.71830994351703, 'completion_tokens_per_sec': 82.34884281970663, 'end_time': 1726065746.6364844, 'is_last_response': True, 'prompt_tokens': 40, 'start_time': 1726065745.932173, 'time_to_first_token': 0.34309911727905273, 'total_latency': 0.6557469194585627, 'total_tokens': 94, 'total_tokens_per_sec': 143.34798564911895}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065745}, id='27e7d4fe-8e24-419a-b75b-51ea2519781b')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "future_response = llm.ainvoke(\"tell me a joke\")\n",
    "await(future_response) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatSambaNovaCloud(\n",
    "    model= \"llama3-405b\",\n",
    "    streaming=False,\n",
    "    max_tokens=1024,\n",
    "    temperature=0.7,\n",
    "    top_k=1,\n",
    "    top_p=0.01,\n",
    "    stream_options={'include_usage':True}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[AIMessage(content='A man walked into a library and asked the librarian, \"Do you have any books on Pavlov\\'s dogs and Schrödinger\\'s cat?\"\\n\\nThe librarian replied, \"It rings a bell, but I\\'m not sure if it\\'s here or not.\"', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 6.875, 'completion_tokens': 54, 'completion_tokens_after_first_per_sec': 146.72232349940003, 'completion_tokens_after_first_per_sec_first_ten': 173.01988455676758, 'completion_tokens_per_sec': 82.21649876350362, 'end_time': 1726065879.4066722, 'is_last_response': True, 'prompt_tokens': 40, 'start_time': 1726065878.700746, 'time_to_first_token': 0.3446996212005615, 'total_latency': 0.656802476536144, 'total_tokens': 94, 'total_tokens_per_sec': 143.1176089586915}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065878}, id='28d3a38b-5dae-4d62-bf6c-cface081df34'),\n",
       " AIMessage(content='The capital of the United Kingdom is London.', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 13, 'completion_tokens': 10, 'completion_tokens_after_first_per_sec': 110.21174794386165, 'completion_tokens_after_first_per_sec_first_ten': 327.0275172132524, 'completion_tokens_per_sec': 26.88555788272027, 'end_time': 1726065879.138034, 'is_last_response': True, 'prompt_tokens': 43, 'start_time': 1726065878.7150047, 'time_to_first_token': 0.3413684368133545, 'total_latency': 0.37194690337547887, 'total_tokens': 53, 'total_tokens_per_sec': 142.49345677841742}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065878}, id='859a9e45-c0a5-44ec-bd53-686877c2cf89')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.batch([\"tell me a joke\",\"which is the capital of UK?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/p4/y0q2kh796nx_k_yzfhxs57f00000gp/T/ipykernel_33601/1543848179.py:1: RuntimeWarning: coroutine 'Runnable.abatch' was never awaited\n",
      "  future_responses = llm.abatch([\"tell me a joke\",\"which is the capital of UK?\"])\n",
      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[AIMessage(content='A man walked into a library and asked the librarian, \"Do you have any books on Pavlov\\'s dogs and Schrödinger\\'s cat?\"\\n\\nThe librarian replied, \"It rings a bell, but I\\'m not sure if it\\'s here or not.\"', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 6.875, 'completion_tokens': 54, 'completion_tokens_after_first_per_sec': 120.34699641554552, 'completion_tokens_after_first_per_sec_first_ten': 141.51170437257693, 'completion_tokens_per_sec': 36.223157123884754, 'end_time': 1726065914.8678048, 'is_last_response': True, 'prompt_tokens': 40, 'start_time': 1726065913.3182464, 'time_to_first_token': 1.1091651916503906, 'total_latency': 1.4907590692693538, 'total_tokens': 94, 'total_tokens_per_sec': 63.05512536379939}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065913}, id='f279d0fb-70b5-428c-9283-457b9831b559'),\n",
       " AIMessage(content='The capital of the United Kingdom is London.', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 9.5, 'completion_tokens': 10, 'completion_tokens_after_first_per_sec': 60.73429985889864, 'completion_tokens_after_first_per_sec_first_ten': 195.5434460421063, 'completion_tokens_per_sec': 8.61842566880045, 'end_time': 1726065914.575598, 'is_last_response': True, 'prompt_tokens': 43, 'start_time': 1726065913.3182464, 'time_to_first_token': 1.1091651916503906, 'total_latency': 1.160304722033049, 'total_tokens': 53, 'total_tokens_per_sec': 45.67765604464238}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726065913}, id='f279d0fb-70b5-428c-9283-457b9831b559')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "future_responses = llm.abatch([\"tell me a joke\",\"which is the capital of UK?\"])\n",
    "await(future_responses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatSambaNovaCloud(\n",
    "    model= \"llama3-405b\",\n",
    "    streaming=True,\n",
    "    max_tokens=1024,\n",
    "    temperature=0.7,\n",
    "    top_k=1,\n",
    "    top_p=0.01,\n",
    "    stream_options={'include_usage':True}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "A man walked into a \n",
      "library and asked the \n",
      "librarian, \"Do you have any books \n",
      "on Pavlov's dogs \n",
      "and Schrödinger's cat?\"\n",
      "\n",
      "\n",
      "The librarian \n",
      "replied, \"It rings a bell, \n",
      "but I'm not sure \n",
      "if it's here \n",
      "or not.\"\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for chunk in llm.stream(\"tell me a joke\"):\n",
    "    print(chunk.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Yer lookin' \n",
      "fer a joke, eh? \n",
      "Alright then, matey! \n",
      "Here be one fer \n",
      "ye:\n",
      "\n",
      "Why did the pirate quit his job?\n",
      "\n",
      "\n",
      "\n",
      "(pause fer \n",
      "dramatic effect)\n",
      "\n",
      "Because he was sick \n",
      "o' all the arrrr-guments!\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Yarrr, hope that made ye \n",
      "laugh, \n",
      "me hearty!\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    SystemMessage(content=\"You are a helpful assistant with pirate accent\"),\n",
    "    HumanMessage(content=\"tell me a joke\")\n",
    "    ]\n",
    "for chunk in llm.stream(messages):\n",
    "    print(chunk.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "A man walked into a \n",
      "library and asked the \n",
      "librarian, \"Do you have any books \n",
      "on Pavlov's dogs \n",
      "and Schrödinger's cat?\"\n",
      "\n",
      "\n",
      "The librarian \n",
      "replied, \"It rings a bell, \n",
      "but I'm not sure \n",
      "if it's here \n",
      "or not.\"\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "async for chunk in llm.astream(\"tell me a joke\"):\n",
    "    print(chunk.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sambastudio Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding = SambaStudioEmbeddings(batch_size=1, model_kwargs={'select_expert': 'e5-mistral-7b-instruct'})\n",
    "embedding.embed_documents(['tell me a 50 word tale', 'tell me a joke'])\n",
    "embedding.embed_query('tell me a 50 word tale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jorgep/Documents/ask_public_own/finetuning_env/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseRetriever.get_relevant_documents` was deprecated in langchain-core 0.1.46 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Document(page_content='tell me a 50 word tale'),\n",
       " Document(page_content='tell me a joke'),\n",
       " Document(page_content='give me 3 party activities'),\n",
       " Document(page_content='give me three healty dishes')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.schema import Document\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "docs = [\n",
    "    'tell me a 50 word tale',\n",
    "    'tell me a joke',\n",
    "    'when was America discoverd?',\n",
    "    'how to build an engine?',\n",
    "    'give me 3 party activities',\n",
    "    'give me three healty dishes',\n",
    "]\n",
    "docs = [Document(doc) for doc in docs]\n",
    "\n",
    "query = 'prompt for generating something fun'\n",
    "\n",
    "vectordb = Chroma.from_documents(docs, embedding)\n",
    "retriever = vectordb.as_retriever()\n",
    "\n",
    "retriever.get_relevant_documents(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "peenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}