{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     /Users/ryanrodriguez/nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger_eng to\n",
      "[nltk_data]     /Users/ryanrodriguez/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger_eng is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain.chains import create_retrieval_chain\n",
    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
    "from backend.app.vectorstore import get_vector_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYSTEM_ROLE_PROMPT = \"\"\"\n",
    "    You are a knowledgeable grading assistant that evaluates student answers based on provided context.\n",
    "    You should determine if answers are correct and provide constructive feedback.\n",
    "\"\"\"\n",
    "\n",
    "USER_ROLE_PROMPT = \"\"\"\n",
    "    Grade the following student answer based on the provided context about {query}.\n",
    "    \n",
    "    Context: {context}\n",
    "    \n",
    "    Question: {problem}\n",
    "    Student Answer: {answer}\n",
    "    \n",
    "    Evaluate if the answer is correct and provide brief feedback. Start with either \"Correct\" or \"Incorrect\" \n",
    "    followed by a brief explanation of why. Focus on the accuracy based on the context provided.\n",
    "    \n",
    "    Always begin your response with \"Correct\" or \"Incorrect\" and then provide a brief explanation of why.\n",
    "\n",
    "    Your response should be direct and clear, for example:\n",
    "    \"Correct. The answer accurately explains [reason]\" or \n",
    "    \"Incorrect. While [partial understanding], the answer misses [key point]\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "chat_prompt = ChatPromptTemplate.from_messages([\n",
    "    (\"system\", SYSTEM_ROLE_PROMPT),\n",
    "    (\"user\", USER_ROLE_PROMPT)\n",
    "])\n",
    "\n",
    "openai_chat_model = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0.7)\n",
    "\n",
    "retriever = get_vector_db().as_retriever(search_kwargs={\"k\": 2})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from operator import itemgetter\n",
    "\n",
    "simple_rag = (\n",
    "    {\n",
    "        # Use the query to retrieve documents from the vectorstore\n",
    "        \"context\": itemgetter(\"query\") | retriever | (lambda docs: \"\\n\\n\".join([doc.page_content for doc in docs])),\n",
    "        # Pass through all other inputs directly\n",
    "        \"query\": itemgetter(\"query\"),\n",
    "        \"problem\": itemgetter(\"problem\"),\n",
    "        \"answer\": itemgetter(\"answer\")\n",
    "    } \n",
    "    | chat_prompt\n",
    "    | openai_chat_model\n",
    "    | StrOutputParser()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Incorrect. While the answer mentions storing and retrieving documents, it misses the key point that the purpose of the indexing component in a RAG application is to ingest data from a source, index it, and facilitate efficient retrieval of relevant data at runtime.'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_result = simple_rag.invoke(\n",
    "    {\n",
    "        \"query\": \"RAG\",\n",
    "        \"problem\": \"What is the purpose of the indexing component in a RAG application?\",\n",
    "        \"answer\": \"The indexing component is used to store and retrieve documents efficiently.\"\n",
    "    }\n",
    ")\n",
    "raw_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['What are the two main components of a typical RAG application?',\n",
       " 'What is the purpose of the indexing component in a RAG application?',\n",
       " \"What are the steps involved in the 'Load' phase of indexing?\",\n",
       " 'Why is splitting text into smaller chunks important in the context of RAG applications?',\n",
       " 'How does the retrieval and generation component of a RAG application process user queries?']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "result = json.loads(raw_result)\n",
    "result[\"questions\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "argument 'text': 'dict' object cannot be converted to 'PyString'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[12], line 7\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mretriever\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mRunnablePassthrough\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mproblem\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mRunnablePassthrough\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43manswer\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mRunnablePassthrough\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m}\u001b[49m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;241;43m|\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mchat_prompt\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m        \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mRAG\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mproblem\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mWhat is the purpose of the indexing component in a RAG application?\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43manswer\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mThe indexing component is used to store and retrieve documents efficiently.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\n\u001b[1;32m     12\u001b[0m \u001b[43m        \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:3022\u001b[0m, in \u001b[0;36mRunnableSequence.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m   3020\u001b[0m context\u001b[38;5;241m.\u001b[39mrun(_set_config_context, config)\n\u001b[1;32m   3021\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m i \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 3022\u001b[0m     \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mcontext\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstep\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3023\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   3024\u001b[0m     \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m context\u001b[38;5;241m.\u001b[39mrun(step\u001b[38;5;241m.\u001b[39minvoke, \u001b[38;5;28minput\u001b[39m, config)\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:3729\u001b[0m, in \u001b[0;36mRunnableParallel.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m   3724\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m get_executor_for_config(config) \u001b[38;5;28;01mas\u001b[39;00m executor:\n\u001b[1;32m   3725\u001b[0m         futures \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m   3726\u001b[0m             executor\u001b[38;5;241m.\u001b[39msubmit(_invoke_step, step, \u001b[38;5;28minput\u001b[39m, config, key)\n\u001b[1;32m   3727\u001b[0m             \u001b[38;5;28;01mfor\u001b[39;00m key, step \u001b[38;5;129;01min\u001b[39;00m steps\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m   3728\u001b[0m         ]\n\u001b[0;32m-> 3729\u001b[0m         output \u001b[38;5;241m=\u001b[39m {key: \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m key, future \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(steps, futures)}\n\u001b[1;32m   3730\u001b[0m \u001b[38;5;66;03m# finish the root run\u001b[39;00m\n\u001b[1;32m   3731\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/.local/share/uv/python/cpython-3.12.0-macos-aarch64-none/lib/python3.12/concurrent/futures/_base.py:449\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    447\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m    448\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 449\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    451\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_condition\u001b[38;5;241m.\u001b[39mwait(timeout)\n\u001b[1;32m    453\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n",
      "File \u001b[0;32m~/.local/share/uv/python/cpython-3.12.0-macos-aarch64-none/lib/python3.12/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m    400\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m    402\u001b[0m     \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    403\u001b[0m         \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m    404\u001b[0m         \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.local/share/uv/python/cpython-3.12.0-macos-aarch64-none/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39mset_exception(exc)\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:3713\u001b[0m, in \u001b[0;36mRunnableParallel.invoke.<locals>._invoke_step\u001b[0;34m(step, input, config, key)\u001b[0m\n\u001b[1;32m   3711\u001b[0m context \u001b[38;5;241m=\u001b[39m copy_context()\n\u001b[1;32m   3712\u001b[0m context\u001b[38;5;241m.\u001b[39mrun(_set_config_context, child_config)\n\u001b[0;32m-> 3713\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcontext\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3714\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstep\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3715\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3716\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchild_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3717\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_core/retrievers.py:259\u001b[0m, in \u001b[0;36mBaseRetriever.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    257\u001b[0m _kwargs \u001b[38;5;241m=\u001b[39m kwargs \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_expects_other_args \u001b[38;5;28;01melse\u001b[39;00m {}\n\u001b[1;32m    258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_new_arg_supported:\n\u001b[0;32m--> 259\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_relevant_documents\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    260\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m_kwargs\u001b[49m\n\u001b[1;32m    261\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    262\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    263\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_relevant_documents(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_kwargs)\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_core/vectorstores/base.py:1073\u001b[0m, in \u001b[0;36mVectorStoreRetriever._get_relevant_documents\u001b[0;34m(self, query, run_manager, **kwargs)\u001b[0m\n\u001b[1;32m   1071\u001b[0m _kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msearch_kwargs \u001b[38;5;241m|\u001b[39m kwargs\n\u001b[1;32m   1072\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msearch_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msimilarity\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m-> 1073\u001b[0m     docs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvectorstore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimilarity_search\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1074\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msearch_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msimilarity_score_threshold\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m   1075\u001b[0m     docs_and_similarities \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m   1076\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvectorstore\u001b[38;5;241m.\u001b[39msimilarity_search_with_relevance_scores(\n\u001b[1;32m   1077\u001b[0m             query, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_kwargs\n\u001b[1;32m   1078\u001b[0m         )\n\u001b[1;32m   1079\u001b[0m     )\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_community/vectorstores/qdrant.py:288\u001b[0m, in \u001b[0;36mQdrant.similarity_search\u001b[0;34m(self, query, k, filter, search_params, offset, score_threshold, consistency, **kwargs)\u001b[0m\n\u001b[1;32m    243\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21msimilarity_search\u001b[39m(\n\u001b[1;32m    244\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    245\u001b[0m     query: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    252\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any,\n\u001b[1;32m    253\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Document]:\n\u001b[1;32m    254\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return docs most similar to query.\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \n\u001b[1;32m    256\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    286\u001b[0m \u001b[38;5;124;03m        List of Documents most similar to the query.\u001b[39;00m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 288\u001b[0m     results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimilarity_search_with_score\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    289\u001b[0m \u001b[43m        \u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    290\u001b[0m \u001b[43m        \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    291\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mfilter\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mfilter\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    292\u001b[0m \u001b[43m        \u001b[49m\u001b[43msearch_params\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m        \u001b[49m\u001b[43moffset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m        \u001b[49m\u001b[43mscore_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mscore_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m        \u001b[49m\u001b[43mconsistency\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconsistency\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    297\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    298\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mmap\u001b[39m(itemgetter(\u001b[38;5;241m0\u001b[39m), results))\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_community/vectorstores/qdrant.py:365\u001b[0m, in \u001b[0;36mQdrant.similarity_search_with_score\u001b[0;34m(self, query, k, filter, search_params, offset, score_threshold, consistency, **kwargs)\u001b[0m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21msimilarity_search_with_score\u001b[39m(\n\u001b[1;32m    320\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    321\u001b[0m     query: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    328\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any,\n\u001b[1;32m    329\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Tuple[Document, \u001b[38;5;28mfloat\u001b[39m]]:\n\u001b[1;32m    330\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return docs most similar to query.\u001b[39;00m\n\u001b[1;32m    331\u001b[0m \n\u001b[1;32m    332\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    362\u001b[0m \u001b[38;5;124;03m        List of documents most similar to the query text and distance for each.\u001b[39;00m\n\u001b[1;32m    363\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m    364\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msimilarity_search_with_score_by_vector(\n\u001b[0;32m--> 365\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_embed_query\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m    366\u001b[0m         k,\n\u001b[1;32m    367\u001b[0m         \u001b[38;5;28mfilter\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mfilter\u001b[39m,\n\u001b[1;32m    368\u001b[0m         search_params\u001b[38;5;241m=\u001b[39msearch_params,\n\u001b[1;32m    369\u001b[0m         offset\u001b[38;5;241m=\u001b[39moffset,\n\u001b[1;32m    370\u001b[0m         score_threshold\u001b[38;5;241m=\u001b[39mscore_threshold,\n\u001b[1;32m    371\u001b[0m         consistency\u001b[38;5;241m=\u001b[39mconsistency,\n\u001b[1;32m    372\u001b[0m         \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m    373\u001b[0m     )\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_community/vectorstores/qdrant.py:2065\u001b[0m, in \u001b[0;36mQdrant._embed_query\u001b[0;34m(self, query)\u001b[0m\n\u001b[1;32m   2054\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Embed query text.\u001b[39;00m\n\u001b[1;32m   2055\u001b[0m \n\u001b[1;32m   2056\u001b[0m \u001b[38;5;124;03mUsed to provide backward compatibility with `embedding_function` argument.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2062\u001b[0m \u001b[38;5;124;03m    List of floats representing the query embedding.\u001b[39;00m\n\u001b[1;32m   2063\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   2064\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 2065\u001b[0m     embedding \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membeddings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membed_query\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2066\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   2067\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_embeddings_function \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_openai/embeddings/base.py:629\u001b[0m, in \u001b[0;36mOpenAIEmbeddings.embed_query\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21membed_query\u001b[39m(\u001b[38;5;28mself\u001b[39m, text: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[\u001b[38;5;28mfloat\u001b[39m]:\n\u001b[1;32m    621\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call out to OpenAI's embedding endpoint for embedding query text.\u001b[39;00m\n\u001b[1;32m    622\u001b[0m \n\u001b[1;32m    623\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    627\u001b[0m \u001b[38;5;124;03m        Embedding for the text.\u001b[39;00m\n\u001b[1;32m    628\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 629\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membed_documents\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_openai/embeddings/base.py:588\u001b[0m, in \u001b[0;36mOpenAIEmbeddings.embed_documents\u001b[0;34m(self, texts, chunk_size)\u001b[0m\n\u001b[1;32m    585\u001b[0m \u001b[38;5;66;03m# NOTE: to keep things simple, we assume the list may contain texts longer\u001b[39;00m\n\u001b[1;32m    586\u001b[0m \u001b[38;5;66;03m#       than the maximum context and use length-safe embedding function.\u001b[39;00m\n\u001b[1;32m    587\u001b[0m engine \u001b[38;5;241m=\u001b[39m cast(\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdeployment)\n\u001b[0;32m--> 588\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_len_safe_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_openai/embeddings/base.py:480\u001b[0m, in \u001b[0;36mOpenAIEmbeddings._get_len_safe_embeddings\u001b[0;34m(self, texts, engine, chunk_size)\u001b[0m\n\u001b[1;32m    464\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    465\u001b[0m \u001b[38;5;124;03mGenerate length-safe embeddings for a list of texts.\u001b[39;00m\n\u001b[1;32m    466\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[38;5;124;03m    List[List[float]]: A list of embeddings for each input text.\u001b[39;00m\n\u001b[1;32m    478\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    479\u001b[0m _chunk_size \u001b[38;5;241m=\u001b[39m chunk_size \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchunk_size\n\u001b[0;32m--> 480\u001b[0m _iter, tokens, indices \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_tokenize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_chunk_size\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    481\u001b[0m batched_embeddings: List[List[\u001b[38;5;28mfloat\u001b[39m]] \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    482\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m _iter:\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/langchain_openai/embeddings/base.py:441\u001b[0m, in \u001b[0;36mOpenAIEmbeddings._tokenize\u001b[0;34m(self, texts, chunk_size)\u001b[0m\n\u001b[1;32m    439\u001b[0m     token \u001b[38;5;241m=\u001b[39m encoding\u001b[38;5;241m.\u001b[39mencode(text, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mencoder_kwargs)\n\u001b[1;32m    440\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 441\u001b[0m     token \u001b[38;5;241m=\u001b[39m \u001b[43mencoding\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode_ordinary\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    443\u001b[0m \u001b[38;5;66;03m# Split tokens into chunks respecting the embedding_ctx_length\u001b[39;00m\n\u001b[1;32m    444\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m j \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(token), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding_ctx_length):\n",
      "File \u001b[0;32m~/src/Simplify/.venv/lib/python3.12/site-packages/tiktoken/core.py:73\u001b[0m, in \u001b[0;36mEncoding.encode_ordinary\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Encodes a string into tokens, ignoring special tokens.\u001b[39;00m\n\u001b[1;32m     65\u001b[0m \n\u001b[1;32m     66\u001b[0m \u001b[38;5;124;03mThis is equivalent to `encode(text, disallowed_special=())` (but slightly faster).\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;124;03m[31373, 995]\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     72\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 73\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_core_bpe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode_ordinary\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mUnicodeEncodeError\u001b[39;00m:\n\u001b[1;32m     75\u001b[0m     \u001b[38;5;66;03m# See comment in encode\u001b[39;00m\n\u001b[1;32m     76\u001b[0m     text \u001b[38;5;241m=\u001b[39m text\u001b[38;5;241m.\u001b[39mencode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-16\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-16\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreplace\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mTypeError\u001b[0m: argument 'text': 'dict' object cannot be converted to 'PyString'"
     ]
    }
   ],
   "source": [
    "({\n",
    "    \"context\": retriever,\n",
    "    \"query\": RunnablePassthrough(),\n",
    "    \"problem\": RunnablePassthrough(),\n",
    "    \"answer\": RunnablePassthrough(),\n",
    "}\n",
    "| chat_prompt).invoke(\n",
    "    {\n",
    "        \"query\": \"RAG\",\n",
    "        \"problem\": \"What is the purpose of the indexing component in a RAG application?\",\n",
    "        \"answer\": \"The indexing component is used to store and retrieve documents efficiently.\"\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': 'f1d61e037f6240d3a3671e435f9d5726', '_collection_name': 'extending_context_window_llama_3'}, page_content=\"Part 1 (this guide) introduces RAG and walks through a minimal implementation.\\n\\nPart 2 extends the implementation to accommodate conversation-style interactions and multi-step retrieval processes.\\n\\nThis tutorial will show how to build a simple Q&A application\\nover a text data source. Along the way we’ll go over a typical Q&A\\narchitecture and highlight additional resources for more advanced Q&A techniques. We’ll also see\\nhow LangSmith can help us trace and understand our application.\\nLangSmith will become increasingly helpful as our application grows in\\ncomplexity.\\n\\nIf you're already familiar with basic retrieval, you might also be interested in\\nthis high-level overview of different retrieval techniques.\\n\\nNote: Here we focus on Q&A for unstructured data. If you are interested for RAG over structured data, check out our tutorial on doing question/answering over SQL data.\\n\\nOverview\\u200b\\n\\nA typical RAG application has two main components:\"),\n",
       " Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': 'aaa4338e68d142febd9bd7969f0b3a2d', '_collection_name': 'extending_context_window_llama_3'}, page_content=\"Overview\\u200b\\n\\nA typical RAG application has two main components:\\n\\nIndexing: a pipeline for ingesting data from a source and indexing it. This usually happens offline.\\n\\nRetrieval and generation: the actual RAG chain, which takes the user query at run time and retrieves the relevant data from the index, then passes that to the model.\\n\\nNote: the indexing portion of this tutorial will largely follow the semantic search tutorial.\\n\\nThe most common full sequence from raw data to answer looks like:\\n\\nIndexing\\u200b\\n\\nLoad: First we need to load our data. This is done with Document Loaders.\\n\\nSplit: Text splitters break large Documents into smaller chunks. This is useful both for indexing data and passing it into a model, as large chunks are harder to search over and won't fit in a model's finite context window.\\n\\nStore: We need somewhere to store and index our splits, so that they can be searched over later. This is often done using a VectorStore and Embeddings model.\\n\\nRetrieval and generation\\u200b\")]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever.invoke(\"RAG\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}