evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ryanrodriguez/src/Simplify/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from ragas.llms import LangchainLLMWrapper\n",
    "from ragas.embeddings import LangchainEmbeddingsWrapper\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "generator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"))\n",
    "generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: static/: File exists\n",
      "mkdir: static/training_data: File exists\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  340k  100  340k    0     0  2188k      0 --:--:-- --:--:-- --:--:-- 2196k\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!mkdir static/\n",
    "!mkdir static/training_data\n",
    "!curl https://python.langchain.com/docs/tutorials/rag/ -o static/training_data/langchain_rag_tutorial.html\n",
    "\n",
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "from langchain_community.document_loaders import BSHTMLLoader\n",
    "\n",
    "path = \"static/training_data/\"\n",
    "text_loader = DirectoryLoader(path, glob=\"*.html\", loader_cls=BSHTMLLoader)\n",
    "docs = text_loader.load()\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating personas: 100%|██████████| 1/1 [00:01<00:00,  1.24s/it]                                           \n",
      "Generating Scenarios: 100%|██████████| 2/2 [00:05<00:00,  2.76s/it]\n",
      "Generating Samples: 100%|██████████| 10/10 [00:56<00:00,  5.68s/it]\n"
     ]
    }
   ],
   "source": [
    "from ragas.testset import TestsetGenerator\n",
    "\n",
    "generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)\n",
    "dataset = generator.generate_with_langchain_docs(docs, testset_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dataset.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_input</th>\n",
       "      <th>reference_contexts</th>\n",
       "      <th>reference</th>\n",
       "      <th>synthesizer_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cud yu explane how Pydantic is used in LangCha...</td>\n",
       "      <td>[Build a Retrieval Augmented Generation (RAG) ...</td>\n",
       "      <td>The context mentions 'How to use LangChain wit...</td>\n",
       "      <td>single_hop_specifc_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>how langsmith help when buildin rag apps with ...</td>\n",
       "      <td>[the most powerful applications enabled by LLM...</td>\n",
       "      <td>LangSmith can help trace and understand your a...</td>\n",
       "      <td>single_hop_specifc_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wht is RAG in the context of AI applcations?</td>\n",
       "      <td>[Retrieval and generation: the actual RAG chai...</td>\n",
       "      <td>RAG, or Retrieval and Generation, is a process...</td>\n",
       "      <td>single_hop_specifc_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How does LangChain facilitate document retriev...</td>\n",
       "      <td>[Detailed walkthrough​ Let’s go through the ab...</td>\n",
       "      <td>LangChain facilitates document retrieval and g...</td>\n",
       "      <td>single_hop_specifc_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How does LangGraph enhance the development of ...</td>\n",
       "      <td>[a TypedDict, but can also be a Pydantic BaseM...</td>\n",
       "      <td>LangGraph enhances the development of RAG appl...</td>\n",
       "      <td>single_hop_specifc_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>How does the LangGraph platform enhance the de...</td>\n",
       "      <td>[&lt;1-hop&gt;\\n\\na TypedDict, but can also be a Pyd...</td>\n",
       "      <td>The LangGraph platform enhances the developmen...</td>\n",
       "      <td>multi_hop_specific_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>How does LangChain utilize document loaders an...</td>\n",
       "      <td>[&lt;1-hop&gt;\\n\\nRetrieval and generation: the actu...</td>\n",
       "      <td>LangChain utilizes document loaders, such as t...</td>\n",
       "      <td>multi_hop_specific_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>How does LangChain, Inc. facilitate the develo...</td>\n",
       "      <td>[&lt;1-hop&gt;\\n\\nstr query: Search context: List[Do...</td>\n",
       "      <td>LangChain, Inc. facilitates the development of...</td>\n",
       "      <td>multi_hop_specific_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>How does the RAG technique facilitate sophisti...</td>\n",
       "      <td>[&lt;1-hop&gt;\\n\\nthe most powerful applications ena...</td>\n",
       "      <td>The RAG (Retrieval Augmented Generation) techn...</td>\n",
       "      <td>multi_hop_specific_query_synthesizer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>How can LangChain JS/TS be utilized to build a...</td>\n",
       "      <td>[&lt;1-hop&gt;\\n\\nRetrieval and generation: the actu...</td>\n",
       "      <td>LangChain JS/TS can be utilized to build a Ret...</td>\n",
       "      <td>multi_hop_specific_query_synthesizer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          user_input  \\\n",
       "0  Cud yu explane how Pydantic is used in LangCha...   \n",
       "1  how langsmith help when buildin rag apps with ...   \n",
       "2       Wht is RAG in the context of AI applcations?   \n",
       "3  How does LangChain facilitate document retriev...   \n",
       "4  How does LangGraph enhance the development of ...   \n",
       "5  How does the LangGraph platform enhance the de...   \n",
       "6  How does LangChain utilize document loaders an...   \n",
       "7  How does LangChain, Inc. facilitate the develo...   \n",
       "8  How does the RAG technique facilitate sophisti...   \n",
       "9  How can LangChain JS/TS be utilized to build a...   \n",
       "\n",
       "                                  reference_contexts  \\\n",
       "0  [Build a Retrieval Augmented Generation (RAG) ...   \n",
       "1  [the most powerful applications enabled by LLM...   \n",
       "2  [Retrieval and generation: the actual RAG chai...   \n",
       "3  [Detailed walkthrough​ Let’s go through the ab...   \n",
       "4  [a TypedDict, but can also be a Pydantic BaseM...   \n",
       "5  [<1-hop>\\n\\na TypedDict, but can also be a Pyd...   \n",
       "6  [<1-hop>\\n\\nRetrieval and generation: the actu...   \n",
       "7  [<1-hop>\\n\\nstr query: Search context: List[Do...   \n",
       "8  [<1-hop>\\n\\nthe most powerful applications ena...   \n",
       "9  [<1-hop>\\n\\nRetrieval and generation: the actu...   \n",
       "\n",
       "                                           reference  \\\n",
       "0  The context mentions 'How to use LangChain wit...   \n",
       "1  LangSmith can help trace and understand your a...   \n",
       "2  RAG, or Retrieval and Generation, is a process...   \n",
       "3  LangChain facilitates document retrieval and g...   \n",
       "4  LangGraph enhances the development of RAG appl...   \n",
       "5  The LangGraph platform enhances the developmen...   \n",
       "6  LangChain utilizes document loaders, such as t...   \n",
       "7  LangChain, Inc. facilitates the development of...   \n",
       "8  The RAG (Retrieval Augmented Generation) techn...   \n",
       "9  LangChain JS/TS can be utilized to build a Ret...   \n",
       "\n",
       "                       synthesizer_name  \n",
       "0  single_hop_specifc_query_synthesizer  \n",
       "1  single_hop_specifc_query_synthesizer  \n",
       "2  single_hop_specifc_query_synthesizer  \n",
       "3  single_hop_specifc_query_synthesizer  \n",
       "4  single_hop_specifc_query_synthesizer  \n",
       "5  multi_hop_specific_query_synthesizer  \n",
       "6  multi_hop_specific_query_synthesizer  \n",
       "7  multi_hop_specific_query_synthesizer  \n",
       "8  multi_hop_specific_query_synthesizer  \n",
       "9  multi_hop_specific_query_synthesizer  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Build a Retrieval Augmented Generation (RAG) App: Part 1 | 🦜️🔗 LangChain Skip to main contentJoin us at Interrupt: The Agent AI Conference by LangChain on May 13 & 14 in San Francisco!IntegrationsAPI ReferenceMoreContributingPeopleError referenceLangSmithLangGraphLangChain HubLangChain JS/TSv0.3v0.3v0.2v0.1💬SearchIntroductionTutorialsBuild a Question Answering application over a Graph DatabaseTutorialsBuild a simple LLM application with chat models and prompt templatesBuild a ChatbotBuild a Retrieval Augmented Generation (RAG) App: Part 2Build an Extraction ChainBuild an AgentTaggingBuild a Retrieval Augmented Generation (RAG) App: Part 1Build a semantic search engineBuild a Question/Answering system over SQL dataSummarize TextHow-to guidesHow-to guidesHow to use tools in a chainHow to use a vectorstore as a retrieverHow to add memory to chatbotsHow to use example selectorsHow to add a semantic layer over graph databaseHow to invoke runnables in parallelHow to stream chat model responsesHow to add default invocation args to a RunnableHow to add retrieval to chatbotsHow to use few shot examples in chat modelsHow to do tool/function callingHow to install LangChain packagesHow to add examples to the prompt for query analysisHow to use few shot examplesHow to run custom functionsHow to use output parsers to parse an LLM response into structured formatHow to handle cases where no queries are generatedHow to route between sub-chainsHow to return structured data from a modelHow to summarize text through parallelizationHow to summarize text through iterative refinementHow to summarize text in a single LLM callHow to use toolkitsHow to add ad-hoc tool calling capability to LLMs and Chat ModelsBuild an Agent with AgentExecutor (Legacy)How to construct knowledge graphsHow to partially format prompt templatesHow to handle multiple queries when doing query analysisHow to use built-in tools and toolkitsHow to pass through arguments from one step to the nextHow to compose prompts togetherHow to handle multiple retrievers when doing query analysisHow to add values to a chain\\'s stateHow to construct filters for query analysisHow to configure runtime chain internalsHow deal with high cardinality categoricals when doing query analysisCustom Document LoaderHow to use the MultiQueryRetrieverHow to add scores to retriever resultsCachingHow to use callbacks in async environmentsHow to attach callbacks to a runnableHow to propagate callbacks constructorHow to dispatch custom callback eventsHow to pass callbacks in at runtimeHow to split by characterHow to cache chat model responsesHow to handle rate limitsHow to init any model in one lineHow to track token usage in ChatModelsHow to add tools to chatbotsHow to split codeHow to do retrieval with contextual compressionHow to convert Runnables to ToolsHow to create custom callback handlersHow to create a custom chat model classCustom EmbeddingsHow to create a custom LLM classCustom RetrieverHow to create toolsHow to debug your LLM appsHow to load CSVsHow to load documents from a directoryHow to load HTMLHow to load JSONHow to load MarkdownHow to load Microsoft Office filesHow to load PDFsHow to load web pagesHow to create a dynamic (self-constructing) chainText embedding modelsHow to combine results from multiple retrieversHow to select examples from a LangSmith datasetHow to select examples by lengthHow to select examples by maximal marginal relevance (MMR)How to select examples by n-gram overlapHow to select examples by similarityHow to use reference examples when doing extractionHow to handle long text when doing extractionHow to use prompting alone (no tool calling) to do extractionHow to add fallbacks to a runnableHow to filter messagesHybrid SearchHow to use the LangChain indexing APIHow to inspect runnablesLangChain Expression Language CheatsheetHow to cache LLM responsesHow to track token usage for LLMsRun models locallyHow to get log probabilitiesHow to reorder retrieved results to mitigate the \"lost in the middle\" effectHow to split Markdown by HeadersHow to merge consecutive messages of the same typeHow to add message historyHow to migrate from legacy LangChain agents to LangGraphHow to retrieve using multiple vectors per documentHow to pass multimodal data directly to modelsHow to use multimodal promptsHow to create a custom Output ParserHow to use the output-fixing parserHow to parse JSON outputHow to retry when a parsing error occursHow to parse text from message objectsHow to parse XML outputHow to parse YAML outputHow to use the Parent Document RetrieverHow to use LangChain with different Pydantic versionsHow to add chat historyHow to get a RAG application to add citationsHow to do per-user retrievalHow to get your RAG application to return sourcesHow to stream results from your RAG applicationHow to split JSON dataHow to recursively split text by charactersResponse metadataHow to pass runtime secrets to runnablesHow to do \"self-querying\" retrievalHow to split text based on semantic similarityHow to chain runnablesHow to save and load LangChain objectsHow to split text by tokensHow to split HTMLHow to do question answering over CSVsHow to deal with large databases when doing SQL question-answeringHow to better prompt when doing SQL question-answeringHow to do query validation as part of SQL question-answeringHow to stream runnablesHow to stream responses from an LLMHow to use a time-weighted vector store retrieverHow to return artifacts from a toolHow to use chat models to call toolsHow to disable parallel tool callingHow to force models to call a toolHow to access the RunnableConfig from a toolHow to pass tool outputs to chat modelsHow to pass run time values to toolsHow to stream events from a toolHow to stream tool callsHow to convert tools to OpenAI FunctionsHow to handle tool errorsHow to use few-shot prompting with tool callingHow to add a human-in-the-loop for toolsHow to bind model-specific toolsHow to trim messagesHow to create and query vector storesConceptual guideAgentsArchitectureAsync programming with langchainCallbacksChat historyChat modelsDocument loadersEmbedding modelsEvaluationExample selectorsFew-shot promptingConceptual guideKey-value storesLangChain Expression Language (LCEL)MessagesMultimodalityOutput parsersPrompt TemplatesRetrieval augmented generation (RAG)RetrievalRetrieversRunnable interfaceStreamingStructured outputsTestingString-in, string-out llmsText splittersTokensTool callingToolsTracingVector storesWhy LangChain?Ecosystem🦜🛠️ LangSmith🦜🕸️ LangGraphVersionsv0.3v0.2Pydantic compatibilityMigrating from v0.0 chainsHow to migrate from v0.0 chainsMigrating from ConstitutionalChainMigrating from ConversationalChainMigrating from ConversationalRetrievalChainMigrating from LLMChainMigrating from LLMMathChainMigrating from LLMRouterChainMigrating from MapReduceDocumentsChainMigrating from MapRerankDocumentsChainMigrating from MultiPromptChainMigrating from RefineDocumentsChainMigrating from RetrievalQAMigrating from StuffDocumentsChainUpgrading to LangGraph memoryHow to migrate to LangGraph memoryHow to use BaseChatMessageHistory with LangGraphMigrating off ConversationBufferMemory or ConversationStringBufferMemoryMigrating off ConversationBufferWindowMemory or ConversationTokenBufferMemoryMigrating off ConversationSummaryMemory or ConversationSummaryBufferMemoryA Long-Term Memory AgentRelease policySecurity PolicyTutorialsBuild a Retrieval Augmented Generation (RAG) App: Part 1On this pageBuild a Retrieval Augmented Generation (RAG) App: Part 1 One of']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print reference_context for the first row\n",
    "df.iloc[0]['reference_contexts']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_input            Cud yu explane how Pydantic is used in LangCha...\n",
       "reference_contexts    [Build a Retrieval Augmented Generation (RAG) ...\n",
       "reference             The context mentions 'How to use LangChain wit...\n",
       "synthesizer_name                   single_hop_specifc_query_synthesizer\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.to_pandas().iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'response': '{\\n    \"questions\": [\\n        \"What is the technique used by Q&A chatbots to answer questions about specific source information?\",\\n        \"What is the main focus of Part 1 of the tutorial on building a Retrieval Augmented Generation (RAG) App?\",\\n        \"What are the two main components of a typical RAG application?\",\\n        \"How does LangSmith assist in understanding and tracing the RAG application\\'s complexity?\",\\n        \"For what type of data does the tutorial focus on Q&A, and where should one look for RAG over structured data?\"\\n    ]\\n}', 'context': [Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': 'fbd91b9b5d134d85bd8ef1f8c00633fd', '_collection_name': 'extending_context_window_llama_3'}, page_content='Tutorials\\n\\nBuild a Retrieval Augmented Generation (RAG) App: Part 1\\n\\nBuild a Retrieval Augmented Generation (RAG) App: Part 1\\n\\nOne of the most powerful applications enabled by LLMs is sophisticated question-answering (Q&A) chatbots. These are applications that can answer questions about specific source information. These applications use a technique known as Retrieval Augmented Generation, or RAG.\\n\\nThis is a multi-part tutorial:\\n\\nPart 1 (this guide) introduces RAG and walks through a minimal implementation.\\n\\nPart 2 extends the implementation to accommodate conversation-style interactions and multi-step retrieval processes.'), Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': '3f1eeb2e5836428abc6450cbdf394386', '_collection_name': 'extending_context_window_llama_3'}, page_content=\"Part 1 (this guide) introduces RAG and walks through a minimal implementation.\\n\\nPart 2 extends the implementation to accommodate conversation-style interactions and multi-step retrieval processes.\\n\\nThis tutorial will show how to build a simple Q&A application\\nover a text data source. Along the way we’ll go over a typical Q&A\\narchitecture and highlight additional resources for more advanced Q&A techniques. We’ll also see\\nhow LangSmith can help us trace and understand our application.\\nLangSmith will become increasingly helpful as our application grows in\\ncomplexity.\\n\\nIf you're already familiar with basic retrieval, you might also be interested in\\nthis high-level overview of different retrieval techniques.\\n\\nNote: Here we focus on Q&A for unstructured data. If you are interested for RAG over structured data, check out our tutorial on doing question/answering over SQL data.\\n\\nOverview\\u200b\\n\\nA typical RAG application has two main components:\")]}\n",
      "{\n",
      "    \"questions\": [\n",
      "        \"What is the technique used by Q&A chatbots to answer questions about specific source information?\",\n",
      "        \"What is the main focus of Part 1 of the tutorial on building a Retrieval Augmented Generation (RAG) App?\",\n",
      "        \"What are the two main components of a typical RAG application?\",\n",
      "        \"How does LangSmith assist in understanding and tracing the RAG application's complexity?\",\n",
      "        \"For what type of data does the tutorial focus on Q&A, and where should one look for RAG over structured data?\"\n",
      "    ]\n",
      "}\n",
      "[Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': 'fbd91b9b5d134d85bd8ef1f8c00633fd', '_collection_name': 'extending_context_window_llama_3'}, page_content='Tutorials\\n\\nBuild a Retrieval Augmented Generation (RAG) App: Part 1\\n\\nBuild a Retrieval Augmented Generation (RAG) App: Part 1\\n\\nOne of the most powerful applications enabled by LLMs is sophisticated question-answering (Q&A) chatbots. These are applications that can answer questions about specific source information. These applications use a technique known as Retrieval Augmented Generation, or RAG.\\n\\nThis is a multi-part tutorial:\\n\\nPart 1 (this guide) introduces RAG and walks through a minimal implementation.\\n\\nPart 2 extends the implementation to accommodate conversation-style interactions and multi-step retrieval processes.'), Document(metadata={'source': 'static/data/langchain_rag_tutorial.html', '_id': '3f1eeb2e5836428abc6450cbdf394386', '_collection_name': 'extending_context_window_llama_3'}, page_content=\"Part 1 (this guide) introduces RAG and walks through a minimal implementation.\\n\\nPart 2 extends the implementation to accommodate conversation-style interactions and multi-step retrieval processes.\\n\\nThis tutorial will show how to build a simple Q&A application\\nover a text data source. Along the way we’ll go over a typical Q&A\\narchitecture and highlight additional resources for more advanced Q&A techniques. We’ll also see\\nhow LangSmith can help us trace and understand our application.\\nLangSmith will become increasingly helpful as our application grows in\\ncomplexity.\\n\\nIf you're already familiar with basic retrieval, you might also be interested in\\nthis high-level overview of different retrieval techniques.\\n\\nNote: Here we focus on Q&A for unstructured data. If you are interested for RAG over structured data, check out our tutorial on doing question/answering over SQL data.\\n\\nOverview\\u200b\\n\\nA typical RAG application has two main components:\")]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating:  38%|███▊      | 23/60 [02:09<10:37, 17.22s/it]Exception raised in Job[7]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 28807, Requested 1497. Please try again in 608ms. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  45%|████▌     | 27/60 [02:27<04:24,  8.01s/it]Exception raised in Job[19]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29408, Requested 1527. Please try again in 1.87s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  48%|████▊     | 29/60 [02:41<03:55,  7.60s/it]Exception raised in Job[25]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29856, Requested 1448. Please try again in 2.608s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  55%|█████▌    | 33/60 [02:54<01:47,  3.99s/it]Exception raised in Job[13]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29753, Requested 1556. Please try again in 2.618s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  58%|█████▊    | 35/60 [02:57<01:11,  2.84s/it]Exception raised in Job[24]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29670, Requested 1443. Please try again in 2.226s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  62%|██████▏   | 37/60 [03:09<01:36,  4.18s/it]Exception raised in Job[1]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29215, Requested 1510. Please try again in 1.45s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  68%|██████▊   | 41/60 [03:48<03:02,  9.63s/it]Exception raised in Job[36]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29802, Requested 1540. Please try again in 2.684s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  70%|███████   | 42/60 [03:52<02:25,  8.11s/it]Exception raised in Job[30]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29739, Requested 1638. Please try again in 2.754s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  77%|███████▋  | 46/60 [04:01<00:48,  3.43s/it]Exception raised in Job[35]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29488, Requested 1456. Please try again in 1.887s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  88%|████████▊ | 53/60 [16:07<12:29, 107.04s/it]Exception raised in Job[23]: TimeoutError()\n",
      "Evaluating:  90%|█████████ | 54/60 [16:29<08:09, 81.52s/it] Exception raised in Job[43]: TimeoutError()\n",
      "Evaluating:  92%|█████████▏| 55/60 [18:55<08:24, 100.94s/it]Exception raised in Job[47]: TimeoutError()\n",
      "Evaluating:  93%|█████████▎| 56/60 [19:16<05:07, 76.98s/it] Exception raised in Job[49]: TimeoutError()\n",
      "Evaluating:  95%|█████████▌| 57/60 [19:20<02:44, 54.91s/it]Exception raised in Job[53]: TimeoutError()\n",
      "Evaluating:  97%|█████████▋| 58/60 [19:34<01:25, 42.75s/it]Exception raised in Job[54]: TimeoutError()\n",
      "Evaluating:  98%|█████████▊| 59/60 [19:40<00:31, 31.75s/it]Exception raised in Job[59]: TimeoutError()\n",
      "Evaluating: 100%|██████████| 60/60 [20:20<00:00, 20.34s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'context_recall': 0.4722, 'faithfulness': 0.3333, 'factual_correctness': 0.3030, 'answer_relevancy': 0.6319, 'context_entity_recall': 0.3208, 'noise_sensitivity_relevant': 0.1333}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ***** DO NOT RUN THIS CELL AGAIN. IT WILL OVERWRITE THE RESULTS AND TAKE A LONG TIME TO RUN. *****\n",
    "\n",
    "from backend.app.problem_generator import ProblemGenerationPipeline\n",
    "\n",
    "problem_generator = ProblemGenerationPipeline(return_context=True)\n",
    "\n",
    "# # test out on first row\n",
    "# test_row = dataset.to_pandas().iloc[0]\n",
    "# response = problem_generator.generate_problems({\"query\" : test_row['user_input']})\n",
    "# print(response)\n",
    "# print(response[\"response\"])\n",
    "# print(response[\"context\"])\n",
    "\n",
    "for test_row in dataset:\n",
    "  response = problem_generator.generate_problems({\"query\" : test_row.eval_sample.user_input})\n",
    "  test_row.eval_sample.response = response[\"response\"]\n",
    "  test_row.eval_sample.retrieved_contexts = [context.page_content for context in response[\"context\"]]\n",
    "\n",
    "from ragas import EvaluationDataset\n",
    "\n",
    "evaluation_dataset = EvaluationDataset.from_pandas(dataset.to_pandas())\n",
    "\n",
    "from ragas import evaluate\n",
    "from ragas.llms import LangchainLLMWrapper\n",
    "\n",
    "evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"))\n",
    "\n",
    "from ragas.metrics import LLMContextRecall, Faithfulness, FactualCorrectness, ResponseRelevancy, ContextEntityRecall, NoiseSensitivity\n",
    "from ragas import evaluate, RunConfig\n",
    "\n",
    "custom_run_config = RunConfig(timeout=360)\n",
    "\n",
    "# result = evaluate(\n",
    "#     dataset=evaluation_dataset,\n",
    "#     metrics=[LLMContextRecall(), Faithfulness(), FactualCorrectness(), ResponseRelevancy(), ContextEntityRecall(), NoiseSensitivity()],\n",
    "#     llm=evaluator_llm,\n",
    "#     run_config=custom_run_config\n",
    "# )\n",
    "# result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Baseline Results:\n",
    "\n",
    "\n",
    "`{'context_recall': 0.4722, 'faithfulness': 0.3333, 'factual_correctness': 0.3030, 'answer_relevancy': 0.6319, 'context_entity_recall': 0.3208, 'noise_sensitivity_relevant': 0.1333}`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     /Users/ryanrodriguez/nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger_eng to\n",
      "[nltk_data]     /Users/ryanrodriguez/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger_eng is already up-to-\n",
      "[nltk_data]       date!\n",
      "Some weights of BertModel were not initialized from the model checkpoint at Rsr2425/simplify-ft-arctic-embed-l and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Evaluating:  45%|████▌     | 27/60 [02:21<03:53,  7.07s/it]Exception raised in Job[13]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29854, Requested 1458. Please try again in 2.624s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  50%|█████     | 30/60 [02:54<05:01, 10.04s/it]Exception raised in Job[24]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29611, Requested 1556. Please try again in 2.334s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  55%|█████▌    | 33/60 [02:57<01:52,  4.15s/it]Exception raised in Job[1]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29449, Requested 1474. Please try again in 1.846s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  57%|█████▋    | 34/60 [03:00<01:35,  3.67s/it]Exception raised in Job[19]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29412, Requested 1539. Please try again in 1.902s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  60%|██████    | 36/60 [03:06<01:21,  3.40s/it]Exception raised in Job[7]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29593, Requested 1482. Please try again in 2.15s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  63%|██████▎   | 38/60 [03:10<00:59,  2.71s/it]Exception raised in Job[25]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29461, Requested 1542. Please try again in 2.006s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  67%|██████▋   | 40/60 [03:15<00:51,  2.59s/it]Exception raised in Job[36]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29637, Requested 1540. Please try again in 2.354s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  68%|██████▊   | 41/60 [03:23<01:20,  4.22s/it]Exception raised in Job[31]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29149, Requested 1439. Please try again in 1.176s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  75%|███████▌  | 45/60 [03:55<02:14,  8.98s/it]Exception raised in Job[29]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29346, Requested 1324. Please try again in 1.34s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  77%|███████▋  | 46/60 [03:56<01:31,  6.56s/it]Exception raised in Job[30]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29711, Requested 1664. Please try again in 2.75s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating:  80%|████████  | 48/60 [04:19<02:01, 10.13s/it]Exception raised in Job[56]: TypeError(ufunc 'invert' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe'')\n",
      "Evaluating:  85%|████████▌ | 51/60 [04:31<00:54,  6.07s/it]Exception raised in Job[37]: RateLimitError(Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-4o in organization org-RPQcqL3OmEI37MmtuiAIn2UP on tokens per min (TPM): Limit 30000, Used 29835, Requested 1405. Please try again in 2.48s. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}})\n",
      "Evaluating: 100%|██████████| 60/60 [05:43<00:00,  5.72s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'context_recall': 0.3000, 'faithfulness': 0.8667, 'factual_correctness': 0.3322, 'answer_relevancy': 0.7749, 'context_entity_recall': 0.3884, 'noise_sensitivity_relevant': 0.2448}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_username = \"Rsr2425\"\n",
    "FINETUNED_MODEL_ID = f\"{hf_username}/simplify-ft-arctic-embed-l\"\n",
    "\n",
    "from backend.app.problem_generator import ProblemGenerationPipeline\n",
    "\n",
    "problem_generator = ProblemGenerationPipeline(return_context=True, embedding_model_id=FINETUNED_MODEL_ID)\n",
    "for test_row in dataset:\n",
    "  response = problem_generator.generate_problems({\"query\" : test_row.eval_sample.user_input})\n",
    "  test_row.eval_sample.response = response[\"response\"]\n",
    "  test_row.eval_sample.retrieved_contexts = [context.page_content for context in response[\"context\"]]\n",
    "\n",
    "from ragas import EvaluationDataset\n",
    "\n",
    "evaluation_dataset = EvaluationDataset.from_pandas(dataset.to_pandas())\n",
    "\n",
    "from ragas import evaluate\n",
    "from ragas.llms import LangchainLLMWrapper\n",
    "\n",
    "evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"))\n",
    "\n",
    "from ragas.metrics import LLMContextRecall, Faithfulness, FactualCorrectness, ResponseRelevancy, ContextEntityRecall, NoiseSensitivity\n",
    "from ragas import evaluate, RunConfig\n",
    "\n",
    "custom_run_config = RunConfig(timeout=360)\n",
    "\n",
    "result = evaluate(\n",
    "    dataset=evaluation_dataset,\n",
    "    metrics=[LLMContextRecall(), Faithfulness(), FactualCorrectness(), ResponseRelevancy(), ContextEntityRecall(), NoiseSensitivity()],\n",
    "    llm=evaluator_llm,\n",
    "    run_config=custom_run_config\n",
    ")\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finetuned Results:\n",
    "\n",
    "`{'context_recall': 0.3000, 'faithfulness': 0.8667, 'factual_correctness': 0.3322, 'answer_relevancy': 0.7749, 'context_entity_recall': 0.3884, 'noise_sensitivity_relevant': 0.2448}`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base Model Results\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'context_recall': 0.4722,\n",
       " 'faithfulness': 0.3333,\n",
       " 'factual_correctness': 0.303,\n",
       " 'answer_relevancy': 0.6319,\n",
       " 'context_entity_recall': 0.3208,\n",
       " 'noise_sensitivity_relevant': 0.1333}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Base Model Results\")\n",
    "{'context_recall': 0.4722, 'faithfulness': 0.3333, 'factual_correctness': 0.3030, 'answer_relevancy': 0.6319, 'context_entity_recall': 0.3208, 'noise_sensitivity_relevant': 0.1333}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-tuned Model Results\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'context_recall': 0.3,\n",
       " 'faithfulness': 0.8667,\n",
       " 'factual_correctness': 0.3322,\n",
       " 'answer_relevancy': 0.7749,\n",
       " 'context_entity_recall': 0.3884,\n",
       " 'noise_sensitivity_relevant': 0.2448}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Fine-tuned Model Results\")\n",
    "{'context_recall': 0.3000, 'faithfulness': 0.8667, 'factual_correctness': 0.3322, 'answer_relevancy': 0.7749, 'context_entity_recall': 0.3884, 'noise_sensitivity_relevant': 0.2448}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}