{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "6300c7d8-d6c8-4178-94c5-4ee767cfd825", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting tonic-validate\n", " Downloading tonic_validate-2.1.1-py3-none-any.whl (14 kB)\n", "Requirement already satisfied: openai>=1.0.0 in /home/mn/.local/lib/python3.10/site-packages (from tonic-validate) (1.4.0)\n", "Requirement already satisfied: pandas>=1.2.3 in /home/mn/.local/lib/python3.10/site-packages (from tonic-validate) (2.1.4)\n", "Requirement already satisfied: sniffio in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (1.3.0)\n", "Requirement already satisfied: tqdm>4 in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (4.66.1)\n", "Requirement already satisfied: typing-extensions<5,>=4.5 in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (4.8.0)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->tonic-validate) (1.7.0)\n", "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (2.5.2)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (0.25.2)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /home/mn/.local/lib/python3.10/site-packages (from openai>=1.0.0->tonic-validate) (3.7.1)\n", "Requirement already satisfied: tzdata>=2022.1 in /home/mn/.local/lib/python3.10/site-packages (from pandas>=1.2.3->tonic-validate) (2023.3)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas>=1.2.3->tonic-validate) (2022.1)\n", "Requirement already satisfied: numpy<2,>=1.22.4 in /home/mn/.local/lib/python3.10/site-packages (from pandas>=1.2.3->tonic-validate) (1.26.2)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /home/mn/.local/lib/python3.10/site-packages (from pandas>=1.2.3->tonic-validate) (2.8.2)\n", "Requirement already satisfied: idna>=2.8 in /usr/lib/python3/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->tonic-validate) (3.3)\n", "Requirement already satisfied: exceptiongroup in /home/mn/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai>=1.0.0->tonic-validate) (1.2.0)\n", "Requirement already satisfied: httpcore==1.* in /home/mn/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->tonic-validate) (1.0.2)\n", "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->tonic-validate) (2020.6.20)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /home/mn/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->tonic-validate) (0.14.0)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /home/mn/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->tonic-validate) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.14.5 in /home/mn/.local/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->tonic-validate) (2.14.5)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.2.3->tonic-validate) (1.16.0)\n", "Installing collected packages: tonic-validate\n", "Successfully installed tonic-validate-2.1.1\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install tonic-validate" ] }, { "cell_type": "code", "execution_count": 10, "id": "31b2d0ec-def3-4df9-9ab6-ba46810590a2", "metadata": {}, "outputs": [], "source": [ "import openai\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "5acf03c7-9e4b-421d-a808-a97ab3da4ed7", "metadata": {}, "outputs": [], "source": [ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "from langchain.document_loaders import TextLoader\n", "from langchain.vectorstores import Chroma\n", "import os\n", "from langchain.document_loaders import DirectoryLoader\n", "from dotenv import load_dotenv\n", "from langchain.llms import OpenAI\n", "import json\n", "from langchain.embeddings.openai import OpenAIEmbeddings\n", "from langchain.chat_models import ChatOpenAI\n", "from langchain.chains import RetrievalQA\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 12, "id": "7f23e6c0-0c41-4437-a8ec-ee09e1805e3a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "page_content='According to a study from the University of Cambridge, at least half of developers’ efforts are spent debugging and not actively programming, which costs the software industry an estimated $312 billion per year. But so far, only a handful of code-generating AI systems have been made freely available to the public — reflecting the commercial incentives of the organizations building them (see: Replit).\\n\\nStarCoder, which by contrast is licensed to allow for royalty-free use by anyone, including corporations, was trained on over 80 programming languages as well as text from GitHub repositories, including documentation and programming notebooks. StarCoder integrates with Microsoft’s Visual Studio Code code editor and, like OpenAI’s ChatGPT, can follow basic instructions (e.g., “create an app UI”) and answer questions about code.' metadata={'source': 'new_articles/05-04-hugging-face-and-servicenow-release-a-free-code-generating-model.txt'}\n" ] } ], "source": [ "#text splitting\n", "loader = DirectoryLoader('./new_articles/', glob=\"./*.txt\", loader_cls=TextLoader)\n", "documents = loader.load()\n", "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n", "texts = text_splitter.split_documents(documents)\n", "print(texts[1]) " ] }, { "cell_type": "code", "execution_count": 13, "id": "6bcbef63-0c39-4236-bb27-87e11dbee5ff", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#load env variables(keys}\n", "load_dotenv()" ] }, { "cell_type": "code", "execution_count": 14, "id": "84d648f5-c84a-40b6-9765-9f04d615906d", "metadata": {}, "outputs": [], "source": [ "from openai import OpenAI\n", "OPENAI_API_KEY= os.getenv(\"OPENAI_API_KEY\")\n", "# embeddings = OpenAIEmbeddings(openai_api_key=openai.api_key)\n", "embedding_function = OpenAIEmbeddings(\n", "api_key=OPENAI_API_KEY)" ] }, { "cell_type": "code", "execution_count": 15, "id": "1b08991b-6190-41c2-883c-05360d7e4682", "metadata": {}, "outputs": [], "source": [ "vectordb = Chroma.from_documents(documents=texts, \n", " embedding=OpenAIEmbeddings())" ] }, { "cell_type": "code", "execution_count": 33, "id": "1a845b21-278a-499e-bddc-735577960e2c", "metadata": {}, "outputs": [], "source": [ "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0.8)\n", "chain = RetrievalQA.from_chain_type(llm=llm,\n", " chain_type=\"stuff\",\n", " retriever=vectordb.as_retriever())\n", "from langchain.chains.question_answering import load_qa_chain\n", "chain = load_qa_chain(llm, chain_type=\"stuff\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "89166788-2e9f-48ab-8d17-7dd353b0ff52", "metadata": {}, "outputs": [], "source": [ "#Load 10 questions and answers about the Paul Graham essays as a benchmark for how the RAG system should answer questions.\n", "with open(\"question_and_answer_list.json\", \"r\") as f:\n", " question_and_answer_list =json.load(f)" ] }, { "cell_type": "code", "execution_count": 18, "id": "238595b9-be93-4b8b-9de6-88f042fb3596", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'What is ChatGPT, and how has it been used in various applications?'" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ex_q_and_a = question_and_answer_list[1]\n", "ex_q_and_a[\"question\"]" ] }, { "cell_type": "code", "execution_count": 21, "id": "8baa0b92-5816-4746-9928-2324ca9e2962", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'ChatGPT is a text-generating AI chatbot developed by OpenAI. It has been widely used for writing essays, code, and more based on short text prompts, enhancing productivity. Major brands have experimented with it for generating ad and marketing copy. OpenAI continually invests in ChatGPT, upgrading it to GPT-4, a more advanced language-writing model. The chatbot has been integrated into various applications, including search engines, customer service, and even an iPhone customization app called SuperChat.'" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ex_q_and_a[\"answer\"]" ] }, { "cell_type": "code", "execution_count": 22, "id": "d545ac1d-cd2f-4704-86f7-da1af5ecc8f7", "metadata": {}, "outputs": [], "source": [ "from tonic_validate import ValidateApi, ValidateScorer, Benchmark, LLMResponse\n", "from tonic_validate.metrics import AnswerConsistencyMetric, AugmentationAccuracyMetric" ] }, { "cell_type": "code", "execution_count": 23, "id": "6d2c0ecc-bb57-488a-965f-1ea1bff1137b", "metadata": {}, "outputs": [], "source": [ "from tonic_validate.validate_scorer import ValidateScorer \n", "# metrics\n", "from tonic_validate.metrics.answer_consistency_metric import AnswerConsistencyMetric\n", "from tonic_validate.metrics.answer_similarity_metric import AnswerSimilarityMetric\n", "from tonic_validate.metrics.augmentation_accuracy_metric import AugmentationAccuracyMetric\n", "from tonic_validate.metrics.augmentation_precision_metric import AugmentationPrecisionMetric\n", "from tonic_validate.metrics.retrieval_precision_metric import RetrievalPrecisionMetric\n", "# llm utils\n", "from tonic_validate.classes.llm_response import LLMResponse\n", "from tonic_validate.classes.benchmark import BenchmarkItem" ] }, { "cell_type": "code", "execution_count": 24, "id": "118570f4-e116-4462-a6fd-01cf4fb0ed4a", "metadata": {}, "outputs": [], "source": [ "metrics = [\n", " AnswerSimilarityMetric(),\n", " RetrievalPrecisionMetric(),\n", " AugmentationAccuracyMetric(),\n", " AugmentationPrecisionMetric(),\n", " AnswerConsistencyMetric()\n", "]\n", "# can use an OpenAI chat completion model\n", "# llm_evaluator = \"gpt-3.5-turbo\"\n", "llm_evaluator = \"gpt-4-1106-preview\"\n", "validate_scorer = ValidateScorer(\n", " metrics, llm_evaluator\n", ")\n" ] }, { "cell_type": "markdown", "id": "21c86d7f-de97-4fc2-801a-3a963825f88e", "metadata": {}, "source": [ "For one benchMark" ] }, { "cell_type": "code", "execution_count": 34, "id": "de465c2e-b0f4-453d-82f9-d5017f8b300f", "metadata": {}, "outputs": [], "source": [ "# example BenchmarkItem\n", "question = ex_q_and_a[\"question\"]\n", "reference_answer = ex_q_and_a[\"answer\"]\n", "benchmark_item = BenchmarkItem(\n", " question=question,\n", " answer=reference_answer\n", ")\n", "\n", "# example LLMResponse\n", "llm_answer = response\n", "context_list = [document.page_content for document in matching_docs]\n", "llm_response = LLMResponse(\n", " llm_answer=llm_answer,\n", " llm_context_list=context_list,\n", " benchmark_item=benchmark_item\n", ")\n", "\n", "responses = [llm_response]\n", "\n", "response_scores = validate_scorer.score_run(responses)" ] }, { "cell_type": "code", "execution_count": 35, "id": "094656b2-01e6-497f-b132-e46e7068fa0c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'answer_similarity': 4.0,\n", " 'retrieval_precision': 1.0,\n", " 'augmentation_accuracy': 0.5,\n", " 'augmentation_precision': 0.5,\n", " 'answer_consistency': 1.0}" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "response_scores.run_data[0].scores" ] }, { "cell_type": "raw", "id": "2b4ee7b4-a1ca-4b1d-9e2a-7491105d0349", "metadata": {}, "source": [ "For Multiple BenchMarks" ] }, { "cell_type": "code", "execution_count": 30, "id": "e27f4bd2-aef0-45ed-b239-9a587521ca90", "metadata": {}, "outputs": [], "source": [ "# responses = []\n", "\n", "# for q_and_a in question_and_answer_list:\n", "# query = ex_q_and_a[\"question\"]\n", "# matching_docs =vectordb.similarity_search(query)\n", "# response=chain.run(input_documents=matching_docs, question=query)\n", " \n", "# benchmark_item = BenchmarkItem(\n", "# question=q_and_a[\"question\"],\n", "# answer=q_and_a[\"answer\"]\n", "# )\n", "\n", "# llm_response = LLMResponse(\n", "# llm_answer=response,\n", "# llm_context_list=[document.page_content for document in matching_docs],\n", "# benchmark_item=benchmark_item\n", "# )\n", "\n", "# responses.append(llm_response)" ] }, { "cell_type": "code", "execution_count": 31, "id": "f4bdd2ac-630d-4313-975b-8220fc69fea6", "metadata": {}, "outputs": [], "source": [ "# response_scores = validate_scorer.score_run(responses)" ] }, { "cell_type": "code", "execution_count": 36, "id": "7bd5591a-faf9-47ec-82bb-9bbe694a3029", "metadata": {}, "outputs": [], "source": [ "validate_api = ValidateApi(\"lYNVCKKhUtssnDKSQGX48HkK4Mr_62dr9Hj7U0_K98c\")\n", "validate_api.upload_run(\"944f02fe-c106-45c0-af5d-74e4bd0518b7\", response_scores)" ] }, { "cell_type": "code", "execution_count": 17, "id": "6147a691-a919-4a4f-9ba7-00ed3c0efb88", "metadata": {}, "outputs": [], "source": [ "# def make_scores_df(response_scores):\n", "# scores_df = {\n", "# \"question\": [],\n", "# \"reference_answer\": [],\n", "# \"llm_answer\": [],\n", "# \"retrieved_context\": []\n", "# }\n", "# for score_name in response_scores.overall_scores:\n", "# scores_df[score_name] = []\n", "# for data in response_scores.run_data:\n", "# scores_df[\"question\"].append(data.reference_question)\n", "# scores_df[\"reference_answer\"].append(data.reference_answer)\n", "# scores_df[\"llm_answer\"].append(data.llm_answer)\n", "# scores_df[\"retrieved_context\"].append(data.llm_context)\n", "# for score_name, score in data.scores.items():\n", "# scores_df[score_name].append(score)\n", "# return pd.DataFrame(scores_df)\n", " \n" ] }, { "cell_type": "code", "execution_count": 18, "id": "d94ac537-38d6-401d-b9bb-55a7f1c5598b", "metadata": {}, "outputs": [], "source": [ "# scores_df = make_scores_df(response_scores)" ] }, { "cell_type": "code", "execution_count": 21, "id": "a8179a51-6d77-4599-9b30-4553faf29d2d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
questionreference_answerllm_answerretrieved_contextanswer_similarityretrieval_precisionaugmentation_accuracyaugmentation_precisionanswer_consistency
0What does Pando plan to use the $30 million ra...Pando intends to use the funds for expanding i...Pando plans to use the $30 million raised in i...[Signaling that investments in the supply chai...4.00.50.250.51.0
\n", "
" ], "text/plain": [ " question \\\n", "0 What does Pando plan to use the $30 million ra... \n", "\n", " reference_answer \\\n", "0 Pando intends to use the funds for expanding i... \n", "\n", " llm_answer \\\n", "0 Pando plans to use the $30 million raised in i... \n", "\n", " retrieved_context answer_similarity \\\n", "0 [Signaling that investments in the supply chai... 4.0 \n", "\n", " retrieval_precision augmentation_accuracy augmentation_precision \\\n", "0 0.5 0.25 0.5 \n", "\n", " answer_consistency \n", "0 1.0 " ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# scores_df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "3c1f4bce-d84e-4e9f-a422-c1d146eb1a32", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }