Refactor retrieval_augmented_qa_chain to improve accuracy and update the 3rd pipeline with MultiQueryRetriever to use the first chunking strategy

deepPDF.ipynb

@@ -48,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -66,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -84,7 +84,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -100,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -115,7 +115,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -124,7 +124,7 @@
        "765"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -142,7 +142,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -153,7 +153,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -169,7 +169,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -185,7 +185,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -194,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -220,7 +220,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -229,16 +229,11 @@
     "from langchain.schema.runnable import RunnablePassthrough\n",
     "\n",
     "retrieval_augmented_qa_chain = (\n",
-    "    # INVOKE CHAIN WITH: {\"question\" : \"<<SOME USER QUESTION>>\"}\n",
-    "    # \"question\" : populated by getting the value of the \"question\" key\n",
-    "    # \"context\"  : populated by getting the value of the \"question\" key and chaining it into the base_retriever\n",
+    "\n",
     "    {\"context\": itemgetter(\"question\") | qdrant_retriever, \"question\": itemgetter(\"question\")}\n",
-    "    # \"context\"  : is assigned to a RunnablePassthrough object (will not be called or considered in the next step)\n",
-    "    #              by getting the value of the \"context\" key from the previous step\n",
+    "\n",
     "    | RunnablePassthrough.assign(context=itemgetter(\"context\"))\n",
-    "    # \"response\" : the \"context\" and \"question\" values are used to format our prompt object and then piped\n",
-    "    #              into the LLM and stored in a key called \"response\"\n",
-    "    # \"context\"  : populated by getting the value of the \"context\" key from the previous step\n",
+    "\n",
     "    | {\"response\": rag_prompt | openai_chat_model, \"context\": itemgetter(\"context\")}\n",
     ")"
    ]
@@ -252,7 +247,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -261,7 +256,7 @@
        "\"The total value of 'Cash and cash equivalents' as of December 31, 2023, was $41,862.\""
       ]
      },
-     "execution_count": 19,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -273,16 +268,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "\"Sorry, the context is unrelated to the query, I can't answer.\""
+       "\"The context provided does not directly mention the names of Meta's Directors (members of the Board of Directors). Therefore, I cannot answer the query based on the given information. Sorry, the context is unrelated to the query, I can't answer.\""
       ]
      },
-     "execution_count": 20,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -294,19 +289,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Document(page_content='to having a skilled, inclusive and diverse workforce because we believe cognitive diversity fuels innovation. To aid in this effort, we have taken steps to reduce\\nbias from our hiring processes and performance management systems, as well as offering learning and development courses for our employees.\\nCorporate Information\\nWe were incorporated in Delaware in July 2004. We completed our initial public offering in May 2012 and our Class\\xa0A common stock is currently listed\\non the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '18b4e9ef498c49b5867839caa18d2e9a', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content=\"(Exact name of registrant as specified in its charter)\\n__________________________\\nDelaware\\n20-1665019\\n(State or other jurisdiction of incorporation or organization)\\n(I.R.S. Employer Identification Number)\\n1 Meta Way, Menlo Park, California 94025\\n(Address of principal executive offices and Zip Code)\\n(650)\\xa0543-4800\\n(Registrant's telephone number, including area code)\\n__________________________\\nSecurities registered pursuant to Section 12(b) of the Act:\\nTitle of each class\\nTrading symbol(s)\\nName of each exchange on which registered\\nClass A Common Stock, $0.000006 par value\\nMETA\\nThe Nasdaq Stock Market LLC\\nSecurities registered pursuant to Section 12(g) of the Act: None\", metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 0, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '91fb05632758461883fe050d26afeae1', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content='making to real-time optimizations to post-campaign analytics. We work directly with these advertisers, as well as through advertising agencies and resellers.\\nWe operate offices in approximately 90\\xa0cities around the globe, the majority of which have a sales presence. We also invest in and rely on self-service tools to\\nprovide direct customer support to our users and partners.\\nFor our RL products, our sales and operations efforts utilize third-party sales channels such as retailers, resellers, and our direct-to-consumer channel,\\nMeta.com. These efforts are focused on driving consumer and enterprise sales and adoption of our Meta Quest portfolio of products and Ray-Ban Meta smart\\nglasses.\\nMarketing\\nHistorically, our communities have generally grown organically with people inviting their friends to connect with them, supported by internal efforts to\\nstimulate awareness and interest. In addition, we have invested and will continue to invest in marketing our products and services to grow our brand and help', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 9, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'a474117d06bd422387c0fa613b547ee5', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content='decision-making and prioritization of cybersecurity countermeasures and risk mitigation strategies. Our risk mitigation strategies include a broad variety of\\ntechnical and operational measures, as well as annual cybersecurity and privacy training for all of our employees.\\nIn addition, we maintain specific policies and practices governing our third-party security risks, including our third-party assessment (TPA) process.\\nUnder our TPA process, we gather information from certain third parties who contract with Meta and share or receive data, or have access to or integrate with\\nour systems, in order to help us assess potential risks associated with their security controls. We also generally require third parties to, among other things,\\nmaintain security controls to protect our confidential information and data, and notify us of material data breaches that may impact our data.\\nOur board of directors has oversight of our strategic and business risk management and has delegated cybersecurity risk management oversight to the', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 51, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '5565a5de93004129a382c31ab609c975', '_collection_name': 'Meta 10-k Fillings'})]"
+       "[Document(page_content='to having a skilled, inclusive and diverse workforce because we believe cognitive diversity fuels innovation. To aid in this effort, we have taken steps to reduce\\nbias from our hiring processes and performance management systems, as well as offering learning and development courses for our employees.\\nCorporate Information\\nWe were incorporated in Delaware in July 2004. We completed our initial public offering in May 2012 and our Class\\xa0A common stock is currently listed\\non the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '7e4798ede3ec432490b98e331a24202f', '_collection_name': 'Meta 10-k Fillings'}),\n",
+       " Document(page_content=\"(Exact name of registrant as specified in its charter)\\n__________________________\\nDelaware\\n20-1665019\\n(State or other jurisdiction of incorporation or organization)\\n(I.R.S. Employer Identification Number)\\n1 Meta Way, Menlo Park, California 94025\\n(Address of principal executive offices and Zip Code)\\n(650)\\xa0543-4800\\n(Registrant's telephone number, including area code)\\n__________________________\\nSecurities registered pursuant to Section 12(b) of the Act:\\nTitle of each class\\nTrading symbol(s)\\nName of each exchange on which registered\\nClass A Common Stock, $0.000006 par value\\nMETA\\nThe Nasdaq Stock Market LLC\\nSecurities registered pursuant to Section 12(g) of the Act: None\", metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 0, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '670e8705b7f94289a225e4f519deec01', '_collection_name': 'Meta 10-k Fillings'}),\n",
+       " Document(page_content='making to real-time optimizations to post-campaign analytics. We work directly with these advertisers, as well as through advertising agencies and resellers.\\nWe operate offices in approximately 90\\xa0cities around the globe, the majority of which have a sales presence. We also invest in and rely on self-service tools to\\nprovide direct customer support to our users and partners.\\nFor our RL products, our sales and operations efforts utilize third-party sales channels such as retailers, resellers, and our direct-to-consumer channel,\\nMeta.com. These efforts are focused on driving consumer and enterprise sales and adoption of our Meta Quest portfolio of products and Ray-Ban Meta smart\\nglasses.\\nMarketing\\nHistorically, our communities have generally grown organically with people inviting their friends to connect with them, supported by internal efforts to\\nstimulate awareness and interest. In addition, we have invested and will continue to invest in marketing our products and services to grow our brand and help', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 9, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '5d43b155f5a94dec91210a4c364e0c83', '_collection_name': 'Meta 10-k Fillings'}),\n",
+       " Document(page_content='decision-making and prioritization of cybersecurity countermeasures and risk mitigation strategies. Our risk mitigation strategies include a broad variety of\\ntechnical and operational measures, as well as annual cybersecurity and privacy training for all of our employees.\\nIn addition, we maintain specific policies and practices governing our third-party security risks, including our third-party assessment (TPA) process.\\nUnder our TPA process, we gather information from certain third parties who contract with Meta and share or receive data, or have access to or integrate with\\nour systems, in order to help us assess potential risks associated with their security controls. We also generally require third parties to, among other things,\\nmaintain security controls to protect our confidential information and data, and notify us of material data breaches that may impact our data.\\nOur board of directors has oversight of our strategic and business risk management and has delegated cybersecurity risk management oversight to the', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 51, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'fa2a1e39a42f4be080a6ab89f649fe84', '_collection_name': 'Meta 10-k Fillings'})]"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -336,7 +331,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -351,7 +346,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -360,7 +355,7 @@
        "220"
       ]
      },
-     "execution_count": 25,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -371,7 +366,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -379,26 +374,26 @@
     "    split_chunks_2,\n",
     "    embedding_model,\n",
     "    location=\":memory:\",\n",
-    "    collection_name=\"Meta 10-k Fillings\",\n",
+    "    collection_name=\"Meta 10-k Fillings 2\",\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
-    "qdrant_retriever_2 = qdrant_vectorstore.as_retriever()"
+    "qdrant_retriever_2 = qdrant_vectorstore_2.as_retriever()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [],
    "source": [
-    "retrieval_augmented_qa_chain = (\n",
+    "retrieval_augmented_qa_chain_2 = (\n",
     "\n",
     "    {\"context\": itemgetter(\"question\") | qdrant_retriever_2, \"question\": itemgetter(\"question\")}\n",
     "\n",
@@ -410,28 +405,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "\"The total value of 'Cash and cash equivalents' as of December 31, 2023, was $41,862.\""
+       "\"The total value of 'Cash and cash equivalents' as of December 31, 2023, was $41.862 billion.\""
       ]
      },
-     "execution_count": 29,
+     "execution_count": 24,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "response_1b = retrieval_augmented_qa_chain.invoke({\"question\" : \"What was the total value of 'Cash and cash equivalents' as of December 31, 2023?\"})\n",
+    "response_1b = retrieval_augmented_qa_chain_2.invoke({\"question\" : \"What was the total value of 'Cash and cash equivalents' as of December 31, 2023?\"})\n",
     "response_1b[\"response\"].content"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
@@ -440,16 +435,39 @@
        "\"Sorry, the context is unrelated to the query, I can't answer.\""
       ]
      },
-     "execution_count": 30,
+     "execution_count": 27,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "response_2b = retrieval_augmented_qa_chain.invoke({\"question\" : \"Who are Meta's 'Directors' (i.e., members of the Board of Directors)?\"})\n",
+    "response_2b = retrieval_augmented_qa_chain_2.invoke({\"question\" : \"Who are Meta's 'Directors' (i.e., members of the Board of Directors)?\"})\n",
     "response_2b[\"response\"].content"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='Table of Contents\\nCompensation, Benefits, Health, and Well-being\\nWe offer competitive compensation to attract and retain the best people, and we help care for our people so they can focus on our mission. Our\\nemployees\\' total compensation package includes market-competitive salary, bonuses or sales incentives, and equity. We generally offer full-time employees\\nequity at the time of hire and through annual equity grants because we want them to be owners of the company and committed to our long-term success. We\\nhave conducted pay equity analyses for many years, and continue to be committed to pay equity. For example, in July 2023, we announced that our analyses\\nconfirm that we continue to have pay equity across genders globally and by race in the United States for people in similar jobs, accounting for factors such as\\nlocation, role, and level.\\nThrough Life@ Meta, our holistic approach to benefits, we continue to provide our employees and their dependents with resources to help them thrive.\\nWe offer a wide range of benefits across areas such as health, family, finance, community, and time away, including family building benefits, family care\\nresources, retirement savings plans, access to legal services, Meta Resource Groups to build community at Meta, and health and well-being benefits.\\nOur health and well-being programs are designed to give employees a choice of flexible benefits to help them reach their personal well-being goals. Our\\nprograms are tailored to help boost employee physical and mental health, create financial peace of mind, provide support for families, and help employees\\nbuild a strong community. Programs are designed and funded to support needs like autism care, cancer care, transgender services, holistic well-being, including\\nmental health programs and retirement savings, which represent a few of the ways we support our employees and their dependents.\\nDiverse and Inclusive Workplace\\nWe work to build a diverse and inclusive workplace where we can leverage our collective cognitive diversity to build the best products and make the\\nbest decisions for the global community we serve.\\nIn our 2023 Responsible Business Practices Report, we published our global diversity and U.S. ethnic diversity workforce data. As of December 31,\\n2022, our global employee base was composed of 45.4% underrepresented people, with 47.9% underrepresented people in the U.S., and 43.1% of our leaders\\nin the U.S. being people of color. As published in our 2023 Responsible Business Practices Report, people with disabilities now represent 7.2% of our U.S.\\nworkforce, and based on voluntary self-identification, veterans represented 2.3% and members of the LGBTQ+ community make up 9.8% of our U.S.\\nworkforce.\\nWe want our products to work for people around the world and we need to grow and keep the best talent in order to do that. We also remain committed\\nto having a skilled, inclusive and diverse workforce because we believe cognitive diversity fuels innovation. To aid in this effort, we have taken steps to reduce\\nbias from our hiring processes and performance management systems, as well as offering learning and development courses for our employees.\\nCorporate Information\\nWe were incorporated in Delaware in July 2004. We completed our initial public offering in May 2012 and our Class\\xa0A common stock is currently listed\\non the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'cba646efdcb945198cbb728b20e01cee', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='Table of Contents\\nItem 1B. Unresolved Staff Comments\\nNone.\\nItem 1C. Cybersecurity\\nAt Meta, cybersecurity risk management is an important part of our overall risk management efforts. Our industry is prone to cybersecurity threats and\\nattacks, and we regularly experience cybersecurity incidents of varying degrees. We believe we are a particularly attractive target as a result of our prominence\\nand scale, the types and volume of personal data and content on our systems, and the evolving nature of our products and services. Our products and services\\nreach billions of users and involve the collection, storage, processing, and transmission of a large amount of data. In addition, our business and operations span\\nnumerous geographies around the world, involve thousands of employees, contractors, vendors, developers, partners, and other third parties, and rely on\\nsoftware and hardware that is highly technical and complex. We maintain an information security program that is comprised of policies and controls designed\\nto mitigate cybersecurity risk. However, at any given time, we face known and unknown cybersecurity risks and threats that are not fully mitigated, and we\\ndiscover vulnerabilities in our program. We continuously work to enhance our information security program and risk management efforts.\\nWe use a risk management framework based on applicable laws and regulations, and informed by industry standards and industry-recognized practices,\\nfor managing cybersecurity risks within our products and services, infrastructure, and corporate resources. To identify and assess risks from cybersecurity\\nthreats, we evaluate a variety of developments including threat intelligence, first- and third-party vulnerabilities, evolving regulatory requirements, and\\nobserved cybersecurity incidents, among others. We regularly conduct risk assessments to evaluate the maturity and effectiveness of our systems and processes\\nin addressing cybersecurity threats and to identify any areas for remediation and opportunities for enhancements. We also engage third-party security experts\\nand consultants to assist with assessment and enhancement of our cybersecurity risk management processes, as well as benchmarking against industry\\npractices. In addition, we maintain a privacy risk management program to assess privacy risks related to how we are collecting, using, sharing, and storing user\\ndata, which is subject to assessment by an independent, third-party privacy assessor. Our internal audit function provides independent assessment and assurance\\non the overall operations of our cybersecurity and privacy programs and the supporting control frameworks. These processes support informed risk-based\\ndecision-making and prioritization of cybersecurity countermeasures and risk mitigation strategies. Our risk mitigation strategies include a broad variety of\\ntechnical and operational measures, as well as annual cybersecurity and privacy training for all of our employees.\\nIn addition, we maintain specific policies and practices governing our third-party security risks, including our third-party assessment (TPA) process.\\nUnder our TPA process, we gather information from certain third parties who contract with Meta and share or receive data, or have access to or integrate with\\nour systems, in order to help us assess potential risks associated with their security controls. We also generally require third parties to, among other things,\\nmaintain security controls to protect our confidential information and data, and notify us of material data breaches that may impact our data.\\nOur board of directors has oversight of our strategic and business risk management and has delegated cybersecurity risk management oversight to the\\naudit & risk oversight committee of our board of directors (Audit & Risk Oversight Committee). Our Audit & Risk Oversight Committee is responsible for\\nensuring that management has processes in place designed to identify and evaluate cybersecurity risks to which the company is exposed and to implement\\nprocesses and programs to manage cybersecurity risks and mitigate cybersecurity incidents. The privacy committee of our board of directors (Privacy\\nCommittee) oversees risks related to privacy and data use, including overseeing compliance with our comprehensive privacy program. Management is\\nresponsible for identifying, assessing, and managing material cybersecurity risks on an ongoing basis, establishing processes to ensure that such potential', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 51, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '5e107ee11aeb4ffd9288f36552015e5c', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='stockholders may be called only by a majority of our board of directors, the chairman of our board of directors, our chief executive officer or our president.\\n•\\nAdvance Notice Requirements for Stockholder Proposals and Director Nominations. Our amended and restated bylaws provide advance notice\\nprocedures for stockholders seeking to bring business before our annual meeting of stockholders, or to nominate candidates for election as directors at any\\nmeeting of stockholders. Our amended and restated bylaws also specify certain requirements regarding the form and content of a stockholder’s notice.\\nThese provisions may preclude our stockholders from bringing matters before our annual meeting of stockholders or from making nominations for\\ndirectors at our meetings of stockholders.\\n•\\nIssuance of Undesignated Preferred Stock. Our board of directors has the authority, without further action by the stockholders, to issue up to\\n100,000,000 shares of undesignated preferred stock with rights and preferences, including voting rights, designated from time to time by the board of\\ndirectors. The existence of authorized but unissued shares of preferred stock enables our board of directors to render more difficult or to discourage an\\nattempt to obtain control of us by means of a merger, tender offer, proxy contest or otherwise.\\nChoice of Forum\\nOur amended and restated certificate of incorporation provides that the Court of Chancery of the State of Delaware is the exclusive forum for any\\nderivative action or proceeding brought on our behalf; any action asserting a breach of fiduciary duty; any action asserting a claim against us arising pursuant\\nto the Delaware General Corporation Law, our amended and restated certificate of incorporation or our amended and restated bylaws; or any action asserting a\\nclaim against us that is governed by the internal affairs doctrine.\\nListing\\nOur Class A common stock is listed on the Nasdaq Global Select Market under the symbol “META.”\\nTransfer Agent and Registrar\\nThe transfer agent and registrar for our common stock is Computershare Trust Company, N.A.\\n4', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 136, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '35060e1b9e9544ba8356c443a17b63a4', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='on the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law\\ntrademarks, service marks, or trade names appearing in this Annual Report on Form 10-K are the property of Meta Platforms, Inc. or its affiliates. Other\\ntrademarks, service marks, or trade names appearing in this Annual Report on Form 10‑K are the property of their respective owners.\\nAvailable Information\\nOur Annual Reports on Form 10-K, Quarterly Reports on Form 10-Q, Current Reports on Form 8-K, and amendments to reports filed pursuant to\\nSections\\xa013(a) and 15(d) of the Securities Exchange Act of 1934, as amended (Exchange Act), are filed with the U.S. Securities and Exchange Commission\\n(SEC). We are subject to the informational requirements of the Exchange Act and file or furnish reports, proxy statements, and other information with the SEC.\\nSuch reports and other information filed by us with the SEC are available free of charge on our website at investor.fb.com when such reports are\\n13', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'a2e94c923813450896d908dca5e04392', '_collection_name': 'Meta 10-k Fillings 2'})]"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response_2b[\"context\"]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -468,18 +486,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 35,
    "metadata": {},
    "outputs": [],
    "source": [
     "from langchain.retrievers import MultiQueryRetriever\n",
     "\n",
-    "multiquery_retriever = MultiQueryRetriever.from_llm(retriever=qdrant_retriever_2, llm=openai_chat_model)"
+    "multiquery_retriever = MultiQueryRetriever.from_llm(retriever=qdrant_retriever, llm=openai_chat_model)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 36,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -495,7 +513,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [
     {
@@ -504,52 +522,53 @@
        "\"The total value of 'Cash and cash equivalents' as of December 31, 2023, was $41,862.\""
       ]
      },
-     "execution_count": 38,
+     "execution_count": 37,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "response_1c = retrieval_augmented_qa_chain.invoke({\"question\" : \"What was the total value of 'Cash and cash equivalents' as of December 31, 2023?\"})\n",
+    "response_1c = retrieval_augmented_qa_chain_3.invoke({\"question\" : \"What was the total value of 'Cash and cash equivalents' as of December 31, 2023?\"})\n",
     "response_1c[\"response\"].content"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "\"Sorry, the context is unrelated to the query, I can't answer.\""
+       "'Signature\\nTitle\\nDate\\n/s/ Mark Zuckerberg\\nBoard Chair and Chief Executive Officer\\n(Principal Executive Officer)\\nFebruary 1, 2024\\nMark Zuckerberg\\n/s/ Susan Li\\nChief Financial Officer\\n(Principal Financial Officer)\\nFebruary 1, 2024\\nSusan Li\\n/S/ Aaron Anderson\\nChief Accounting Officer\\n(Principal Accounting Officer)\\nFebruary 1, 2024\\nAaron Anderson\\n/s/ Peggy Alford\\nDirector\\nFebruary 1, 2024\\nPeggy Alford\\n/s/ Marc L. Andreessen\\nDirector\\nFebruary 1, 2024\\nMarc L. Andreessen\\n/s/ Andrew W. Houston\\nDirector\\nFebruary 1, 2024\\nAndrew W. Houston\\n/s/ Nancy Killefer\\nDirector\\nFebruary 1, 2024\\nNancy Killefer\\n/s/ Robert M. Kimmitt\\nDirector\\nFebruary 1, 2024'"
       ]
      },
-     "execution_count": 39,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "response_2c = retrieval_augmented_qa_chain.invoke({\"question\" : \"Who are Meta's 'Directors' (i.e., members of the Board of Directors)?\"})\n",
+    "response_2c = retrieval_augmented_qa_chain_3.invoke({\"question\" : \"Who are Meta's 'Directors' (i.e., members of the Board of Directors)?\"})\n",
     "response_2c[\"response\"].content"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Document(page_content='to having a skilled, inclusive and diverse workforce because we believe cognitive diversity fuels innovation. To aid in this effort, we have taken steps to reduce\\nbias from our hiring processes and performance management systems, as well as offering learning and development courses for our employees.\\nCorporate Information\\nWe were incorporated in Delaware in July 2004. We completed our initial public offering in May 2012 and our Class\\xa0A common stock is currently listed\\non the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '18b4e9ef498c49b5867839caa18d2e9a', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content=\"(Exact name of registrant as specified in its charter)\\n__________________________\\nDelaware\\n20-1665019\\n(State or other jurisdiction of incorporation or organization)\\n(I.R.S. Employer Identification Number)\\n1 Meta Way, Menlo Park, California 94025\\n(Address of principal executive offices and Zip Code)\\n(650)\\xa0543-4800\\n(Registrant's telephone number, including area code)\\n__________________________\\nSecurities registered pursuant to Section 12(b) of the Act:\\nTitle of each class\\nTrading symbol(s)\\nName of each exchange on which registered\\nClass A Common Stock, $0.000006 par value\\nMETA\\nThe Nasdaq Stock Market LLC\\nSecurities registered pursuant to Section 12(g) of the Act: None\", metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 0, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '91fb05632758461883fe050d26afeae1', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content='making to real-time optimizations to post-campaign analytics. We work directly with these advertisers, as well as through advertising agencies and resellers.\\nWe operate offices in approximately 90\\xa0cities around the globe, the majority of which have a sales presence. We also invest in and rely on self-service tools to\\nprovide direct customer support to our users and partners.\\nFor our RL products, our sales and operations efforts utilize third-party sales channels such as retailers, resellers, and our direct-to-consumer channel,\\nMeta.com. These efforts are focused on driving consumer and enterprise sales and adoption of our Meta Quest portfolio of products and Ray-Ban Meta smart\\nglasses.\\nMarketing\\nHistorically, our communities have generally grown organically with people inviting their friends to connect with them, supported by internal efforts to\\nstimulate awareness and interest. In addition, we have invested and will continue to invest in marketing our products and services to grow our brand and help', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 9, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'a474117d06bd422387c0fa613b547ee5', '_collection_name': 'Meta 10-k Fillings'}),\n",
-       " Document(page_content='decision-making and prioritization of cybersecurity countermeasures and risk mitigation strategies. Our risk mitigation strategies include a broad variety of\\ntechnical and operational measures, as well as annual cybersecurity and privacy training for all of our employees.\\nIn addition, we maintain specific policies and practices governing our third-party security risks, including our third-party assessment (TPA) process.\\nUnder our TPA process, we gather information from certain third parties who contract with Meta and share or receive data, or have access to or integrate with\\nour systems, in order to help us assess potential risks associated with their security controls. We also generally require third parties to, among other things,\\nmaintain security controls to protect our confidential information and data, and notify us of material data breaches that may impact our data.\\nOur board of directors has oversight of our strategic and business risk management and has delegated cybersecurity risk management oversight to the', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 51, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '5565a5de93004129a382c31ab609c975', '_collection_name': 'Meta 10-k Fillings'})]"
+       "[Document(page_content='Table of Contents\\nCompensation, Benefits, Health, and Well-being\\nWe offer competitive compensation to attract and retain the best people, and we help care for our people so they can focus on our mission. Our\\nemployees\\' total compensation package includes market-competitive salary, bonuses or sales incentives, and equity. We generally offer full-time employees\\nequity at the time of hire and through annual equity grants because we want them to be owners of the company and committed to our long-term success. We\\nhave conducted pay equity analyses for many years, and continue to be committed to pay equity. For example, in July 2023, we announced that our analyses\\nconfirm that we continue to have pay equity across genders globally and by race in the United States for people in similar jobs, accounting for factors such as\\nlocation, role, and level.\\nThrough Life@ Meta, our holistic approach to benefits, we continue to provide our employees and their dependents with resources to help them thrive.\\nWe offer a wide range of benefits across areas such as health, family, finance, community, and time away, including family building benefits, family care\\nresources, retirement savings plans, access to legal services, Meta Resource Groups to build community at Meta, and health and well-being benefits.\\nOur health and well-being programs are designed to give employees a choice of flexible benefits to help them reach their personal well-being goals. Our\\nprograms are tailored to help boost employee physical and mental health, create financial peace of mind, provide support for families, and help employees\\nbuild a strong community. Programs are designed and funded to support needs like autism care, cancer care, transgender services, holistic well-being, including\\nmental health programs and retirement savings, which represent a few of the ways we support our employees and their dependents.\\nDiverse and Inclusive Workplace\\nWe work to build a diverse and inclusive workplace where we can leverage our collective cognitive diversity to build the best products and make the\\nbest decisions for the global community we serve.\\nIn our 2023 Responsible Business Practices Report, we published our global diversity and U.S. ethnic diversity workforce data. As of December 31,\\n2022, our global employee base was composed of 45.4% underrepresented people, with 47.9% underrepresented people in the U.S., and 43.1% of our leaders\\nin the U.S. being people of color. As published in our 2023 Responsible Business Practices Report, people with disabilities now represent 7.2% of our U.S.\\nworkforce, and based on voluntary self-identification, veterans represented 2.3% and members of the LGBTQ+ community make up 9.8% of our U.S.\\nworkforce.\\nWe want our products to work for people around the world and we need to grow and keep the best talent in order to do that. We also remain committed\\nto having a skilled, inclusive and diverse workforce because we believe cognitive diversity fuels innovation. To aid in this effort, we have taken steps to reduce\\nbias from our hiring processes and performance management systems, as well as offering learning and development courses for our employees.\\nCorporate Information\\nWe were incorporated in Delaware in July 2004. We completed our initial public offering in May 2012 and our Class\\xa0A common stock is currently listed\\non the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'cba646efdcb945198cbb728b20e01cee', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='Table of Contents\\nItem 1B. Unresolved Staff Comments\\nNone.\\nItem 1C. Cybersecurity\\nAt Meta, cybersecurity risk management is an important part of our overall risk management efforts. Our industry is prone to cybersecurity threats and\\nattacks, and we regularly experience cybersecurity incidents of varying degrees. We believe we are a particularly attractive target as a result of our prominence\\nand scale, the types and volume of personal data and content on our systems, and the evolving nature of our products and services. Our products and services\\nreach billions of users and involve the collection, storage, processing, and transmission of a large amount of data. In addition, our business and operations span\\nnumerous geographies around the world, involve thousands of employees, contractors, vendors, developers, partners, and other third parties, and rely on\\nsoftware and hardware that is highly technical and complex. We maintain an information security program that is comprised of policies and controls designed\\nto mitigate cybersecurity risk. However, at any given time, we face known and unknown cybersecurity risks and threats that are not fully mitigated, and we\\ndiscover vulnerabilities in our program. We continuously work to enhance our information security program and risk management efforts.\\nWe use a risk management framework based on applicable laws and regulations, and informed by industry standards and industry-recognized practices,\\nfor managing cybersecurity risks within our products and services, infrastructure, and corporate resources. To identify and assess risks from cybersecurity\\nthreats, we evaluate a variety of developments including threat intelligence, first- and third-party vulnerabilities, evolving regulatory requirements, and\\nobserved cybersecurity incidents, among others. We regularly conduct risk assessments to evaluate the maturity and effectiveness of our systems and processes\\nin addressing cybersecurity threats and to identify any areas for remediation and opportunities for enhancements. We also engage third-party security experts\\nand consultants to assist with assessment and enhancement of our cybersecurity risk management processes, as well as benchmarking against industry\\npractices. In addition, we maintain a privacy risk management program to assess privacy risks related to how we are collecting, using, sharing, and storing user\\ndata, which is subject to assessment by an independent, third-party privacy assessor. Our internal audit function provides independent assessment and assurance\\non the overall operations of our cybersecurity and privacy programs and the supporting control frameworks. These processes support informed risk-based\\ndecision-making and prioritization of cybersecurity countermeasures and risk mitigation strategies. Our risk mitigation strategies include a broad variety of\\ntechnical and operational measures, as well as annual cybersecurity and privacy training for all of our employees.\\nIn addition, we maintain specific policies and practices governing our third-party security risks, including our third-party assessment (TPA) process.\\nUnder our TPA process, we gather information from certain third parties who contract with Meta and share or receive data, or have access to or integrate with\\nour systems, in order to help us assess potential risks associated with their security controls. We also generally require third parties to, among other things,\\nmaintain security controls to protect our confidential information and data, and notify us of material data breaches that may impact our data.\\nOur board of directors has oversight of our strategic and business risk management and has delegated cybersecurity risk management oversight to the\\naudit & risk oversight committee of our board of directors (Audit & Risk Oversight Committee). Our Audit & Risk Oversight Committee is responsible for\\nensuring that management has processes in place designed to identify and evaluate cybersecurity risks to which the company is exposed and to implement\\nprocesses and programs to manage cybersecurity risks and mitigate cybersecurity incidents. The privacy committee of our board of directors (Privacy\\nCommittee) oversees risks related to privacy and data use, including overseeing compliance with our comprehensive privacy program. Management is\\nresponsible for identifying, assessing, and managing material cybersecurity risks on an ongoing basis, establishing processes to ensure that such potential', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 51, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '5e107ee11aeb4ffd9288f36552015e5c', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='stockholders may be called only by a majority of our board of directors, the chairman of our board of directors, our chief executive officer or our president.\\n•\\nAdvance Notice Requirements for Stockholder Proposals and Director Nominations. Our amended and restated bylaws provide advance notice\\nprocedures for stockholders seeking to bring business before our annual meeting of stockholders, or to nominate candidates for election as directors at any\\nmeeting of stockholders. Our amended and restated bylaws also specify certain requirements regarding the form and content of a stockholder’s notice.\\nThese provisions may preclude our stockholders from bringing matters before our annual meeting of stockholders or from making nominations for\\ndirectors at our meetings of stockholders.\\n•\\nIssuance of Undesignated Preferred Stock. Our board of directors has the authority, without further action by the stockholders, to issue up to\\n100,000,000 shares of undesignated preferred stock with rights and preferences, including voting rights, designated from time to time by the board of\\ndirectors. The existence of authorized but unissued shares of preferred stock enables our board of directors to render more difficult or to discourage an\\nattempt to obtain control of us by means of a merger, tender offer, proxy contest or otherwise.\\nChoice of Forum\\nOur amended and restated certificate of incorporation provides that the Court of Chancery of the State of Delaware is the exclusive forum for any\\nderivative action or proceeding brought on our behalf; any action asserting a breach of fiduciary duty; any action asserting a claim against us arising pursuant\\nto the Delaware General Corporation Law, our amended and restated certificate of incorporation or our amended and restated bylaws; or any action asserting a\\nclaim against us that is governed by the internal affairs doctrine.\\nListing\\nOur Class A common stock is listed on the Nasdaq Global Select Market under the symbol “META.”\\nTransfer Agent and Registrar\\nThe transfer agent and registrar for our common stock is Computershare Trust Company, N.A.\\n4', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 136, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '35060e1b9e9544ba8356c443a17b63a4', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='Table of Contents\\namong other things, required us to significantly enhance our practices and processes for privacy compliance and oversight. The FTC also continues to monitor\\nus and our compliance with the modified consent order and initiated an administrative proceeding against us, which we are challenging, that alleges deficient\\ncompliance and violations of the Children\\'s Online Privacy Protection Act (COPPA), the COPPA Rule, and Section 5 of the Federal Trade Commission Act and\\nseeks changes to our business. If we are unsuccessful in our challenge to the FTC\\'s action and the agency imposes its proposed order in its current form, we\\nwould be subject to significant limitations, including on our ability to launch new and modified products or use data of users under 18 years old. Orders issued\\nby, or inquiries or enforcement actions initiated by, government or regulatory authorities could cause us to incur substantial costs, expose us to civil and\\ncriminal liability (including liability for our personnel) or penalties (including substantial monetary remedies), interrupt or require us to change our business\\npractices in a manner materially adverse to our business (including changes to our products or user data practices), result in negative publicity and reputational\\nharm, divert resources and the time and attention of management from our business, or subject us to other structural or behavioral remedies that adversely\\naffect our business.\\nFor additional information about government regulation applicable to our business, see Part\\xa0I, Item\\xa01A, \"Risk Factors\" in this Annual Report on Form\\n10-K.\\nHuman Capital\\nAt Meta, everything we do is about helping people feel connected and closer, and we are proud of our unique company culture.\\nWe had a global workforce of 67,317 employees as of December\\xa031, 2023, and we have offices in approximately 90 cities around the world. We are\\ncommitted to fostering an enriching environment for our workforce and we are focused on supporting our people in doing the best work of their careers. We\\noffer competitive compensation and a wide range of benefits, including many learning and development resources, and we work to build a diverse and\\ninclusive workplace.\\nBeginning in November 2022, we took a number of steps to reduce our expense base. For example, our cost reduction efforts have included scaling back\\nbudgets, reducing company perks, shrinking our real estate footprint, and employee layoffs and restructurings. We make it a priority to treat outgoing\\nemployees with respect and provide a generous severance package. For U.S. employees, that includes severance of 16 weeks of base pay plus two additional\\nweeks for every year of service, payment for all remaining paid time off, restricted stock unit vesting through their last day on payroll, health insurance,\\ncoverage of the cost of healthcare for employees and their families for six months, career services that included three months of career support with an external\\nvendor and early access to unpublished job leads, immigration support that included dedicated immigration specialists to help guide employees based on their\\nneeds. We offer similar support for outgoing employees outside of the United States while taking into account local employment laws.\\nEmployee Learning and Development\\nWe value our investment in growing and keeping a highly skilled workforce. We aim to provide all of our employees with regular performance reviews\\ntwice a year as we believe it is an important part of how we support their growth and career development while also recognizing and rewarding their impact at\\nMeta. We also offer career development opportunities and work experience programs that extend beyond the physical and virtual classroom. To do this, we\\nutilize various learning modalities, such as live virtual and in-person learning experiences, on-demand e-learning, self-service resources, learning communities,\\nand coaching engagements.\\nThe Pulse of Our Workforce\\nEach year, we conduct company-wide employee surveys to help us understand how employees feel about working at Meta and what we can do to', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 12, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': '9bdc238ea0134d3b886ea7d78460a1c9', '_collection_name': 'Meta 10-k Fillings 2'}),\n",
+       " Document(page_content='on the Nasdaq Global Select Market under the symbol \"META.\" Our principal executive offices are located at 1 Meta Way, Menlo Park, California 94025, and\\nour telephone number is (650) 543-4800.\\nMeta, the Meta logo, Meta Quest, Meta Horizon, Facebook, FB, Instagram, Oculus, WhatsApp, Reels, and our other registered or common law\\ntrademarks, service marks, or trade names appearing in this Annual Report on Form 10-K are the property of Meta Platforms, Inc. or its affiliates. Other\\ntrademarks, service marks, or trade names appearing in this Annual Report on Form 10‑K are the property of their respective owners.\\nAvailable Information\\nOur Annual Reports on Form 10-K, Quarterly Reports on Form 10-Q, Current Reports on Form 8-K, and amendments to reports filed pursuant to\\nSections\\xa013(a) and 15(d) of the Securities Exchange Act of 1934, as amended (Exchange Act), are filed with the U.S. Securities and Exchange Commission\\n(SEC). We are subject to the informational requirements of the Exchange Act and file or furnish reports, proxy statements, and other information with the SEC.\\nSuch reports and other information filed by us with the SEC are available free of charge on our website at investor.fb.com when such reports are\\n13', metadata={'source': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'file_path': 'https://d18rn0p25nwr6d.cloudfront.net/CIK-0001326801/c7318154-f6ae-4866-89fa-f0c589f2ee3d.pdf', 'page': 13, 'total_pages': 147, 'format': 'PDF 1.4', 'title': '0001326801-24-000012', 'author': 'EDGAR® Online LLC, a subsidiary of OTC Markets Group', 'subject': 'Form 10-K filed on 2024-02-02 for the period ending 2023-12-31', 'keywords': '0001326801-24-000012; ; 10-K', 'creator': 'EDGAR Filing HTML Converter', 'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creationDate': \"D:20240202060356-05'00'\", 'modDate': \"D:20240202060413-05'00'\", 'trapped': '', 'encryption': 'Standard V2 R3 128-bit RC4', '_id': 'a2e94c923813450896d908dca5e04392', '_collection_name': 'Meta 10-k Fillings 2'})]"
       ]
      },
-     "execution_count": 40,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -557,6 +576,14 @@
    "source": [
     "response_2c[\"context\"]"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Third pipeline results analysis\n",
+    "- The third pipeline shows good results and can answer both questions. The retrieval strategy has been upgraded and is now working properly."
+   ]
   }
  ],
  "metadata": {