Api/test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "from PyPDF2 import PdfReader\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from src.db_local_storage.files_db import VECTOR_FILES_DIRECTORY\n",
    "\n",
    "\n",
    "with open(VECTOR_FILES_DIRECTORY, \"r\") as file:\n",
    "    loaded_data = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.09148407727479935, -0.3296155333518982, 0.19553515315055847, -0.13005883991718292, -0.023037904873490334, -0.27768027782440186, -0.15001057088375092, 0.061617735773324966, -0.21048811078071594, -0.1158759668469429, 0.00262309773825109, -0.18505680561065674, -0.36614593863487244, 0.01263430155813694, -0.23454351723194122, 0.026990391314029694, 0.10782455652952194, 0.20590373873710632, 0.01619311049580574, -0.29646965861320496, -0.49789899587631226, -0.3155633509159088, 0.22015143930912018, -0.17993934452533722, -0.2728782892227173, -0.24151268601417542, -0.2213350087404251, -0.1546640843153, 0.18654145300388336, -0.3817594647407532, 0.27131369709968567, -0.16452756524085999, 0.4666154384613037, -0.23997938632965088, -0.16251197457313538, 0.45274969935417175, -0.06473690271377563, 0.030141692608594894, 0.18030805885791779, 0.3573063910007477, 0.057827189564704895, -0.5050416588783264, 0.14762045443058014, -0.24115344882011414, 0.12714245915412903, -0.30607476830482483, 0.03628716245293617, -0.1717076599597931, -0.07239308208227158, -0.10611303895711899, -0.20394517481327057, -0.357207328081131, 0.07243036478757858, -0.27321791648864746, -0.005590340122580528, 0.18584828078746796, 0.14657430350780487, 0.00868619792163372, -0.23141932487487793, -0.15293444693088531, 0.4537033438682556, -0.050794072449207306, -0.1081295758485794, 0.24416445195674896, 0.32709094882011414, -0.1073112040758133, -0.002160100731998682, -0.14978720247745514, -0.08192038536071777, 0.04925680160522461, 0.08451664447784424, -0.10695666819810867, -0.3269720673561096, -0.011846603825688362, 0.1591711789369583, 0.03670593723654747, 0.27853694558143616, 0.057576391845941544, 0.627304196357727, -0.09007762372493744, 0.3147648274898529, 0.040534473955631256, -0.1886814385652542, 0.07377851009368896, -0.11277369409799576, -0.06923136860132217, 0.23174744844436646, 0.12746192514896393, 0.24344752728939056, -0.07483936846256256, 0.16249415278434753, -0.279635488986969, -0.16499225795269012, -0.28518205881118774, 0.2098291516304016, 0.27612876892089844, -0.28646302223205566, 0.1196957528591156, -0.3333701193332672, 0.04434604197740555, -0.2624748647212982, -0.21890603005886078, -0.08524201810359955, -0.0075501492246985435, -0.4198438227176666, -0.0592561811208725, -0.10367451608181, -0.1156446784734726, 0.1438761055469513, 0.0052966708317399025, -0.09201934933662415, 0.36266717314720154, -0.21685782074928284, 0.1420959085226059, 0.10085764527320862, 0.02429366298019886, -0.10447590798139572, -0.1402379870414734, 0.14102685451507568, 0.10799302160739899, -0.17230382561683655, -0.15043006837368011, 0.07193934917449951, 0.1495690643787384, -0.019986119121313095, -0.0347922220826149, -0.27049851417541504, 0.13336730003356934, -0.11616193503141403, 0.2649662494659424, 0.330400675535202, 0.04918519780039787, 0.12089464068412781, -0.09495944529771805, -0.28842809796333313, 0.04238158464431763, 0.05436383932828903, -0.08257465064525604, -0.031113239005208015, -0.11439044773578644, 0.06356312334537506, 0.16110284626483917, -0.4002474248409271, -0.014672113582491875, 0.330425888299942, -0.03440795838832855, 0.19075387716293335, -0.3702341616153717, -0.19534951448440552, 0.1684616357088089, -0.09864974766969681, 0.12697818875312805, 0.1819300651550293, 0.07670610398054123, 0.3232901692390442, 0.04667458310723305, 0.09191926568746567, 0.06742186844348907, 0.026456736028194427, -0.02663232572376728, -0.17225997149944305, 0.19716386497020721, 0.02224832773208618, -0.03200436756014824, -0.08198100328445435, -0.2442745566368103, 0.06113841384649277, 0.0807797834277153, 0.04088904708623886, 0.2907366454601288, 0.22261427342891693, 0.10915300250053406, 0.017830494791269302, -0.02144521474838257, 0.4247247576713562, -0.007936686277389526, 0.20204278826713562, 0.17006978392601013, 0.04741619527339935, 0.31061092019081116, 0.12313715368509293, -0.13362251222133636, 0.3784801661968231, 0.16054050624370575, 0.08082814514636993, 0.05647144094109535, -0.0050701238214969635, 0.07090428471565247, -0.41181671619415283, 0.08139584213495255, -0.2981043756008148, 0.011231742799282074, 0.02322234958410263, 0.03604107350111008, 0.01680818758904934, 0.07644037157297134, 0.2796001136302948, 0.014765125699341297, 0.025432690978050232, 0.2253454178571701, -0.21646393835544586, -0.21112558245658875, -0.12944644689559937, 0.08615317195653915, -0.19481784105300903, -0.02482978068292141, -0.2655474543571472, 0.18308250606060028, 0.1057799756526947, 0.46467578411102295, 0.13872161507606506, -0.16284668445587158, -0.10319432616233826, 0.28903090953826904, -0.049684423953294754, -0.11852547526359558, 0.15179845690727234, 0.4315755367279053, 0.16684380173683167, 0.21633920073509216, -0.23727011680603027, 0.23591041564941406, -0.236463725566864, -0.10205617547035217, -0.38227447867393494, -0.3208659291267395, -0.16321201622486115, -0.2047148197889328, 0.035120557993650436, 0.1217074990272522, -0.09423492103815079, 0.10800852626562119, -0.12016260623931885, -0.01766962558031082, -0.21207456290721893, 0.18253299593925476, 0.09813184291124344, -0.14369548857212067, -0.4996097981929779, 0.05356284976005554, -0.13140559196472168, -0.10310748219490051, -0.3297559916973114, 0.13165785372257233, 0.29243674874305725, 0.6235761046409607, -0.018105201423168182, 0.4351204037666321, -0.3157229423522949, 0.14044661819934845, 0.32561150193214417, -0.045231807976961136, -0.22902421653270721, -0.04736926779150963, 0.00638209143653512, 0.2657054662704468, -0.1663697361946106, 0.21721374988555908, 0.06596962362527847, 0.4710530638694763, 0.1495056301355362, -0.2131609320640564, -0.09281381219625473, 0.33491361141204834, 0.26699742674827576, -0.0664362758398056, -0.3143211007118225, -0.11612315475940704, 0.09384771436452866, -0.31450754404067993, -0.14105857908725739, -0.1692599505186081, 0.15440012514591217, 0.021970681846141815, -0.2492946833372116, -0.19831828773021698, 0.11038780957460403, -0.2564508020877838, -0.09789401292800903, 0.014864001423120499, 0.33063599467277527, -0.08587673306465149, 0.3573816120624542, 0.262990802526474, -0.23019948601722717, -0.06639422476291656, 0.13446615636348724, -0.3160199522972107, -0.2009458839893341, 0.0011719531612470746, 0.05401983484625816, 0.11232183128595352, -0.2915894389152527, 0.04823492094874382, -0.11993903666734695, 0.1691748946905136, 0.10592252761125565, 0.11075671762228012, -0.20100106298923492, -0.0010831565596163273, 0.1340765357017517, -0.18890692293643951, -0.04269295558333397, -0.008131876587867737, -0.16146209836006165, -0.09385553747415543, 0.14291250705718994, 0.11386755108833313, 0.10874597728252411, 0.30922049283981323, -0.1316504329442978, 0.09756156802177429, 0.10346060246229172, -0.09867636114358902, -0.022087613120675087, 0.00240744068287313, -0.18236050009727478, 0.14601346850395203, -0.03863060474395752, -0.05868404731154442, 0.09617110341787338, -0.031769558787345886, 0.2660558223724365, 0.327562540769577, -0.083442322909832, 0.20325987040996552, -0.5998589396476746, -0.312290757894516, -0.060986027121543884, -0.03271651640534401, -0.21090127527713776, -0.22981834411621094, 0.05871429294347763, 0.34688451886177063, 0.20736956596374512, 0.13843078911304474, -0.08390390127897263, 0.38931000232696533, -0.1773708462715149, -0.35399889945983887, 0.2373276948928833, 0.05349014699459076, 0.08039376884698868, -0.11689640581607819, -0.14542360603809357, -0.2895090878009796, -0.12660722434520721, 0.5266510248184204, -0.09322528541088104, 0.02377072535455227, -0.12724241614341736, 0.3590814471244812, -0.13123689591884613, -0.10675929486751556, 0.3575019836425781, 0.09733907878398895, -0.027689877897500992, -0.0075003644451498985, 0.08631216734647751, 0.07380516082048416, -0.03189918026328087, 0.5251277089118958, 0.007510041352361441, 0.022245241329073906, 0.0560171902179718, 0.17317497730255127, -0.06186212971806526, -0.22594772279262543, 0.2666807770729065, 0.016032816842198372, 0.12344001233577728, -0.12985070049762726, 0.19014285504817963, 0.17215973138809204, 0.20680564641952515, 0.08599716424942017, 0.057784561067819595, -0.21372999250888824, 0.14019109308719635, 0.287504106760025, -0.019741084426641464, -0.2658323645591736, -0.4153008759021759, -0.0035018271300941706]\n"
     ]
    }
   ],
   "source": [
    "for document in loaded_data.values():\n",
    "  print(document[\"data\"][0][\"embedding\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def organize_db(db):\n",
    "  text_data = [] \n",
    "  embeddings = []\n",
    "\n",
    "  for document in db.values():\n",
    "    for page in document[\"data\"]:\n",
    "      text_data.append(page[\"metadata\"][\"original_text\"])\n",
    "      embeddings.append(page[\"embedding\"])\n",
    "  return text_data, embeddings\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_data, embeddings = organize_db(loaded_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Dear Hiring Manager,\\nI am writing to express my interest in the Machine Learning / AI Scientist position at Grazper.\\nWith a strong background in human-centered artificial intelligence, hands-on experience in\\ndeveloping machine learning models, and a deep passion for applying AI to solve real-world\\nproblems, I am excited about the opportunity to contribute to Grazper’s innovative work in\\nhuman pose estimation and behavioral analysis.\\nMy role as Co-Founder of NeoCareU had provided me with extensive experience',\n",
       " ' in\\ndeveloping Python-based infrastructures for graph search functionality, utilizing frameworks\\nsuch as FastAPI and Uvicorn. I have led end-to-end development of features, improving user\\nexperiences and backend processes with technologies like Node.js and TypeScript. This\\nexperience has sharpened my software development skills and my ability to translate\\ncomplex machine learning concepts into practical applications, a key requirement for the role\\nat Grazper.\\nIn addition to my professional experience, I hol',\n",
       " 'd a Master’s in Human-Centered Artificial\\nIntelligence from Denmark’s Technical University. During my studies, I focused on computer\\nvision and deep learning, gaining familiarity with models such as AlphaPose with its\\nResNet-50 backbone for 2D pose estimation, and MotionBERT for 3D pose estimation.\\nThese skills will be invaluable in collaborating with the diverse and talented team at Grazper.\\nThank you for considering my application. I am eager to bring my skills in machine learning,\\ncomputer vision, and so',\n",
       " 'ftware development to Grazper and to help drive the continued\\ninnovation that the company is known for. I look forward to the possibility of discussing how\\nmy background, skills, and experiences align with the needs of your team.\\nSincerely,']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lexical_search(query: str, chunks: list) -> list:\n",
    "        return [chunk for chunk in chunks if query.lower() in chunk.lower()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "qa_pipeline = pipeline(\n",
    "            \"question-answering\", model=\"deepset/roberta-base-squad2\"\n",
    "        )\n",
    "\n",
    "query = \"What is the capital of Germany?\"\n",
    "context = \"The capital of Germany is Paris\"\n",
    "response = qa_pipeline(question=query, context=context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'message': 'Paris',\n",
       " 'context_used': 'The capital of Germany is Paris',\n",
       " 'chunks': 'The capital of Germany is Paris'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{\"message\": response['answer'], \"context_used\": context, \"chunks\": context}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "import numpy as np\n",
    "\n",
    "def semantic_search(query: str, chunks: List[str], embeddings: np.ndarray, model) -> List[str]:\n",
    "    query_embedding = model.encode([query])\n",
    "    similarities = np.dot(embeddings, query_embedding.T).flatten()\n",
    "    top_indices = np.argsort(-similarities)[:3]  # Get top 3 results\n",
    "    return [chunks[i] for i in top_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' in\\ndeveloping Python-based infrastructures for graph search functionality, utilizing frameworks\\nsuch as FastAPI and Uvicorn. I have led end-to-end development of features, improving user\\nexperiences and backend processes with technologies like Node.js and TypeScript. This\\nexperience has sharpened my software development skills and my ability to translate\\ncomplex machine learning concepts into practical applications, a key requirement for the role\\nat Grazper.\\nIn addition to my professional experience, I hol',\n",
       " 'ftware development to Grazper and to help drive the continued\\ninnovation that the company is known for. I look forward to the possibility of discussing how\\nmy background, skills, and experiences align with the needs of your team.\\nSincerely,']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexical_search(\"ware\", text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# file_path = \"/home/alexabades/DocuRAG/Api/src/db_local_storage/files/doc_test.pdf\"\n",
    "file_path = \"/home/alexabades/DocuRAG/Api/doc_test.pdf\"\n",
    "\n",
    "print(os.path.exists(file_path))\n",
    "print(os.access(file_path, os.R_OK))\n",
    "\n",
    "with open(file_path, \"rb\") as pdf_file:\n",
    "    reader = PdfReader(pdf_file)\n",
    "    text = \"\"\n",
    "    for page in reader.pages:\n",
    "        text += page.extract_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pdfplumber\n",
    "\n",
    "# with pdfplumber.open(file_path) as pdf:\n",
    "#     first_page = pdf.pages[0]\n",
    "#     print(first_page.extract_text())\n",
    "\n",
    "with pdfplumber.open(file_path) as pdf:\n",
    "    text = \"\"\n",
    "    for page in pdf.pages:\n",
    "        text += page.extract_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunk_text(text, chunk_size=512):\n",
    "    chunks = [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]\n",
    "    return chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks = chunk_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alexabades/DocuRAG/Api/venv/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import tqdm, trange\n",
      "/home/alexabades/DocuRAG/Api/venv/lib/python3.10/site-packages/torch/cuda/__init__.py:128: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 11070). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:108.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n",
      "/home/alexabades/DocuRAG/Api/venv/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dear Hiring Manager,\n",
      "I am writing to express my interest in the Machine Learning / AI Scientist position at Grazper.\n",
      "With a strong background in human-centered artificial intelligence, hands-on experience in\n",
      "developing machine learning models, and a deep passion for applying AI to solve real-world\n",
      "problems, I am excited about the opportunity to contribute to Grazper’s innovative work in\n",
      "human pose estimation and behavioral analysis.\n",
      "My role as Co-Founder of NeoCareU had provided me with extensive experience\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "\n",
    "model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\")\n",
    "\n",
    "embeddings = []\n",
    "for chunk in chunks:\n",
    "    embeddings.append(model.encode(chunk))\n",
    "    print(chunk)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/home/alexabades/DocuRAG/Api/src/db_local_storage/vectorized_db/vectorized_data.json\"\n",
    "\n",
    "import json\n",
    "\n",
    "with open(path, \"r\") as f:\n",
    "    loaded_data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_data[\"emedding_data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf_reader = PyPDF2.PdfReader(io.BytesIO(pdf_file.file.read()))\n",
    "text = \"\"\n",
    "for page_num in range(pdf_reader.numPages):\n",
    "    page = pdf_reader.getPage(page_num)\n",
    "    text += page.extract_text()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}