delete documents

App/Embedding/utils/Initialize.py

@@ -1,24 +1,73 @@
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.docstore.document import Document
-# from langchain.vectorstores import Pinecone
-import os
-# import pinecone
+from langchain.vectorstores import MongoDBAtlasVectorSearch
+from pymongo import MongoClient
+from motor.motor_asyncio import AsyncIOMotorClient
+import os,pprint
 
-index_name = "movie-recommender-fast"
-model_name = "thenlper/gte-base"
-embeddings = HuggingFaceEmbeddings(model_name=model_name)
 
+completion_base = os.environ.get("completion_base")
+openai_api_key = os.environ.get("openai_api_key")
+mongoDB = os.environ.get("MONGO_DB")
+template = """### Given the following context 
+### Context
+{context}
+### Use it to explain the question: {question}
+ """
 
-# get api key from app.pinecone.io
-# PINECONE_API_KEY = (
-#     os.environ.get("PINECONE_API_KEY") or "0712a5e4-bcf3-4152-a726-27ee3a2676bb"
-# )
-# # find your environment next to the api key in pinecone console
-# PINECONE_ENV = os.environ.get("PINECONE_ENVIRONMENT") or "us-west4-gcp-free"
 
-# pinecone.init(api_key=PINECONE_API_KEY, environment=PINECONE_ENV)
+async def fetch_data(question, context):
+    url = completion_base
+
+    payload = json.dumps(
+        {
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "### You provide explanations based on the provided context",
+                },
+                {
+                    "role": "user",
+                    "content": template.format(context=context, question=question),
+                },
+            ],
+            "model": "gpt-3.5-turbo",
+            "temperature": 1,
+            "presence_penalty": 0,
+            "top_p": 0.95,
+            "frequency_penalty": 0,
+            "stream": False,
+        }
+    )
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {openai_api_key}",
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, data=payload) as response:
+            response = await response.json()
+            return response["choices"][0]["message"]["content"]
+
 
-# docsearch = Pinecone.from_existing_index(index_name, embeddings)
+async def delete_documents(task_id):
+    client = AsyncIOMotorClient(mongoDB)
+    db = client["transcriptions"]
+    collection = db["videos"]
+
+    result = await collection.delete_many({"task_id": task_id})
+    print(f"Deleted {result.deleted_count} document(s)")
+
+
+# mongo_client = MongoClient(
+#    mongoDB
+# )
+# model_name = "BAAI/bge-base-en"
+# collection = mongo_client["transcriptions"]["videos"]
+# embeddings = HuggingFaceEmbeddings(model_name=model_name)
+# index_name = "test_embeddings"
+# vectorstore = MongoDBAtlasVectorSearch(collection, embeddings, index_name=index_name)
 
 
 def generateChunks(chunks, task_id, n=100):
@@ -53,6 +102,7 @@ def search(query: str, task_id: str):
         embedding=embeddings,
         index_name="test_embedding",
     )
+
     data = vectorstore.similarity_search(
         query=query,
         pre_filter={"text": {"path": "task_id", "query": task_id}},
@@ -65,10 +115,7 @@ def search(query: str, task_id: str):
     # data =[d.dict() for d in data]
     # print(data[0].metadata.exclude({'_id','embedding'}))
     # pprint.pprint(data[0].metadata)
-    return [
-        {"text": d.page_content, "start": d.metadata["start"], "end": d.metadata["end"]}
-        for d in data
-    ]
+    return [{"text": d.page_content,'start':d.metadata['start'],"end":d.metadata['end']} for d in data]
     # agent =vectorstore.as_retriever(
 
     # )
@@ -87,4 +134,4 @@ def encode(temp: list[Document]):
     vectorstore.from_documents(
         temp, embedding=embeddings, collection=collection, index_name=index_name
     )
-    # return  embeddings.embed_documents(texts = [d.page_content for d in temp])
+    # return  embeddings.embed_documents(texts = [d.page_content for d in temp])

App/Transcription/TranscriptionRoutes.py

@@ -20,7 +20,7 @@ from .Model import Transcriptions
 from .Utils.fastapi_tasks import perform_background_task
 import yt_dlp
 from fastapi_jwt_auth import AuthJWT
-from App.Embedding.utils.Initialize import delete_documents
+# from App.Embedding.utils.Initialize import delete_documents
 
 # from .Model import User
 # from sqlalchemy import and_
@@ -28,11 +28,13 @@ from App.Embedding.utils.Initialize import delete_documents
 
 transcription_router = APIRouter(tags=["Transcription"])
 
+
 def genUUID():
     uuid_value = uuid.uuid4()
     short_uuid = str(uuid_value)[:6]
     return short_uuid
 
+
 @transcription_router.get("/download-audio")
 async def download_audio(
     url: str,
@@ -43,7 +45,6 @@ async def download_audio(
     ),
     user: UserSchema = Depends(get_token_owner),
 ):
-
     youtube_url = url
     parsed_url = urlparse(youtube_url)
 
@@ -78,7 +79,7 @@ async def download_audio(
     }
 
     task = downloadfile.delay(url=url, ydl_opts=ydl_opts, model_size=model)
-    response = {"task_id": task.id, "file_name": video_title }
+    response = {"task_id": task.id, "file_name": video_title}
     transcription_enrty = await Transcriptions.objects.create(
         user=user, youtubeLink=url, **response
     )
@@ -111,7 +112,7 @@ async def delete_transcription(
         await transcript.delete()
         task = AsyncResult(task_id)
         task.revoke(terminate=True)
-        await delete_documents(task_id=task_id)
+        # await delete_documents(task_id=task_id)
         return {"code": 200, "message": f"deleted {task_id}", "payload": None}
     else:
         return {
@@ -132,8 +133,8 @@ async def create_file(
     ),
     user: UserSchema = Depends(get_token_owner),
 ):
-    extension = file.filename.split('.')[-1]
-    file_name = f'{genUUID()}.{extension}'
+    extension = file.filename.split(".")[-1]
+    file_name = f"{genUUID()}.{extension}"
     # Write the file to disk asynchronously
     Upload_dir = ""
     try:
@@ -155,7 +156,9 @@ async def create_file(
     transcription_enrty = await Transcriptions.objects.create(
         task_id=task.id, user=user, file_name=file_name
     )
-    background_tasks.add_task(perform_background_task,file_name ,file=file, task_id=task.id)
+    background_tasks.add_task(
+        perform_background_task, file_name, file=file, task_id=task.id
+    )
     return {
         "file_size": file.size,
         "file_name": file.filename,