Update pinecone_embeddings.py

pinecone_embeddings.py

@@ -1,5 +1,5 @@
 from pinecone import Pinecone, ServerlessSpec
-from pinecone_text.sparse import BM25Encoder # For BM25 sparse vectors
+# from pinecone_text.sparse import BM25Encoder # For BM25 sparse vectors
 import numpy as np
 import re
 import hashlib
@@ -66,7 +66,7 @@ class PineconeVectorStore:
         self.index = pinecone.Index(index_name)
         
         # Initialize BM25 encoder for sparse vectors
-        self.bm25_encoder = BM25Encoder()
+        # self.bm25_encoder = BM25Encoder()
         
         # Fit BM25 encoder on a representative corpus of your data.
         # This is crucial for BM25's effectiveness.
@@ -95,7 +95,7 @@ class PineconeVectorStore:
         records_to_upsert = []
         for i, chunk_text in enumerate(document_chunks):
             doc_id = hashlib.md5(f"{pdf_filename}-{chunk_text['text']}".encode('utf-8')).hexdigest()
-            sparse_vector = self.bm25_encoder.encode_documents([chunk_text["text"]])
+            # sparse_vector = self.bm25_encoder.encode_documents([chunk_text["text"]])
 
             records_to_upsert.append({
                 "id": doc_id,
@@ -122,7 +122,7 @@ class PineconeVectorStore:
         Retrieves top-k chunks based on the query using hybrid search.
         """
         # Generate sparse vector for the query using BM25Encoder
-        sparse_query_vector = self.bm25_encoder.encode_queries([query_text])
+        # sparse_query_vector = self.bm25_encoder.encode_queries([query_text])
 
         model = SentenceTransformer('BAAI/bge-base-en-v1.5')
         embeddings = model.encode(f"query: {query_text}", batch_size=32, convert_to_numpy=True).tolist()