Update app.py

app.py

@@ -40,7 +40,7 @@ else:
     gpu_index = faiss.IndexFlatL2(dimension)  # fall back to CPU
 
 # Ensure embeddings are stacked as float32
-embeddings = np.vstack(data['embedding'].values).astype(np.float32)
+embeddings = np.vstack(data['embedding'].values).astype(np.float16)
 logging.debug(f"Embeddings shape: {embeddings.shape}, dtype: {embeddings.dtype}")
 gpu_index.add(embeddings)
 
@@ -61,7 +61,7 @@ def embed_question(question, model, tokenizer):
         logging.debug(f"Tokenized inputs: {inputs}")
         with torch.no_grad():
             outputs = model(**inputs)
-        embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy().astype(np.float32)
+        embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy().astype(np.float16)
         logging.debug(f"Question embedding shape: {embedding.shape}")
         logging.debug(f"Question embedding content: {embedding}")
         return embedding
@@ -78,7 +78,7 @@ def retrieve_and_generate(question):
         question_embedding = embed_question(question, model, tokenizer)
 
         # Ensure the embedding is in the correct format for FAISS search
-        question_embedding = question_embedding.astype(np.float32)
+        question_embedding = question_embedding.astype(np.float16)
 
         # Search in FAISS index
         try: