Update app.py

app.py

@@ -26,8 +26,7 @@ snapshot_download(repo_id=repo_id, repo_type=repo_type, local_dir=local_dir)
 
 # Function to convert dense vector to binary vector
 def dense_to_binary(dense_vector):
-    return np.packbits(np.where(dense_vector >= 0, 1, 0))
-
+    return np.packbits(np.where(dense_vector >= 0, 1, 0)).tobytes()
 
 # Gather fp32 files
 floats = glob('data/*.parquet')
@@ -47,11 +46,6 @@ for file in floats:
     
     df.to_parquet(f'binary_embeddings/{os.path.basename(file)}')
 
-    # Print the size of the original and binary files
-    original_size = os.path.getsize(file)
-    binary_size = os.path.getsize(f'binary_embeddings/{os.path.basename(file)}')
-    print(f'Original size: {original_size} bytes, Binary size: {binary_size} bytes')
-
 #######################################################################################
 
 
@@ -81,32 +75,27 @@ print("Upload complete")
 #######################################################################################
 
 # Function to convert dense vector to binary vector
-def binary_to_mrl(binary_vector, size=512):
-    return np.packbits(np.unpackbits(binary_vector)[:size])
+def dense_to_bmrl(binary_vector, size=512):
+    return np.packbits(np.where(dense_vector >= 0, 1, 0)[:size]).tobytes()
 
 # Gather fp32 files
-binaries = glob('binary_embeddings/*.parquet')
+floats = glob('data/*.parquet')
 
 # Create a folder to store binary embeddings
 os.makedirs('bmrl_embeddings', exist_ok=True)
 
 # Convert and save each file
-for file in binaries:
+for file in floats:
     
 
     print(f"Processing file: {file}")
 
     df = pd.read_parquet(file)
 
-    df['vector'] = df['vector'].progress_apply(binary_to_mrl)
+    df['vector'] = df['vector'].progress_apply(dense_to_bmrl)
     
     df.to_parquet(f'bmrl_embeddings/{os.path.basename(file)}')
 
-    # Print the size of the original and binary files
-    original_size = os.path.getsize(file)
-    binary_size = os.path.getsize(f'bmrl_embeddings/{os.path.basename(file)}')
-    print(f'Original size: {original_size} bytes, Binary size: {binary_size} bytes')
-
 #######################################################################################