Optimize player similarity score calculation in predict_dupes function

This update enhances the calculate_player_similarity_score function by implementing vectorized operations for calculating Jaccard distances, significantly improving performance. The conversion of player selections to binary vectors allows for efficient pairwise distance computations, resulting in faster and more accurate similarity score calculations for lineups.

global_func/predict_dupes.py

@@ -41,7 +41,7 @@ def calculate_weighted_ownership(row_ownerships):
 def calculate_player_similarity_score(portfolio, player_columns):
     """
     Calculate a similarity score that measures how different each row is from all other rows
-    based on actual player selection. Converts players to numeric IDs for faster comparison.
+    based on actual player selection. Optimized for speed using vectorized operations.
     Higher scores indicate more unique/different lineups.
     
     Args:
@@ -65,46 +65,46 @@ def calculate_player_similarity_score(portfolio, player_columns):
     # Create player ID mapping
     player_to_id = {player: idx for idx, player in enumerate(sorted(all_players))}
     
-    # Convert each row to a list of player IDs
-    row_ids = []
-    for _, row in player_data.iterrows():
-        # Get player IDs for this row, sorted for consistency
-        player_ids = sorted([player_to_id[str(val)] for val in row.values 
-                           if isinstance(val, str) and str(val).strip() != '' and str(val) in player_to_id])
-        row_ids.append(player_ids)
+    # Convert each row to a binary vector (1 if player is present, 0 if not)
+    n_players = len(all_players)
+    n_rows = len(portfolio)
+    binary_matrix = np.zeros((n_rows, n_players), dtype=np.int8)
     
-    # Calculate similarity scores using Jaccard distance on player ID sets
-    similarity_scores = []
+    for i, (_, row) in enumerate(player_data.iterrows()):
+        for val in row.values:
+            if isinstance(val, str) and str(val).strip() != '' and str(val) in player_to_id:
+                binary_matrix[i, player_to_id[str(val)]] = 1
     
-    for i in range(len(portfolio)):
-        distances = []
-        for j in range(len(portfolio)):
-            if i != j:
-                # Convert to sets for Jaccard calculation
-                set_i = set(row_ids[i])
-                set_j = set(row_ids[j])
-                
-                # Calculate Jaccard distance
-                if len(set_i) == 0 and len(set_j) == 0:
-                    # Both lineups are empty
-                    distance = 0.0
-                elif len(set_i) == 0 or len(set_j) == 0:
-                    # One lineup is empty, other is not
-                    distance = 1.0
-                else:
-                    # Jaccard distance = 1 - (intersection / union)
-                    intersection = len(set_i & set_j)
-                    union = len(set_i | set_j)
-                    distance = 1 - (intersection / union)
-                
-                distances.append(distance)
-        
-        # Average distance to all other lineups
-        avg_distance = np.mean(distances) if distances else 0
-        similarity_scores.append(avg_distance)
+    # Vectorized Jaccard distance calculation
+    # Use matrix operations to compute all pairwise distances at once
+    similarity_scores = np.zeros(n_rows)
+    
+    # Compute intersection and union matrices
+    # intersection[i,j] = number of players in common between row i and row j
+    # union[i,j] = total number of unique players between row i and row j
+    intersection_matrix = np.dot(binary_matrix, binary_matrix.T)
+    
+    # For union, we need: |A ∪ B| = |A| + |B| - |A ∩ B|
+    row_sums = np.sum(binary_matrix, axis=1)
+    union_matrix = row_sums[:, np.newaxis] + row_sums - intersection_matrix
+    
+    # Calculate Jaccard distance: 1 - (intersection / union)
+    # Avoid division by zero
+    with np.errstate(divide='ignore', invalid='ignore'):
+        jaccard_similarity = np.divide(intersection_matrix, union_matrix, 
+                                     out=np.zeros_like(intersection_matrix, dtype=float), 
+                                     where=union_matrix != 0)
+    
+    # Convert similarity to distance and calculate average distance for each row
+    jaccard_distance = 1 - jaccard_similarity
+    
+    # For each row, calculate average distance to all other rows
+    # Exclude self-comparison (diagonal elements)
+    np.fill_diagonal(jaccard_distance, 0)
+    row_counts = n_rows - 1  # Exclude self
+    similarity_scores = np.sum(jaccard_distance, axis=1) / row_counts
     
     # Normalize to 0-1 scale where 1 = most unique/different
-    similarity_scores = np.array(similarity_scores)
     if similarity_scores.max() > similarity_scores.min():
         similarity_scores = (similarity_scores - similarity_scores.min()) / (similarity_scores.max() - similarity_scores.min())