Create src/main.py

src/main.py

@@ -0,0 +1,211 @@
+import os
+import logging
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def load_data(start_year=2000, end_year=2017):
+    csv_files = [f'atp_matches_{year}.csv' for year in range(start_year, end_year + 1)]
+    dataframes = []
+
+    for file in csv_files:
+        try:
+            data = pd.read_csv(file)
+            logging.info(f"Loaded {file}: {len(data)} rows")
+            dataframes.append(data)
+        except FileNotFoundError:
+            logging.warning(f"File {file} not found.")
+            continue
+
+    if not dataframes:
+        raise FileNotFoundError("No CSV files found. Ensure data files are present.")
+
+    combined_df = pd.concat(dataframes, ignore_index=True)
+    logging.info(f"Total rows after combining all dataframes: {len(combined_df)}")
+    return combined_df
+
+def preprocess_data(df):
+    logging.info(f"Before preprocessing: {len(df)} rows")
+    df = df.loc[:, ~df.columns.duplicated()]
+    df = df.drop_duplicates().reset_index(drop=True)
+
+    df['tourney_date'] = pd.to_datetime(df['tourney_date'], format='%Y%m%d', errors='coerce')
+    df['tourney_date_ordinal'] = df['tourney_date'].apply(lambda x: x.toordinal() if pd.notnull(x) else None)
+
+    df['winner_id'] = df['winner_id'].fillna(df['winner_id'].mode().iloc[0])
+    df['loser_id'] = df['loser_id'].fillna(df['loser_id'].mode().iloc[0])
+    df['winner_name'] = df['winner_name'].fillna('Unknown')
+    df['loser_name'] = df['loser_name'].fillna('Unknown')
+
+    logging.info(f"After preprocessing: {len(df)} rows")
+    logging.info(f"Date range: {df['tourney_date'].min()} to {df['tourney_date'].max()}")
+    logging.info(f"Years present in the data: {sorted(df['tourney_date'].dt.year.unique())}")
+    return df
+
+def engineer_features(df):
+    numeric_columns = ['winner_rank', 'loser_rank', 'winner_seed', 'loser_seed', 
+                       'winner_age', 'loser_age', 'w_svpt', 'l_svpt', 'w_ace', 
+                       'l_ace', 'w_df', 'l_df', 'w_bpSaved', 'l_bpSaved',
+                       'tourney_date_ordinal']
+
+    for col in numeric_columns:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors='coerce')
+            logging.info(f"Column {col}: {df[col].isnull().sum()} null values")
+        else:
+            logging.warning(f"Column '{col}' not found in dataframe.")
+
+    df[numeric_columns] = df[numeric_columns].fillna(df[numeric_columns].median())
+
+    df['age_diff'] = df['winner_age'] - df['loser_age']
+    df['service_diff'] = df['w_svpt'] - df['l_svpt']
+    df['ace_diff'] = df['w_ace'] - df['l_ace']
+    df['df_diff'] = df['w_df'] - df['l_df']
+    df['bp_saved_diff'] = df['w_bpSaved'] - df['l_bpSaved']
+
+    numeric_columns.extend(['age_diff', 'service_diff', 'ace_diff', 'df_diff', 'bp_saved_diff'])
+
+    logging.info(f"After feature engineering: {len(df)} rows")
+    return df, numeric_columns
+
+def create_vae_model(input_dim, latent_dim=2):
+    encoder = tf.keras.Sequential([
+        tf.keras.layers.Dense(16, activation='relu', input_shape=(input_dim,)),
+        tf.keras.layers.Dense(latent_dim)
+    ])
+
+    decoder = tf.keras.Sequential([
+        tf.keras.layers.Dense(16, activation='relu', input_shape=(latent_dim,)),
+        tf.keras.layers.Dense(input_dim, activation='sigmoid')
+    ])
+
+    class VAEModel(tf.keras.Model):
+        def __init__(self, encoder, decoder, **kwargs):
+            super(VAEModel, self).__init__(**kwargs)
+            self.encoder = encoder
+            self.decoder = decoder
+
+        def call(self, inputs):
+            encoded = self.encoder(inputs)
+            decoded = self.decoder(encoded)
+            return decoded
+
+    vae = VAEModel(encoder, decoder)
+    vae.compile(optimizer='adam', loss='mse')
+
+    return vae
+
+def detect_anomalies(df, threshold=None):
+    if threshold is None:
+        threshold = df['rank_diff'].abs().quantile(0.85)  # 85th percentile
+    logging.info(f"Using anomaly threshold: {threshold}")
+    logging.info(f"Years in the data before anomaly detection: {sorted(df['tourney_date'].dt.year.unique())}")
+
+    anomalies = []
+    for i, row in df.iterrows():
+        rank_diff = row['winner_rank'] - row['loser_rank']
+        if abs(rank_diff) > threshold:
+            anomalies.append(row)
+
+    anomalies_df = pd.DataFrame(anomalies)
+
+    if anomalies_df.empty:
+        logging.warning("No anomalies detected!")
+        return anomalies_df
+
+    yearly_counts = anomalies_df['tourney_date'].dt.year.value_counts().sort_index()
+    logging.info(f"Anomalies per year:\n{yearly_counts}")
+
+    logging.info(f"Years in the anomalies: {sorted(anomalies_df['tourney_date'].dt.year.unique())}")
+    logging.info(f"Total anomalies: {len(anomalies_df)}")
+
+    return anomalies_df
+
+def analyze_anomalies(anomalies):
+    anomalies['tourney_name'] = anomalies['tourney_name'].fillna('Unknown')
+
+    anomalies_per_year = anomalies.groupby(anomalies['tourney_date'].dt.year).size()
+    anomalies_per_player = pd.concat([anomalies['winner_name'], anomalies['loser_name']]).value_counts()
+    anomalies_per_tournament = anomalies['tourney_name'].value_counts()
+
+    grand_slams = anomalies[anomalies['tourney_name'].str.contains('Grand Slam', case=False, na=False)]['tourney_name'].value_counts()
+    masters_1000 = anomalies[anomalies['tourney_name'].str.contains('Masters 1000', case=False, na=False)]['tourney_name'].value_counts()
+
+    return anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000
+
+def save_results(anomalies, anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000):
+    anomalies.to_csv('anomalies.csv', index=False)
+    anomalies_per_year.to_csv('anomalies_per_year.csv')
+    anomalies_per_player.to_csv('anomalies_per_player.csv')
+    anomalies_per_tournament.to_csv('anomalies_per_tournament.csv')
+    grand_slams.to_csv('anomalies_per_grand_slam.csv')
+    masters_1000.to_csv('anomalies_per_masters_1000.csv')
+
+    pd.DataFrame(anomalies_per_player.index.tolist(), columns=['Player']).to_csv('most_anomalies_players.csv', index=False)
+    pd.DataFrame(anomalies_per_tournament.index.tolist(), columns=['Tournament']).to_csv('most_anomalies_tournaments.csv', index=False)
+    pd.DataFrame(grand_slams.index.tolist(), columns=['Grand Slam']).to_csv('most_anomalies_grand_slams.csv', index=False)
+    pd.DataFrame(masters_1000.index.tolist(), columns=['Masters 1000']).to_csv('most_anomalies_masters_1000.csv', index=False)
+
+def main():
+    logging.info("Starting script...")
+
+    df = load_data()
+    logging.info(f"Total rows after loading: {len(df)}")
+
+    df = preprocess_data(df)
+    df, numeric_columns = engineer_features(df)
+    logging.info(f"Total rows after preprocessing and feature engineering: {len(df)}")
+
+    # Calculate rank difference
+    df['rank_diff'] = df['winner_rank'] - df['loser_rank']
+
+    # Log rank difference statistics
+    logging.info(f"Rank difference stats:\n{df['rank_diff'].describe()}")
+    logging.info(f"Rank difference percentiles:")
+    for percentile in [50, 75, 90, 95, 99]:
+        logging.info(f"{percentile}th percentile: {df['rank_diff'].abs().quantile(percentile/100)}")
+
+    # Log some statistics about the 'winner_rank' and 'loser_rank' columns
+    logging.info(f"Winner rank stats:\n{df['winner_rank'].describe()}")
+    logging.info(f"Loser rank stats:\n{df['loser_rank'].describe()}")
+
+    X = df[numeric_columns].copy()
+    y = df['winner_rank'] - df['loser_rank']
+
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X)
+
+    X_train_scaled, X_test_scaled, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)
+
+    X_train_scaled = X_train_scaled.astype('float32')
+    X_test_scaled = X_test_scaled.astype('float32')
+
+    vae = create_vae_model(X_train_scaled.shape[1])
+
+    logging.info("Model compiled. Starting training...")
+    history = vae.fit(X_train_scaled, X_train_scaled, epochs=10, batch_size=32, 
+                      validation_data=(X_test_scaled, X_test_scaled), verbose=1)
+    logging.info("Model training complete.")
+
+    anomalies = detect_anomalies(df)
+    if not anomalies.empty:
+        logging.info(f"Number of anomalies: {len(anomalies)}")
+        logging.info(f"Anomalies date range: {anomalies['tourney_date'].min()} to {anomalies['tourney_date'].max()}")
+
+        anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000 = analyze_anomalies(anomalies)
+
+        logging.info("Saving results...")
+        save_results(anomalies, anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000)
+    else:
+        logging.warning("No anomalies detected. Skipping analysis and result saving.")
+
+    logging.info("Script execution completed.")
+
+if __name__ == "__main__":
+    main()