src/main.py

import os
import logging
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

def load_data(start_year=2000, end_year=2017):
    csv_files = [f'atp_matches_{year}.csv' for year in range(start_year, end_year + 1)]
    dataframes = []

    for file in csv_files:
        try:
            data = pd.read_csv(file)
            logging.info(f"Loaded {file}: {len(data)} rows")
            dataframes.append(data)
        except FileNotFoundError:
            logging.warning(f"File {file} not found.")
            continue

    if not dataframes:
        raise FileNotFoundError("No CSV files found. Ensure data files are present.")

    combined_df = pd.concat(dataframes, ignore_index=True)
    logging.info(f"Total rows after combining all dataframes: {len(combined_df)}")
    return combined_df

def preprocess_data(df):
    logging.info(f"Before preprocessing: {len(df)} rows")
    df = df.loc[:, ~df.columns.duplicated()]
    df = df.drop_duplicates().reset_index(drop=True)

    df['tourney_date'] = pd.to_datetime(df['tourney_date'], format='%Y%m%d', errors='coerce')
    df['tourney_date_ordinal'] = df['tourney_date'].apply(lambda x: x.toordinal() if pd.notnull(x) else None)

    df['winner_id'] = df['winner_id'].fillna(df['winner_id'].mode().iloc[0])
    df['loser_id'] = df['loser_id'].fillna(df['loser_id'].mode().iloc[0])
    df['winner_name'] = df['winner_name'].fillna('Unknown')
    df['loser_name'] = df['loser_name'].fillna('Unknown')

    logging.info(f"After preprocessing: {len(df)} rows")
    logging.info(f"Date range: {df['tourney_date'].min()} to {df['tourney_date'].max()}")
    logging.info(f"Years present in the data: {sorted(df['tourney_date'].dt.year.unique())}")
    return df

def engineer_features(df):
    numeric_columns = ['winner_rank', 'loser_rank', 'winner_seed', 'loser_seed', 
                       'winner_age', 'loser_age', 'w_svpt', 'l_svpt', 'w_ace', 
                       'l_ace', 'w_df', 'l_df', 'w_bpSaved', 'l_bpSaved',
                       'tourney_date_ordinal']

    for col in numeric_columns:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors='coerce')
            logging.info(f"Column {col}: {df[col].isnull().sum()} null values")
        else:
            logging.warning(f"Column '{col}' not found in dataframe.")

    df[numeric_columns] = df[numeric_columns].fillna(df[numeric_columns].median())

    df['age_diff'] = df['winner_age'] - df['loser_age']
    df['service_diff'] = df['w_svpt'] - df['l_svpt']
    df['ace_diff'] = df['w_ace'] - df['l_ace']
    df['df_diff'] = df['w_df'] - df['l_df']
    df['bp_saved_diff'] = df['w_bpSaved'] - df['l_bpSaved']

    numeric_columns.extend(['age_diff', 'service_diff', 'ace_diff', 'df_diff', 'bp_saved_diff'])

    logging.info(f"After feature engineering: {len(df)} rows")
    return df, numeric_columns

def create_vae_model(input_dim, latent_dim=2):
    encoder = tf.keras.Sequential([
        tf.keras.layers.Dense(16, activation='relu', input_shape=(input_dim,)),
        tf.keras.layers.Dense(latent_dim)
    ])

    decoder = tf.keras.Sequential([
        tf.keras.layers.Dense(16, activation='relu', input_shape=(latent_dim,)),
        tf.keras.layers.Dense(input_dim, activation='sigmoid')
    ])

    class VAEModel(tf.keras.Model):
        def __init__(self, encoder, decoder, **kwargs):
            super(VAEModel, self).__init__(**kwargs)
            self.encoder = encoder
            self.decoder = decoder

        def call(self, inputs):
            encoded = self.encoder(inputs)
            decoded = self.decoder(encoded)
            return decoded

    vae = VAEModel(encoder, decoder)
    vae.compile(optimizer='adam', loss='mse')

    return vae

def detect_anomalies(df, threshold=None):
    if threshold is None:
        threshold = df['rank_diff'].abs().quantile(0.85)  # 85th percentile
    logging.info(f"Using anomaly threshold: {threshold}")
    logging.info(f"Years in the data before anomaly detection: {sorted(df['tourney_date'].dt.year.unique())}")

    anomalies = []
    for i, row in df.iterrows():
        rank_diff = row['winner_rank'] - row['loser_rank']
        if abs(rank_diff) > threshold:
            anomalies.append(row)

    anomalies_df = pd.DataFrame(anomalies)

    if anomalies_df.empty:
        logging.warning("No anomalies detected!")
        return anomalies_df

    yearly_counts = anomalies_df['tourney_date'].dt.year.value_counts().sort_index()
    logging.info(f"Anomalies per year:\n{yearly_counts}")

    logging.info(f"Years in the anomalies: {sorted(anomalies_df['tourney_date'].dt.year.unique())}")
    logging.info(f"Total anomalies: {len(anomalies_df)}")

    return anomalies_df

def analyze_anomalies(anomalies):
    anomalies['tourney_name'] = anomalies['tourney_name'].fillna('Unknown')

    anomalies_per_year = anomalies.groupby(anomalies['tourney_date'].dt.year).size()
    anomalies_per_player = pd.concat([anomalies['winner_name'], anomalies['loser_name']]).value_counts()
    anomalies_per_tournament = anomalies['tourney_name'].value_counts()

    grand_slams = anomalies[anomalies['tourney_name'].str.contains('Grand Slam', case=False, na=False)]['tourney_name'].value_counts()
    masters_1000 = anomalies[anomalies['tourney_name'].str.contains('Masters 1000', case=False, na=False)]['tourney_name'].value_counts()

    return anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000

def save_results(anomalies, anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000):
    anomalies.to_csv('anomalies.csv', index=False)
    anomalies_per_year.to_csv('anomalies_per_year.csv')
    anomalies_per_player.to_csv('anomalies_per_player.csv')
    anomalies_per_tournament.to_csv('anomalies_per_tournament.csv')
    grand_slams.to_csv('anomalies_per_grand_slam.csv')
    masters_1000.to_csv('anomalies_per_masters_1000.csv')

    pd.DataFrame(anomalies_per_player.index.tolist(), columns=['Player']).to_csv('most_anomalies_players.csv', index=False)
    pd.DataFrame(anomalies_per_tournament.index.tolist(), columns=['Tournament']).to_csv('most_anomalies_tournaments.csv', index=False)
    pd.DataFrame(grand_slams.index.tolist(), columns=['Grand Slam']).to_csv('most_anomalies_grand_slams.csv', index=False)
    pd.DataFrame(masters_1000.index.tolist(), columns=['Masters 1000']).to_csv('most_anomalies_masters_1000.csv', index=False)

def main():
    logging.info("Starting script...")

    df = load_data()
    logging.info(f"Total rows after loading: {len(df)}")

    df = preprocess_data(df)
    df, numeric_columns = engineer_features(df)
    logging.info(f"Total rows after preprocessing and feature engineering: {len(df)}")

    # Calculate rank difference
    df['rank_diff'] = df['winner_rank'] - df['loser_rank']

    # Log rank difference statistics
    logging.info(f"Rank difference stats:\n{df['rank_diff'].describe()}")
    logging.info(f"Rank difference percentiles:")
    for percentile in [50, 75, 90, 95, 99]:
        logging.info(f"{percentile}th percentile: {df['rank_diff'].abs().quantile(percentile/100)}")

    # Log some statistics about the 'winner_rank' and 'loser_rank' columns
    logging.info(f"Winner rank stats:\n{df['winner_rank'].describe()}")
    logging.info(f"Loser rank stats:\n{df['loser_rank'].describe()}")

    X = df[numeric_columns].copy()
    y = df['winner_rank'] - df['loser_rank']

    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)

    X_train_scaled, X_test_scaled, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)

    X_train_scaled = X_train_scaled.astype('float32')
    X_test_scaled = X_test_scaled.astype('float32')

    vae = create_vae_model(X_train_scaled.shape[1])

    logging.info("Model compiled. Starting training...")
    history = vae.fit(X_train_scaled, X_train_scaled, epochs=10, batch_size=32, 
                      validation_data=(X_test_scaled, X_test_scaled), verbose=1)
    logging.info("Model training complete.")

    anomalies = detect_anomalies(df)
    if not anomalies.empty:
        logging.info(f"Number of anomalies: {len(anomalies)}")
        logging.info(f"Anomalies date range: {anomalies['tourney_date'].min()} to {anomalies['tourney_date'].max()}")

        anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000 = analyze_anomalies(anomalies)

        logging.info("Saving results...")
        save_results(anomalies, anomalies_per_year, anomalies_per_player, anomalies_per_tournament, grand_slams, masters_1000)
    else:
        logging.warning("No anomalies detected. Skipping analysis and result saving.")

    logging.info("Script execution completed.")

if __name__ == "__main__":
    main()