import pypistats
from huggingface_hub import list_datasets
from datasets import load_dataset, concatenate_datasets, Dataset

def update_pip_installs():
    metrics_datasets = [d.id for d in list_datasets(author='open-source-metrics', use_auth_token=True)]
    pip_installs_datasets = [d for d in metrics_datasets if 'pip' in d]

    for dataset_name in pip_installs_datasets:
        dataset = load_dataset(dataset_name, use_auth_token=True)
        library_name = dataset.split('/')[1].split('-')[0]
        data = pypistats.overall(library_name, total=True, format="pandas")
        data = data.groupby("category").get_group("without_mirrors").sort_values("date")

        data.drop('category', axis=1, inplace=True)
        data.drop('percent', axis=1, inplace=True)

        pypistats_dataset = Dataset.from_pandas(data)
        pypistats_dataset.rename_column('downloads', 'num_downloads')
        pypistats_dataset.rename_column('date', 'day')

        column_names = pypistats_dataset.column_names
        column_names.remove('num_downloads')
        column_names.remove('day')

        pypistats_dataset.remove_columns(column_names)

        days = pypistats_dataset['day']





update_pip_installs()