dataset_loader.py

from huggingface_hub import Repository
from huggingface_hub import login
import os

def load_dataset():
    login(token = os.environ['HUB_TOKEN'])

    repo = Repository(
        local_dir="agent_function",
        repo_type="dataset",
        clone_from="gheng/lgms",
        token=True,
        git_email='[email protected]'
    )
    repo.git_pull()

load_dataset()