File size: 878 Bytes
c33448f ecdb99d c33448f fb18007 c33448f fb18007 c33448f fb18007 c33448f fb18007 c33448f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
import logging
import datasets
def check_dataset_and_get_config(dataset_id):
try:
configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
return configs
except Exception:
# Dataset may not exist
return None
def check_dataset_and_get_split(dataset_id, dataset_config):
try:
splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
except Exception as e:
# Dataset may not exist
logging.warning(
f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
)
return None
if isinstance(splits, list):
return splits
else:
# Dataset has no splits
logging.warning(
f"Dataset {dataset_id} with config {dataset_config} has no splits"
)
return None
|