Spaces:
Runtime error
Runtime error
File size: 651 Bytes
3f8d76d 9bbcc22 322ebac 9bbcc22 62828bf 322ebac 62828bf 322ebac 9988244 3f8d76d c015c4c 0ee5810 c015c4c 3f8d76d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
import yaml
from datasets import load_dataset
import pandas as pd
def make_dataset(dataset='cnn_dailymail', split='train'):
"""make dataset for summarisation"""
dataset = load_dataset(dataset, '3.0.0', split=split)
df = pd.DataFrame()
df['article'] = dataset['article']
df['highlights'] = dataset['highlights']
df.to_csv('data/raw/{}.csv'.format(split))
if __name__ == '__main__':
with open("params.yml") as f:
params = yaml.safe_load(f)
make_dataset(dataset=params['data'], split='train')
make_dataset(dataset=params['data'], split='test')
make_dataset(dataset=params['data'], split='validation')
|