Spaces:
Runtime error
Runtime error
File size: 772 Bytes
3f8d76d 9bbcc22 322ebac f6b4508 5d9f40a f6b4508 9bbcc22 62828bf 322ebac f6b4508 62828bf 322ebac 9988244 f6b4508 c015c4c 0ee5810 c015c4c 3f8d76d 5d9f40a 3f8d76d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
import yaml
from datasets import load_dataset
import pandas as pd
import os
import pprint
def make_dataset(dataset='cnn_dailymail', split='train'):
"""make dataset for summarisation"""
if not os.path.exists('data/raw'):
os.makedirs('data/raw')
dataset = load_dataset(dataset, '3.0.0', split=split)
df = pd.DataFrame()
df['article'] = dataset['article']
df['highlights'] = dataset['highlights']
df.to_csv('data/raw/{}.csv'.format(split))
if __name__ == '__main__':
with open("params.yml") as f:
params = yaml.safe_load(f)
pprint.pprint(params)
make_dataset(dataset=params['data'], split='train')
make_dataset(dataset=params['data'], split='test')
make_dataset(dataset=params['data'], split='validation')
|