Upload load_dataset.py
Browse files- data/load_dataset.py +2 -1
data/load_dataset.py
CHANGED
@@ -4,7 +4,8 @@ import pickle
|
|
4 |
from datasets import load_dataset
|
5 |
from config import ConfigConstants # For saving the dataset locally
|
6 |
|
7 |
-
def load_data(data_set_name
|
|
|
8 |
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
9 |
dataset_file = os.path.join(local_path, f"{data_set_name}_test.pkl")
|
10 |
|
|
|
4 |
from datasets import load_dataset
|
5 |
from config import ConfigConstants # For saving the dataset locally
|
6 |
|
7 |
+
def load_data(data_set_name):
|
8 |
+
local_path = ConfigConstants.DATA_SET_PATH + 'local_datasets'
|
9 |
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
10 |
dataset_file = os.path.join(local_path, f"{data_set_name}_test.pkl")
|
11 |
|