HaileyStorm
commited on
Update chess-mamba-vs-xformer/train_bygame.py
Browse files
chess-mamba-vs-xformer/train_bygame.py
CHANGED
@@ -148,7 +148,9 @@ ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=
|
|
148 |
# poor man's data loader
|
149 |
data_dir = os.path.join('data', dataset)
|
150 |
current_train_file_index = 0
|
151 |
-
train_files = glob.glob(os.path.join(data_dir, 'train*.parquet'))
|
|
|
|
|
152 |
train_datasets = []
|
153 |
for f in train_files:
|
154 |
dataset = pq.read_table(f).to_pandas()
|
|
|
148 |
# poor man's data loader
|
149 |
data_dir = os.path.join('data', dataset)
|
150 |
current_train_file_index = 0
|
151 |
+
train_files = glob.glob(os.path.join(data_dir, 'train*.parquet')) + \
|
152 |
+
glob.glob(os.path.join(data_dir, 'stable*.parquet')) + \
|
153 |
+
glob.glob(os.path.join(data_dir, 'anneal*.parquet'))
|
154 |
train_datasets = []
|
155 |
for f in train_files:
|
156 |
dataset = pq.read_table(f).to_pandas()
|