Spaces:
Runtime error
Runtime error
File size: 448 Bytes
0b7b08a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
import os
import glob
import random
OUT_DIR = "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/instruct/all"
if __name__ == "__main__":
tars = glob.glob(os.path.join("/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/instruct", "*", "*.tar"))
random.shuffle(tars)
os.makedirs(OUT_DIR, exist_ok=True)
for i, tar in enumerate(tars):
dst = os.path.join(OUT_DIR, f"{str(i).zfill(6)}.tar")
os.symlink(tar, dst)
|