lalalalalalalalalala commited on
Commit
39ff54e
·
verified ·
1 Parent(s): 62c053c

Update run.py

Browse files
Files changed (1) hide show
  1. run.py +7 -1
run.py CHANGED
@@ -38,7 +38,13 @@ def fast_caption(sys_prompt, usr_prompt, temp, top_p, max_tokens, model, key, en
38
  progress_info.append('Begin processing Hugging Face dataset.')
39
  os.environ["TOKIO_WORKER_THREADS"] = "8"
40
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
41
- pqfile = hf_hub_download(repo_id=video_hf, filename='data/' + str(parquet_index).zfill(6) + '.parquet', repo_type="dataset", local_dir="/dev/shm", token=video_hf_auth)
 
 
 
 
 
 
42
 
43
  pf = pq.ParquetFile(pqfile)
44
  for batch in pf.iter_batches(1):
 
38
  progress_info.append('Begin processing Hugging Face dataset.')
39
  os.environ["TOKIO_WORKER_THREADS"] = "8"
40
  os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
41
+ pqfile = hf_hub_download(
42
+ repo_id=video_hf,
43
+ filename='data/' + str(parquet_index).zfill(6) + '.parquet',
44
+ repo_type="dataset",
45
+ local_dir="/dev/shm",
46
+ token=video_hf_auth,
47
+ )
48
 
49
  pf = pq.ParquetFile(pqfile)
50
  for batch in pf.iter_batches(1):