Spaces:
Runtime error
Runtime error
Update run.py
Browse files
run.py
CHANGED
@@ -38,7 +38,13 @@ def fast_caption(sys_prompt, usr_prompt, temp, top_p, max_tokens, model, key, en
|
|
38 |
progress_info.append('Begin processing Hugging Face dataset.')
|
39 |
os.environ["TOKIO_WORKER_THREADS"] = "8"
|
40 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
41 |
-
pqfile = hf_hub_download(
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
|
43 |
pf = pq.ParquetFile(pqfile)
|
44 |
for batch in pf.iter_batches(1):
|
|
|
38 |
progress_info.append('Begin processing Hugging Face dataset.')
|
39 |
os.environ["TOKIO_WORKER_THREADS"] = "8"
|
40 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
41 |
+
pqfile = hf_hub_download(
|
42 |
+
repo_id=video_hf,
|
43 |
+
filename='data/' + str(parquet_index).zfill(6) + '.parquet',
|
44 |
+
repo_type="dataset",
|
45 |
+
local_dir="/dev/shm",
|
46 |
+
token=video_hf_auth,
|
47 |
+
)
|
48 |
|
49 |
pf = pq.ParquetFile(pqfile)
|
50 |
for batch in pf.iter_batches(1):
|