Spaces:
Running
Running
File size: 4,292 Bytes
eafbf97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
"""Uploads dataset to huggingface datasets."""
import os
import sys
import pandas as pd
import numpy as np
from huggingface_hub import HfApi
import shared.utils as su
from sound_of_water.data.csv_loader import (
load_csv_sound_of_water,
configure_paths_sound_of_water,
)
if __name__ == "__main__":
api = HfApi()
data_root = "/work/piyush/from_nfs2/datasets/SoundOfWater"
repo_id = "bpiyush/sound-of-water"
save_splits = False
if save_splits:
# Load CSV
paths = configure_paths_sound_of_water(data_root)
df = load_csv_sound_of_water(paths)
del df["video_clip_path"]
del df["audio_clip_path"]
del df["box_path"]
del df["mask_path"]
# Splits
train_ids = su.io.load_txt(os.path.join(data_root, "splits/train.txt"))
df_train = df[df.item_id.isin(train_ids)]
df_train["file_name"] = df_train["item_id"].apply(lambda x: f"videos/{x}.mp4")
df_train.to_csv(os.path.join(data_root, "splits/train.csv"), index=False)
print(" [:::] Train split saved.")
test_I_ids = su.io.load_txt(os.path.join(data_root, "splits/test_I.txt"))
df_test_I = df[df.item_id.isin(test_I_ids)]
df_test_I["file_name"] = df_test_I["item_id"].apply(lambda x: f"videos/{x}.mp4")
df_test_I.to_csv(os.path.join(data_root, "splits/test_I.csv"), index=False)
print(" [:::] Test I split saved.")
test_II_ids = su.io.load_txt(os.path.join(data_root, "splits/test_II.txt"))
df_test_II = df[df.item_id.isin(test_II_ids)]
df_test_II["file_name"] = df_test_II["item_id"].apply(lambda x: f"videos/{x}.mp4")
df_test_II.to_csv(os.path.join(data_root, "splits/test_II.csv"), index=False)
print(" [:::] Test II split saved.")
test_III_ids = su.io.load_txt(os.path.join(data_root, "splits/test_III.txt"))
df_test_III = df[df.item_id.isin(test_III_ids)]
df_test_III["file_name"] = df_test_III["item_id"].apply(lambda x: f"videos/{x}.mp4")
df_test_III.to_csv(os.path.join(data_root, "splits/test_III.csv"), index=False)
print(" [:::] Test III split saved.")
create_splits = False
if create_splits:
train_ids = su.io.load_txt(os.path.join(data_root, "splits/train.txt"))
train_ids = np.unique(train_ids)
test_I_ids = su.io.load_txt(os.path.join(data_root, "splits/test_I.txt"))
test_I_ids = np.unique(test_I_ids)
other_ids = np.array(
list(set(df.item_id.unique()) - set(train_ids) - set(test_I_ids))
)
sub_df = df[~df.item_id.isin(set(train_ids) | set(test_I_ids))]
X = sub_df[
(sub_df.visibility != "transparent") & (sub_df["shape"].isin(["cylindrical", "semiconical"]))
]
test_II_ids = list(X.item_id.unique())
assert set(test_II_ids).intersection(set(train_ids)) == set()
assert set(test_II_ids).intersection(set(test_I_ids)) == set()
su.io.save_txt(test_II_ids, os.path.join(data_root, "splits/test_II.txt"))
X = sub_df[
(sub_df.visibility.isin(["transparent", "opaque"])) & \
(sub_df["shape"].isin(["cylindrical", "semiconical", "bottleneck"]))
]
test_III_ids = list(X.item_id.unique())
assert set(test_III_ids).intersection(set(train_ids)) == set()
assert set(test_III_ids).intersection(set(test_I_ids)) == set()
assert set(test_III_ids).intersection(set(test_II_ids)) != set()
su.io.save_txt(test_III_ids, os.path.join(data_root, "splits/test_III.txt"))
upload_file = True
if upload_file:
file = "README.md"
print(f" [:::] Uploading file: {file}")
api.upload_file(
path_or_fileobj=os.path.join(data_root, file),
path_in_repo=file,
repo_id=repo_id,
repo_type="dataset",
)
upload_folder = False
if upload_folder:
# Upload splits folder
foldername = "annotations"
print(f" [:::] Uploading folder: {foldername}")
api.upload_folder(
folder_path=os.path.join(data_root, foldername),
path_in_repo=foldername, # Upload to a specific folder
repo_id=repo_id,
repo_type="dataset",
)
|