Update app.py

app.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from huggingface_hub import CommitScheduler
 from uuid import uuid4
 from datasets import load_dataset
+import shutil
 
 DATASET_DIR = Path("json_dataset")
 DATASET_DIR.mkdir(parents=True, exist_ok=True)
@@ -17,19 +18,52 @@ scheduler = CommitScheduler(
     path_in_repo="data"
 )
 
-def download_dataset_file(dataset_id, file_name, local_dir):
-    dataset = load_dataset(dataset_id, split="test")
-    local_file_path = os.path.join(local_dir, file_name)
-
-    df = pd.DataFrame(dataset)  
-    df.to_csv(local_file_path, index=False, encoding="utf-8")  
+def download_dataset_file(dataset_id, local_dir):
+    dataset = load_dataset(dataset_id)
+    cache_file_info = dataset.cache_files
+    print(f"Cache File Info: {cache_file_info}\n")
+    filename = cache_file_info['test'][0]['filename']
+    print(f"Filename: {filename}")
+    snapshot_id = filename.split('\\')[-2]
+    print(f"Snapshot ID: {snapshot_id}")
+    dataset_name = filename.split('\\')[6]
+    dataset_name = dataset_name.replace('___', '--')
+    print(f"Dataset Name: {dataset_name}")
+    base_path = os.path.join(
+        filename.split('\\')[0],'\\',
+        filename.split('\\')[1],
+        filename.split('\\')[2], 
+        filename.split('\\')[3],
+        'huggingface', 
+        'hub', 
+        'datasets--' + dataset_name
+    )
+    print("Base Path: ", base_path)
+    snapshot_path = os.path.join(base_path, "snapshots", snapshot_id)
+    print(f"snapshot_path: {snapshot_path}")
+
+    contents = os.listdir(snapshot_path)
+    print("Contents of snapshot path:")
+    print(contents)
+
+    for file_name in contents:
+        print("Checking file: ", file_name)
+        if file_name.endswith(".csv"):
+            print("Found CSV file: ", file_name)
+            source_file = os.path.join(snapshot_path, file_name)
+            local_file_path = os.path.join(local_dir, file_name)
+            shutil.move(source_file, local_file_path)
+          
     return local_file_path
 
 DATASET_ID = "350016z/flores_plus_Taiwanese"
 data_path = "test.csv"
 current_dir = os.getcwd()
-csv_files = [data_path] 
-data_path = download_dataset_file(DATASET_ID, data_path, current_dir)
+data_path = download_dataset_file(DATASET_ID, current_dir)
+print(f"Data path: {data_path}")
+
+
+csv_files = [f for f in os.listdir(current_dir) if f.endswith('.csv')]
 
 # data_path = "test.csv"
 # current_dir = os.path.dirname(os.path.abspath(data_path))