Create data_loader.py

data_loader.py

@@ -0,0 +1,18 @@
+import dask.dataframe as dd
+import polars as pl
+
+# Function to read parquet files using Dask
+def read_dask_data():
+    splits = {'train': 'data/train-*.parquet', 'test': 'data/test-00000-of-00001.parquet'}
+    dask_df = dd.read_parquet("hf://datasets/MatrixStudio/Codeforces-Python-Submissions/" + splits["train"])
+    return dask_df
+
+# Function to read parquet files using Polars
+def read_polars_data():
+    polars_df = pl.read_parquet('hf://datasets/sdiazlor/python-reasoning-dataset/data/train-00000-of-00001.parquet')
+    return polars_df
+
+# Function to read parquet files using Dask for another dataset
+def read_another_dask_data():
+    dask_df = dd.read_parquet("hf://datasets/angie-chen55/python-github-code/data/train-*-of-*.parquet")
+    return dask_df