Update app.py

app.py

@@ -11,7 +11,7 @@
 
 import marimo
 
-__generated_with = "0.13.7"
+__generated_with = "0.13.8"
 app = marimo.App(width="full")
 
 
@@ -23,13 +23,24 @@ def _():
     from pathlib import Path
     from pyiceberg.partitioning import PartitionSpec, PartitionField
     from pyiceberg.transforms import IdentityTransform
-    return IdentityTransform, PartitionField, PartitionSpec, mo, pl
+    from zipfile import ZipFile
+    return (
+        IdentityTransform,
+        PartitionField,
+        PartitionSpec,
+        Path,
+        ZipFile,
+        mo,
+        pl,
+    )
 
 
 @app.cell
-def _():
+def _(Path):
     from pyiceberg.catalog import load_catalog
 
+    Path("warehouse").mkdir(exist_ok=True, parents=True)
+
     warehouse_path = "warehouse"
     catalog = load_catalog(
         "default",
@@ -43,9 +54,10 @@ def _():
 
 
 @app.cell
-def _(pl):
-    df_taxi = pl.read_csv("yellow_tripdata_2015-01.csv.zip").to_arrow()
-    return (df_taxi,)
+def _(ZipFile, pl):
+    df_orig = pl.read_csv(ZipFile("yellow_tripdata_2015-01.csv.zip").open("yellow_tripdata_2015-01.csv").read())
+    df_taxi = df_orig.to_arrow()
+    return df_orig, df_taxi
 
 
 @app.cell
@@ -100,40 +112,63 @@ def _(catalog):
     return
 
 
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Let's write the original zipped file into a csv file. We can read this and perform the same query to compare speeds.""")
+    return
+
+
+@app.cell
+def _(df_orig):
+    df_orig.write_csv("taxi.csv")
+    return
+
+
 @app.cell
 def _(pl):
-    df_orig = pl.read_csv("yellow_tripdata_2015-01.csv.zip")
-    df_orig.group_by("passenger_count").len().sort("passenger_count").collect()
+    pl.scan_csv("taxi.csv").group_by("passenger_count").len().sort("passenger_count").collect()
     return
 
 
 @app.cell
 def _(pl):
-    df_orig.group_by("passenger_count").len().sort("passenger_count")
+    pl.read_csv("taxi.csv").group_by("passenger_count").len().sort("passenger_count")
     return
 
 
 @app.cell(hide_code=True)
 def _(mo):
-    mo.md(r"""The partition is great, but the comparison with `read_csv` is a bit unfair. Let's convert the `.csv` file to `.parquet` and also add a partition in polars with statistics. """)
+    mo.md(
+        r"""
+    That's a bunch slower!
+
+    A part of the reason is that iceberg had partitions in it, which is great, but the comparison with `read_csv` is a bit unfair. Let's convert the `.csv` file to `.parquet` and also add a partition in polars with statistics. You will now see that we get a similar performance. 
+    """
+    )
     return
 
 
 @app.cell
-def _(pl):
+def _(df_orig):
     df_orig.write_parquet("taxi.parquet", partition_by=["passenger_count"], statistics=True)
     return
 
 
 @app.cell
 def _(pl):
-    df_orig.group_by("passenger_count").len().sort("passenger_count").collect()
+    pl.scan_parquet("taxi.parquet").group_by("passenger_count").len().sort("passenger_count").collect()
     return
 
 
 @app.cell
 def _(pl):
-    df_orig.group_by("passenger_count").len().sort("passenger_count")
+    pl.read_parquet("taxi.parquet").group_by("passenger_count").len().sort("passenger_count")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""So keep in mind that polars can for sure also speed things up if you are aware of what you are doing. But one nice thing about iceberg is that can be seen as a catalogue with *a bunch* of good habbits for performance later down the line.""")
     return