Spaces:
Sleeping
Sleeping
Merge pull request #86 from debajyotid2/add_polars_tutorials
Browse files- polars/08_working_with_columns.py +605 -0
- polars/09_data_types.py +296 -0
polars/08_working_with_columns.py
ADDED
@@ -0,0 +1,605 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# /// script
|
2 |
+
# requires-python = ">=3.11"
|
3 |
+
# dependencies = [
|
4 |
+
# "polars==1.18.0",
|
5 |
+
# "marimo",
|
6 |
+
# ]
|
7 |
+
# ///
|
8 |
+
|
9 |
+
import marimo
|
10 |
+
|
11 |
+
__generated_with = "0.12.0"
|
12 |
+
app = marimo.App(width="medium")
|
13 |
+
|
14 |
+
|
15 |
+
@app.cell(hide_code=True)
|
16 |
+
def _(mo):
|
17 |
+
mo.md(
|
18 |
+
r"""
|
19 |
+
# Working with Columns
|
20 |
+
|
21 |
+
Author: [Deb Debnath](https://github.com/debajyotid2)
|
22 |
+
|
23 |
+
**Note**: The following tutorial has been adapted from the Polars [documentation](https://docs.pola.rs/user-guide/expressions/expression-expansion).
|
24 |
+
"""
|
25 |
+
)
|
26 |
+
return
|
27 |
+
|
28 |
+
|
29 |
+
@app.cell(hide_code=True)
|
30 |
+
def _(mo):
|
31 |
+
mo.md(
|
32 |
+
r"""
|
33 |
+
## Expressions
|
34 |
+
|
35 |
+
Data transformations are sometimes complicated, or involve massive computations which are time-consuming. You can make a small version of the dataset with the schema you are trying to work your transformation into. But there is a better way to do it in Polars.
|
36 |
+
|
37 |
+
A Polars expression is a lazy representation of a data transformation. "Lazy" means that the transformation is not eagerly (immediately) executed.
|
38 |
+
|
39 |
+
Expressions are modular and flexible. They can be composed to build more complex expressions. For example, to calculate speed from distance and time, you can have an expression as:
|
40 |
+
"""
|
41 |
+
)
|
42 |
+
return
|
43 |
+
|
44 |
+
|
45 |
+
@app.cell
|
46 |
+
def _(pl):
|
47 |
+
speed_expr = pl.col("distance") / (pl.col("time"))
|
48 |
+
speed_expr
|
49 |
+
return (speed_expr,)
|
50 |
+
|
51 |
+
|
52 |
+
@app.cell(hide_code=True)
|
53 |
+
def _(mo):
|
54 |
+
mo.md(
|
55 |
+
r"""
|
56 |
+
## Expression expansion
|
57 |
+
|
58 |
+
Expression expansion lets you write a single expression that can expand to multiple different expressions. So rather than repeatedly defining separate expressions, you can avoid redundancy while adhering to clean code principles (Do not Repeat Yourself - [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself)). Since expressions are reusable, they aid in writing concise code.
|
59 |
+
"""
|
60 |
+
)
|
61 |
+
return
|
62 |
+
|
63 |
+
|
64 |
+
@app.cell(hide_code=True)
|
65 |
+
def _(mo):
|
66 |
+
mo.md("""For the examples in this notebook, we will use a sliver of the *AI4I 2020 Predictive Maintenance Dataset*. This dataset comprises of measurements taken from sensors in industrial machinery undergoing preventive maintenance checks - basically being tested for failure conditions.""")
|
67 |
+
return
|
68 |
+
|
69 |
+
|
70 |
+
@app.cell
|
71 |
+
def _(StringIO, pl):
|
72 |
+
data_csv = """
|
73 |
+
Product ID,Type,Air temperature,Process temperature,Rotational speed,Tool wear,Machine failure,TWF,HDF,PWF,OSF,RNF
|
74 |
+
L51172,L,302.3,311.3,1614,129,0,0,1,0,0,0
|
75 |
+
M22586,M,300.8,311.9,1761,113,1,0,0,0,1,0
|
76 |
+
L51639,L,302.6,310.4,1743,191,0,1,0,0,0,1
|
77 |
+
L50250,L,300,309.1,1631,110,0,0,0,1,0,0
|
78 |
+
M20109,M,303.4,312.9,1422,63,1,0,0,0,0,0
|
79 |
+
"""
|
80 |
+
|
81 |
+
data = pl.read_csv(StringIO(data_csv))
|
82 |
+
data
|
83 |
+
return data, data_csv
|
84 |
+
|
85 |
+
|
86 |
+
@app.cell(hide_code=True)
|
87 |
+
def _(mo):
|
88 |
+
mo.md(
|
89 |
+
r"""
|
90 |
+
## Function `col`
|
91 |
+
|
92 |
+
The function `col` is used to refer to one column of a dataframe. It is one of the fundamental building blocks of expressions in Polars. `col` is also really handy in expression expansion.
|
93 |
+
"""
|
94 |
+
)
|
95 |
+
return
|
96 |
+
|
97 |
+
|
98 |
+
@app.cell(hide_code=True)
|
99 |
+
def _(mo):
|
100 |
+
mo.md(
|
101 |
+
r"""
|
102 |
+
### Explicit expansion by column name
|
103 |
+
|
104 |
+
The simplest form of expression expansion happens when you provide multiple column names to the function `col`.
|
105 |
+
|
106 |
+
Say you wish to convert all temperature values in deg. Kelvin (K) to deg. Fahrenheit (F). One way to do this would be to define individual expressions for each column as follows:
|
107 |
+
"""
|
108 |
+
)
|
109 |
+
return
|
110 |
+
|
111 |
+
|
112 |
+
@app.cell
|
113 |
+
def _(data, pl):
|
114 |
+
exprs = [
|
115 |
+
((pl.col("Air temperature") - 273.15) * 1.8 + 32).round(2),
|
116 |
+
((pl.col("Process temperature") - 273.15) * 1.8 + 32).round(2)
|
117 |
+
]
|
118 |
+
|
119 |
+
result = data.with_columns(exprs)
|
120 |
+
result
|
121 |
+
return exprs, result
|
122 |
+
|
123 |
+
|
124 |
+
@app.cell(hide_code=True)
|
125 |
+
def _(mo):
|
126 |
+
mo.md(r"""Expression expansion can reduce this verbosity when you list the column names you want the expression to expand to inside the `col` function. The result is the same as before.""")
|
127 |
+
return
|
128 |
+
|
129 |
+
|
130 |
+
@app.cell
|
131 |
+
def _(data, pl, result):
|
132 |
+
result_2 = data.with_columns(
|
133 |
+
(
|
134 |
+
(pl.col(
|
135 |
+
"Air temperature",
|
136 |
+
"Process temperature"
|
137 |
+
)
|
138 |
+
- 273.15) * 1.8 + 32
|
139 |
+
).round(2)
|
140 |
+
)
|
141 |
+
result_2.equals(result)
|
142 |
+
return (result_2,)
|
143 |
+
|
144 |
+
|
145 |
+
@app.cell(hide_code=True)
|
146 |
+
def _(mo):
|
147 |
+
mo.md(r"""In this case, the expression that does the temperature conversion is expanded to a list of two expressions. The expansion of the expression is predictable and intuitive.""")
|
148 |
+
return
|
149 |
+
|
150 |
+
|
151 |
+
@app.cell(hide_code=True)
|
152 |
+
def _(mo):
|
153 |
+
mo.md(
|
154 |
+
r"""
|
155 |
+
### Expansion by data type
|
156 |
+
|
157 |
+
Can we do better than explicitly writing the names of every columns we want transformed? Yes.
|
158 |
+
|
159 |
+
If you provide data types instead of column names, the expression is expanded to all columns that match one of the data types provided.
|
160 |
+
|
161 |
+
The example below performs the exact same computation as before:
|
162 |
+
"""
|
163 |
+
)
|
164 |
+
return
|
165 |
+
|
166 |
+
|
167 |
+
@app.cell
|
168 |
+
def _(data, pl, result):
|
169 |
+
result_3 = data.with_columns(((pl.col(pl.Float64) - 273.15) * 1.8 + 32).round(2))
|
170 |
+
result_3.equals(result)
|
171 |
+
return (result_3,)
|
172 |
+
|
173 |
+
|
174 |
+
@app.cell(hide_code=True)
|
175 |
+
def _(mo):
|
176 |
+
mo.md(
|
177 |
+
r"""
|
178 |
+
However, you should be careful to ensure that the transformation is only applied to the columns you want. For ensuring this it is important to know the schema of the data beforehand.
|
179 |
+
|
180 |
+
`col` accepts multiple data types in case the columns you need have more than one data type.
|
181 |
+
"""
|
182 |
+
)
|
183 |
+
return
|
184 |
+
|
185 |
+
|
186 |
+
@app.cell
|
187 |
+
def _(data, pl, result):
|
188 |
+
result_4 = data.with_columns(
|
189 |
+
(
|
190 |
+
(pl.col(
|
191 |
+
pl.Float32,
|
192 |
+
pl.Float64,
|
193 |
+
)
|
194 |
+
- 273.15) * 1.8 + 32
|
195 |
+
).round(2)
|
196 |
+
)
|
197 |
+
result.equals(result_4)
|
198 |
+
return (result_4,)
|
199 |
+
|
200 |
+
|
201 |
+
@app.cell(hide_code=True)
|
202 |
+
def _(mo):
|
203 |
+
mo.md(
|
204 |
+
r"""
|
205 |
+
### Expansion by pattern matching
|
206 |
+
|
207 |
+
`col` also accepts regular expressions for selecting columns by pattern matching. Regular expressions start and end with ^ and $, respectively.
|
208 |
+
"""
|
209 |
+
)
|
210 |
+
return
|
211 |
+
|
212 |
+
|
213 |
+
@app.cell
|
214 |
+
def _(data, pl):
|
215 |
+
data.select(pl.col("^.*temperature$"))
|
216 |
+
return
|
217 |
+
|
218 |
+
|
219 |
+
@app.cell(hide_code=True)
|
220 |
+
def _(mo):
|
221 |
+
mo.md(r"""Regular expressions can be combined with exact column names.""")
|
222 |
+
return
|
223 |
+
|
224 |
+
|
225 |
+
@app.cell
|
226 |
+
def _(data, pl):
|
227 |
+
data.select(pl.col("^.*temperature$", "Tool wear"))
|
228 |
+
return
|
229 |
+
|
230 |
+
|
231 |
+
@app.cell(hide_code=True)
|
232 |
+
def _(mo):
|
233 |
+
mo.md(r"""**Note**: You _cannot_ mix strings (exact names, regular expressions) and data types in a `col` function.""")
|
234 |
+
return
|
235 |
+
|
236 |
+
|
237 |
+
@app.cell
|
238 |
+
def _(data, pl):
|
239 |
+
try:
|
240 |
+
data.select(pl.col("Air temperature", pl.Float64))
|
241 |
+
except TypeError as err:
|
242 |
+
print("TypeError:", err)
|
243 |
+
return
|
244 |
+
|
245 |
+
|
246 |
+
@app.cell(hide_code=True)
|
247 |
+
def _(mo):
|
248 |
+
mo.md(
|
249 |
+
r"""
|
250 |
+
## Selecting all columns
|
251 |
+
|
252 |
+
To select all columns, you can use the `all` function.
|
253 |
+
"""
|
254 |
+
)
|
255 |
+
return
|
256 |
+
|
257 |
+
|
258 |
+
@app.cell
|
259 |
+
def _(data, pl):
|
260 |
+
result_6 = data.select(pl.all())
|
261 |
+
result_6.equals(data)
|
262 |
+
return (result_6,)
|
263 |
+
|
264 |
+
|
265 |
+
@app.cell(hide_code=True)
|
266 |
+
def _(mo):
|
267 |
+
mo.md(
|
268 |
+
r"""
|
269 |
+
## Excluding columns
|
270 |
+
|
271 |
+
There are scenarios where we might want to exclude specific columns from the ones selected by building expressions, e.g. by the `col` or `all` functions. For this purpose, we use the function `exclude`, which accepts exactly the same types of arguments as `col`:
|
272 |
+
"""
|
273 |
+
)
|
274 |
+
return
|
275 |
+
|
276 |
+
|
277 |
+
@app.cell
|
278 |
+
def _(data, pl):
|
279 |
+
data.select(pl.all().exclude("^.*F$"))
|
280 |
+
return
|
281 |
+
|
282 |
+
|
283 |
+
@app.cell(hide_code=True)
|
284 |
+
def _(mo):
|
285 |
+
mo.md(r"""`exclude` can also be used after the function `col`:""")
|
286 |
+
return
|
287 |
+
|
288 |
+
|
289 |
+
@app.cell
|
290 |
+
def _(data, pl):
|
291 |
+
data.select(pl.col(pl.Int64).exclude("^.*F$"))
|
292 |
+
return
|
293 |
+
|
294 |
+
|
295 |
+
@app.cell(hide_code=True)
|
296 |
+
def _(mo):
|
297 |
+
mo.md(
|
298 |
+
r"""
|
299 |
+
## Column renaming
|
300 |
+
|
301 |
+
When applying a transformation with an expression to a column, the data in the column gets overwritten with the transformed data. However, this might not be the intended outcome in all situations - ideally you would want to store transformed data in a new column. Applying multiple transformations to the same column at the same time without renaming leads to errors.
|
302 |
+
"""
|
303 |
+
)
|
304 |
+
return
|
305 |
+
|
306 |
+
|
307 |
+
@app.cell
|
308 |
+
def _(data, pl):
|
309 |
+
from polars.exceptions import DuplicateError
|
310 |
+
|
311 |
+
try:
|
312 |
+
data.select(
|
313 |
+
(pl.col("Air temperature") - 273.15) * 1.8 + 32, # This would be named "Air temperature"...
|
314 |
+
pl.col("Air temperature") - 273.15, # And so would this.
|
315 |
+
)
|
316 |
+
except DuplicateError as err:
|
317 |
+
print("DuplicateError:", err)
|
318 |
+
return (DuplicateError,)
|
319 |
+
|
320 |
+
|
321 |
+
@app.cell(hide_code=True)
|
322 |
+
def _(mo):
|
323 |
+
mo.md(
|
324 |
+
r"""
|
325 |
+
### Renaming a single column with `alias`
|
326 |
+
|
327 |
+
The function `alias` lets you rename a single column:
|
328 |
+
"""
|
329 |
+
)
|
330 |
+
return
|
331 |
+
|
332 |
+
|
333 |
+
@app.cell
|
334 |
+
def _(data, pl):
|
335 |
+
data.select(
|
336 |
+
((pl.col("Air temperature") - 273.15) * 1.8 + 32).round(2).alias("Air temperature [F]"),
|
337 |
+
(pl.col("Air temperature") - 273.15).round(2).alias("Air temperature [C]")
|
338 |
+
)
|
339 |
+
return
|
340 |
+
|
341 |
+
|
342 |
+
@app.cell(hide_code=True)
|
343 |
+
def _(mo):
|
344 |
+
mo.md(
|
345 |
+
r"""
|
346 |
+
### Prefixing and suffixing column names
|
347 |
+
|
348 |
+
As `alias` renames a single column at a time, it cannot be used during expression expansion. If it is sufficient add a static prefix or a static suffix to the existing names, you can use the functions `name.prefix` and `name.suffix` with `col`:
|
349 |
+
"""
|
350 |
+
)
|
351 |
+
return
|
352 |
+
|
353 |
+
|
354 |
+
@app.cell
|
355 |
+
def _(data, pl):
|
356 |
+
data.select(
|
357 |
+
((pl.col("Air temperature") - 273.15) * 1.8 + 32).round(2).name.prefix("deg F "),
|
358 |
+
(pl.col("Process temperature") - 273.15).round(2).name.suffix(" C"),
|
359 |
+
)
|
360 |
+
return
|
361 |
+
|
362 |
+
|
363 |
+
@app.cell(hide_code=True)
|
364 |
+
def _(mo):
|
365 |
+
mo.md(
|
366 |
+
r"""
|
367 |
+
### Dynamic name replacement
|
368 |
+
|
369 |
+
If a static prefix/suffix is not enough, use `name.map`. `name.map` requires a function that transforms column names to the desired. The transformation should lead to unique names to avoid `DuplicateError`.
|
370 |
+
"""
|
371 |
+
)
|
372 |
+
return
|
373 |
+
|
374 |
+
|
375 |
+
@app.cell
|
376 |
+
def _(data, pl):
|
377 |
+
# There is also `.name.to_lowercase`, so this usage of `.map` is moot.
|
378 |
+
data.select(pl.col("^.*F$").name.map(str.lower))
|
379 |
+
return
|
380 |
+
|
381 |
+
|
382 |
+
@app.cell(hide_code=True)
|
383 |
+
def _(mo):
|
384 |
+
mo.md(
|
385 |
+
r"""
|
386 |
+
## Programmatically generating expressions
|
387 |
+
|
388 |
+
For this example, we will first create four additional columns with the rolling mean temperatures of the two temperature columns. Such transformations are sometimes used to create additional features for machine learning models or data analysis.
|
389 |
+
"""
|
390 |
+
)
|
391 |
+
return
|
392 |
+
|
393 |
+
|
394 |
+
@app.cell
|
395 |
+
def _(data, pl):
|
396 |
+
ext_temp_data = data.with_columns(
|
397 |
+
pl.col("^.*temperature$").rolling_mean(window_size=2).round(2).name.prefix("Rolling mean ")
|
398 |
+
).select(pl.col("^.*temperature*$"))
|
399 |
+
ext_temp_data
|
400 |
+
return (ext_temp_data,)
|
401 |
+
|
402 |
+
|
403 |
+
@app.cell(hide_code=True)
|
404 |
+
def _(mo):
|
405 |
+
mo.md(r"""Now, suppose we want to calculate the difference between the rolling mean and actual temperatures. We cannot use expression expansion here as we want differences between specific columns.""")
|
406 |
+
return
|
407 |
+
|
408 |
+
|
409 |
+
@app.cell(hide_code=True)
|
410 |
+
def _(mo):
|
411 |
+
mo.md(r"""At first, you may think about using a `for` loop:""")
|
412 |
+
return
|
413 |
+
|
414 |
+
|
415 |
+
@app.cell
|
416 |
+
def _(ext_temp_data, pl):
|
417 |
+
_result = ext_temp_data
|
418 |
+
for col_name in ["Air", "Process"]:
|
419 |
+
_result = _result.with_columns(
|
420 |
+
(abs(pl.col(f"Rolling mean {col_name} temperature") - pl.col(f"{col_name} temperature")))
|
421 |
+
.round(2).alias(f"Delta {col_name} temperature")
|
422 |
+
)
|
423 |
+
_result
|
424 |
+
return (col_name,)
|
425 |
+
|
426 |
+
|
427 |
+
@app.cell(hide_code=True)
|
428 |
+
def _(mo):
|
429 |
+
mo.md(r"""Using a `for` loop is functional, but not scalable, as each expression needs to be defined in an iteration and executed serially. Instead we can use a generator in Python to programmatically create all expressions at once. In conjunction with the `with_columns` context, we can take advantage of parallel execution of computations and query optimization from Polars.""")
|
430 |
+
return
|
431 |
+
|
432 |
+
|
433 |
+
@app.cell
|
434 |
+
def _(ext_temp_data, pl):
|
435 |
+
def delta_expressions(colnames: list[str]) -> pl.Expr:
|
436 |
+
for col_name in colnames:
|
437 |
+
yield (abs(pl.col(f"Rolling mean {col_name} temperature") - pl.col(f"{col_name} temperature"))
|
438 |
+
.round(2).alias(f"Delta {col_name} temperature"))
|
439 |
+
|
440 |
+
|
441 |
+
ext_temp_data.with_columns(delta_expressions(["Air", "Process"]))
|
442 |
+
return (delta_expressions,)
|
443 |
+
|
444 |
+
|
445 |
+
@app.cell(hide_code=True)
|
446 |
+
def _(mo):
|
447 |
+
mo.md(
|
448 |
+
r"""
|
449 |
+
## More flexible column selections
|
450 |
+
|
451 |
+
For more flexible column selections, you can use column selectors from `selectors`. Column selectors allow for more expressiveness in the way you specify selections. For example, column selectors can perform the familiar set operations of union, intersection, difference, etc. We can use the union operation with the functions `string` and `ends_with` to select all string columns and the columns whose names end with "`_high`":
|
452 |
+
"""
|
453 |
+
)
|
454 |
+
return
|
455 |
+
|
456 |
+
|
457 |
+
@app.cell
|
458 |
+
def _(data):
|
459 |
+
import polars.selectors as cs
|
460 |
+
|
461 |
+
data.select(cs.string() | cs.ends_with("F"))
|
462 |
+
return (cs,)
|
463 |
+
|
464 |
+
|
465 |
+
@app.cell(hide_code=True)
|
466 |
+
def _(mo):
|
467 |
+
mo.md(r"""Likewise, you can pick columns based on the category of the type of data, offering more flexibility than the `col` function. As an example, `cs.numeric` selects numeric data types (including `pl.Float32`, `pl.Float64`, `pl.Int32`, etc.) or `cs.temporal` for all dates, times and similar data types.""")
|
468 |
+
return
|
469 |
+
|
470 |
+
|
471 |
+
@app.cell(hide_code=True)
|
472 |
+
def _(mo):
|
473 |
+
mo.md(
|
474 |
+
r"""
|
475 |
+
### Combining selectors with set operations
|
476 |
+
|
477 |
+
Multiple selectors can be combined using set operations and the usual Python operators:
|
478 |
+
|
479 |
+
|
480 |
+
| Operator | Operation |
|
481 |
+
|:--------:|:--------------------:|
|
482 |
+
| `A | B` | Union |
|
483 |
+
| `A & B` | Intersection |
|
484 |
+
| `A - B` | Difference |
|
485 |
+
| `A ^ B` | Symmetric difference |
|
486 |
+
| `~A` | Complement |
|
487 |
+
|
488 |
+
For example, to select all failure indicator variables excluding the failure variables due to wear, we can perform a set difference between the column selectors.
|
489 |
+
"""
|
490 |
+
)
|
491 |
+
return
|
492 |
+
|
493 |
+
|
494 |
+
@app.cell
|
495 |
+
def _(cs, data):
|
496 |
+
data.select(cs.contains("F") - cs.contains("W"))
|
497 |
+
return
|
498 |
+
|
499 |
+
|
500 |
+
@app.cell(hide_code=True)
|
501 |
+
def _(mo):
|
502 |
+
mo.md(
|
503 |
+
r"""
|
504 |
+
### Resolving operator ambiguity
|
505 |
+
|
506 |
+
Expression functions can be chained on top of selectors:
|
507 |
+
"""
|
508 |
+
)
|
509 |
+
return
|
510 |
+
|
511 |
+
|
512 |
+
@app.cell
|
513 |
+
def _(cs, data, pl):
|
514 |
+
ext_failure_data = data.select(cs.contains("F")).cast(pl.Boolean)
|
515 |
+
ext_failure_data
|
516 |
+
return (ext_failure_data,)
|
517 |
+
|
518 |
+
|
519 |
+
@app.cell(hide_code=True)
|
520 |
+
def _(mo):
|
521 |
+
mo.md(
|
522 |
+
r"""
|
523 |
+
However, operators that perform set operations on column selectors operate on both selectors and on expressions. For example, the operator `~` on a selector represents the set operation “complement” and on an expression represents the Boolean operation of negation.
|
524 |
+
|
525 |
+
For instance, if you want to negate the Boolean values in the columns “HDF”, “OSF”, and “RNF”, at first you would think about using the `~` operator with the column selector to choose all failure variables containing "W". Because of the operator ambiguity here, the columns that are not of interest are selected here.
|
526 |
+
"""
|
527 |
+
)
|
528 |
+
return
|
529 |
+
|
530 |
+
|
531 |
+
@app.cell
|
532 |
+
def _(cs, ext_failure_data):
|
533 |
+
ext_failure_data.select((~cs.ends_with("WF")).name.prefix("No"))
|
534 |
+
return
|
535 |
+
|
536 |
+
|
537 |
+
@app.cell(hide_code=True)
|
538 |
+
def _(mo):
|
539 |
+
mo.md(r"""To resolve the operator ambiguity, we use `as_expr`:""")
|
540 |
+
return
|
541 |
+
|
542 |
+
|
543 |
+
@app.cell
|
544 |
+
def _(cs, ext_failure_data):
|
545 |
+
ext_failure_data.select((~cs.ends_with("WF").as_expr()).name.prefix("No"))
|
546 |
+
return
|
547 |
+
|
548 |
+
|
549 |
+
@app.cell(hide_code=True)
|
550 |
+
def _(mo):
|
551 |
+
mo.md(
|
552 |
+
r"""
|
553 |
+
### Debugging selectors
|
554 |
+
|
555 |
+
The function `cs.is_selector` helps check whether a complex chain of selectors and operators ultimately results in a selector. For example, to resolve any ambiguity with the selector in the last example, we can do:
|
556 |
+
"""
|
557 |
+
)
|
558 |
+
return
|
559 |
+
|
560 |
+
|
561 |
+
@app.cell
|
562 |
+
def _(cs):
|
563 |
+
cs.is_selector(~cs.ends_with("WF").as_expr())
|
564 |
+
return
|
565 |
+
|
566 |
+
|
567 |
+
@app.cell(hide_code=True)
|
568 |
+
def _(mo):
|
569 |
+
mo.md(r"""Additionally we can use `expand_selector` to see what columns a selector expands into. Note that for this function we need to provide additional context in the form of the dataframe.""")
|
570 |
+
return
|
571 |
+
|
572 |
+
|
573 |
+
@app.cell
|
574 |
+
def _(cs, ext_failure_data):
|
575 |
+
cs.expand_selector(
|
576 |
+
ext_failure_data,
|
577 |
+
cs.ends_with("WF"),
|
578 |
+
)
|
579 |
+
return
|
580 |
+
|
581 |
+
|
582 |
+
@app.cell(hide_code=True)
|
583 |
+
def _(mo):
|
584 |
+
mo.md(
|
585 |
+
r"""
|
586 |
+
### References
|
587 |
+
|
588 |
+
1. AI4I 2020 Predictive Maintenance Dataset [Dataset]. (2020). UCI Machine Learning Repository. ([link](https://doi.org/10.24432/C5HS5C)).
|
589 |
+
2. Polars documentation ([link](https://docs.pola.rs/user-guide/expressions/expression-expansion/#more-flexible-column-selections))
|
590 |
+
"""
|
591 |
+
)
|
592 |
+
return
|
593 |
+
|
594 |
+
|
595 |
+
@app.cell(hide_code=True)
|
596 |
+
def _():
|
597 |
+
import csv
|
598 |
+
import marimo as mo
|
599 |
+
import polars as pl
|
600 |
+
from io import StringIO
|
601 |
+
return StringIO, csv, mo, pl
|
602 |
+
|
603 |
+
|
604 |
+
if __name__ == "__main__":
|
605 |
+
app.run()
|
polars/09_data_types.py
ADDED
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# /// script
|
2 |
+
# requires-python = ">=3.11"
|
3 |
+
# dependencies = [
|
4 |
+
# "polars==1.18.0",
|
5 |
+
# "marimo",
|
6 |
+
# ]
|
7 |
+
# ///
|
8 |
+
|
9 |
+
import marimo
|
10 |
+
|
11 |
+
__generated_with = "0.12.0"
|
12 |
+
app = marimo.App(width="medium")
|
13 |
+
|
14 |
+
|
15 |
+
@app.cell(hide_code=True)
|
16 |
+
def _(mo):
|
17 |
+
mo.md(
|
18 |
+
r"""
|
19 |
+
# Data Types
|
20 |
+
|
21 |
+
Author: [Deb Debnath](https://github.com/debajyotid2)
|
22 |
+
|
23 |
+
**Note**: The following tutorial has been adapted from the Polars [documentation](https://docs.pola.rs/user-guide/concepts/data-types-and-structures/).
|
24 |
+
"""
|
25 |
+
)
|
26 |
+
return
|
27 |
+
|
28 |
+
|
29 |
+
@app.cell(hide_code=True)
|
30 |
+
def _(mo):
|
31 |
+
mo.md(
|
32 |
+
r"""
|
33 |
+
Polars supports a variety of data types that fall broadly under the following categories:
|
34 |
+
|
35 |
+
- Numeric data types: integers and floating point numbers.
|
36 |
+
- Nested data types: lists, structs, and arrays.
|
37 |
+
- Temporal: dates, datetimes, times, and time deltas.
|
38 |
+
- Miscellaneous: strings, binary data, Booleans, categoricals, enums, and objects.
|
39 |
+
|
40 |
+
All types support missing values represented by `null` which is different from `NaN` used in floating point data types. The numeric datatypes in Polars loosely follow the type system of the Rust language, since its core functionalities are built in Rust.
|
41 |
+
|
42 |
+
[Here](https://docs.pola.rs/api/python/stable/reference/datatypes.html) is a full list of all data types Polars supports.
|
43 |
+
"""
|
44 |
+
)
|
45 |
+
return
|
46 |
+
|
47 |
+
|
48 |
+
@app.cell(hide_code=True)
|
49 |
+
def _(mo):
|
50 |
+
mo.md(
|
51 |
+
r"""
|
52 |
+
## Series
|
53 |
+
|
54 |
+
A series is a 1-dimensional data structure that can hold only one data type.
|
55 |
+
"""
|
56 |
+
)
|
57 |
+
return
|
58 |
+
|
59 |
+
|
60 |
+
@app.cell
|
61 |
+
def _(pl):
|
62 |
+
s = pl.Series("emojis", ["😀", "🤣", "🥶", "💀", "🤖"])
|
63 |
+
s
|
64 |
+
return (s,)
|
65 |
+
|
66 |
+
|
67 |
+
@app.cell(hide_code=True)
|
68 |
+
def _(mo):
|
69 |
+
mo.md(r"""Unless specified, Polars infers the datatype from the supplied values.""")
|
70 |
+
return
|
71 |
+
|
72 |
+
|
73 |
+
@app.cell
|
74 |
+
def _(pl):
|
75 |
+
s1 = pl.Series("friends", ["Евгений", "अभिषेक", "秀良", "Federico", "Bob"])
|
76 |
+
s2 = pl.Series("uints", [0x00, 0x01, 0x10, 0x11], dtype=pl.UInt8)
|
77 |
+
s1.dtype, s2.dtype
|
78 |
+
return s1, s2
|
79 |
+
|
80 |
+
|
81 |
+
@app.cell(hide_code=True)
|
82 |
+
def _(mo):
|
83 |
+
mo.md(
|
84 |
+
r"""
|
85 |
+
## Dataframe
|
86 |
+
|
87 |
+
A dataframe is a 2-dimensional data structure that contains uniquely named series and can hold multiple data types. Dataframes are more commonly used for data manipulation using the functionality of Polars.
|
88 |
+
|
89 |
+
The snippet below shows how to create a dataframe from a dictionary of lists:
|
90 |
+
"""
|
91 |
+
)
|
92 |
+
return
|
93 |
+
|
94 |
+
|
95 |
+
@app.cell
|
96 |
+
def _(pl):
|
97 |
+
data = pl.DataFrame(
|
98 |
+
{
|
99 |
+
"Product ID": ["L51172", "M22586", "L51639", "L50250", "M20109"],
|
100 |
+
"Type": ["L", "M", "L", "L", "M"],
|
101 |
+
"Air temperature": [302.3, 300.8, 302.6, 300, 303.4], # (K)
|
102 |
+
"Machine Failure": [False, True, False, False, True]
|
103 |
+
}
|
104 |
+
)
|
105 |
+
data
|
106 |
+
return (data,)
|
107 |
+
|
108 |
+
|
109 |
+
@app.cell(hide_code=True)
|
110 |
+
def _(mo):
|
111 |
+
mo.md(
|
112 |
+
r"""
|
113 |
+
### Inspecting a dataframe
|
114 |
+
|
115 |
+
Polars has various functions to explore the data in a dataframe. We will use the dataframe `data` defined above in our examples. Alongside we can also see a view of the dataframe rendered by `marimo` as the cells are executed.
|
116 |
+
|
117 |
+
///note
|
118 |
+
We can also use `marimo`'s built in data-inspection elements/features such as [`mo.ui.dataframe`](https://docs.marimo.io/api/inputs/dataframe/#marimo.ui.dataframe) & [`mo.ui.data_explorer`](https://docs.marimo.io/api/inputs/data_explorer/). For more check out our Polars tutorials at [`marimo learn`](https://marimo-team.github.io/learn/)!
|
119 |
+
"""
|
120 |
+
)
|
121 |
+
return
|
122 |
+
|
123 |
+
|
124 |
+
@app.cell(hide_code=True)
|
125 |
+
def _(mo):
|
126 |
+
mo.md(
|
127 |
+
"""
|
128 |
+
#### Head
|
129 |
+
|
130 |
+
The function `head` shows the first rows of a dataframe. Unless specified, it shows the first 5 rows.
|
131 |
+
"""
|
132 |
+
)
|
133 |
+
return
|
134 |
+
|
135 |
+
|
136 |
+
@app.cell
|
137 |
+
def _(data):
|
138 |
+
data.head(3)
|
139 |
+
return
|
140 |
+
|
141 |
+
|
142 |
+
@app.cell(hide_code=True)
|
143 |
+
def _(mo):
|
144 |
+
mo.md(
|
145 |
+
r"""
|
146 |
+
#### Glimpse
|
147 |
+
|
148 |
+
The function `glimpse` is an alternative to `head` to view the first few columns, but displays each line of the output corresponding to a single column. That way, it makes inspecting wider dataframes easier.
|
149 |
+
"""
|
150 |
+
)
|
151 |
+
return
|
152 |
+
|
153 |
+
|
154 |
+
@app.cell
|
155 |
+
def _(data):
|
156 |
+
print(data.glimpse(return_as_string=True))
|
157 |
+
return
|
158 |
+
|
159 |
+
|
160 |
+
@app.cell(hide_code=True)
|
161 |
+
def _(mo):
|
162 |
+
mo.md(
|
163 |
+
r"""
|
164 |
+
#### Tail
|
165 |
+
|
166 |
+
The `tail` function, just like its name suggests, shows the last rows of a dataframe. Unless the number of rows is specified, it will show the last 5 rows.
|
167 |
+
"""
|
168 |
+
)
|
169 |
+
return
|
170 |
+
|
171 |
+
|
172 |
+
@app.cell
|
173 |
+
def _(data):
|
174 |
+
data.tail(3)
|
175 |
+
return
|
176 |
+
|
177 |
+
|
178 |
+
@app.cell(hide_code=True)
|
179 |
+
def _(mo):
|
180 |
+
mo.md(
|
181 |
+
r"""
|
182 |
+
#### Sample
|
183 |
+
|
184 |
+
`sample` can be used to show a specified number of randomly selected rows from the dataframe. Unless the number of rows is specified, it will show a single row. `sample` does not preserve order of the rows.
|
185 |
+
"""
|
186 |
+
)
|
187 |
+
return
|
188 |
+
|
189 |
+
|
190 |
+
@app.cell
|
191 |
+
def _(data):
|
192 |
+
import random
|
193 |
+
|
194 |
+
random.seed(42) # For reproducibility.
|
195 |
+
|
196 |
+
data.sample(3)
|
197 |
+
return (random,)
|
198 |
+
|
199 |
+
|
200 |
+
@app.cell(hide_code=True)
|
201 |
+
def _(mo):
|
202 |
+
mo.md(
|
203 |
+
r"""
|
204 |
+
#### Describe
|
205 |
+
|
206 |
+
The function `describe` describes the summary statistics for all columns of a dataframe.
|
207 |
+
"""
|
208 |
+
)
|
209 |
+
return
|
210 |
+
|
211 |
+
|
212 |
+
@app.cell
|
213 |
+
def _(data):
|
214 |
+
data.describe()
|
215 |
+
return
|
216 |
+
|
217 |
+
|
218 |
+
@app.cell(hide_code=True)
|
219 |
+
def _(mo):
|
220 |
+
mo.md(
|
221 |
+
r"""
|
222 |
+
## Schema
|
223 |
+
|
224 |
+
A schema is a mapping showing the datatype corresponding to every column of a dataframe. The schema of a dataframe can be viewed using the attribute `schema`.
|
225 |
+
"""
|
226 |
+
)
|
227 |
+
return
|
228 |
+
|
229 |
+
|
230 |
+
@app.cell
|
231 |
+
def _(data):
|
232 |
+
data.schema
|
233 |
+
return
|
234 |
+
|
235 |
+
|
236 |
+
@app.cell(hide_code=True)
|
237 |
+
def _(mo):
|
238 |
+
mo.md(r"""Since a schema is a mapping, it can be specified in the form of a Python dictionary. Then this dictionary can be used to specify the schema of a dataframe on definition. If not specified or the entry is `None`, Polars infers the datatype from the contents of the column. Note that if the schema is not specified, it will be inferred automatically by default.""")
|
239 |
+
return
|
240 |
+
|
241 |
+
|
242 |
+
@app.cell
|
243 |
+
def _(pl):
|
244 |
+
pl.DataFrame(
|
245 |
+
{
|
246 |
+
"Product ID": ["L51172", "M22586", "L51639", "L50250", "M20109"],
|
247 |
+
"Type": ["L", "M", "L", "L", "M"],
|
248 |
+
"Air temperature": [302.3, 300.8, 302.6, 300, 303.4], # (K)
|
249 |
+
"Machine Failure": [False, True, False, False, True]
|
250 |
+
},
|
251 |
+
schema={"Product ID": pl.String, "Type": pl.String, "Air temperature": None, "Machine Failure": None},
|
252 |
+
)
|
253 |
+
return
|
254 |
+
|
255 |
+
|
256 |
+
@app.cell(hide_code=True)
|
257 |
+
def _(mo):
|
258 |
+
mo.md(r"""Sometimes the automatically inferred schema is enough for some columns, but we might wish to override the inference of only some columns. We can specify the schema for those columns using `schema_overrides`.""")
|
259 |
+
return
|
260 |
+
|
261 |
+
|
262 |
+
@app.cell
|
263 |
+
def _(pl):
|
264 |
+
pl.DataFrame(
|
265 |
+
{
|
266 |
+
"Product ID": ["L51172", "M22586", "L51639", "L50250", "M20109"],
|
267 |
+
"Type": ["L", "M", "L", "L", "M"],
|
268 |
+
"Air temperature": [302.3, 300.8, 302.6, 300, 303.4], # (K)
|
269 |
+
"Machine Failure": [False, True, False, False, True]
|
270 |
+
},
|
271 |
+
schema_overrides={"Air temperature": pl.Float32},
|
272 |
+
)
|
273 |
+
return
|
274 |
+
|
275 |
+
|
276 |
+
@app.cell(hide_code=True)
|
277 |
+
def _(mo):
|
278 |
+
mo.md(
|
279 |
+
r"""
|
280 |
+
### References
|
281 |
+
|
282 |
+
1. Polars documentation ([link](https://docs.pola.rs/api/python/stable/reference/datatypes.html))
|
283 |
+
"""
|
284 |
+
)
|
285 |
+
return
|
286 |
+
|
287 |
+
|
288 |
+
@app.cell(hide_code=True)
|
289 |
+
def _():
|
290 |
+
import marimo as mo
|
291 |
+
import polars as pl
|
292 |
+
return mo, pl
|
293 |
+
|
294 |
+
|
295 |
+
if __name__ == "__main__":
|
296 |
+
app.run()
|