Akshay Agrawal commited on
Commit
7b36ccd
·
unverified ·
2 Parent(s): 9b7fd9a 0a4006f

Merge pull request #52 from peter-gy/peter-gy/polars_09_strings

Browse files
Files changed (1) hide show
  1. polars/09_strings.py +1004 -0
polars/09_strings.py ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.12"
3
+ # dependencies = [
4
+ # "altair==5.5.0",
5
+ # "marimo",
6
+ # "numpy==2.2.3",
7
+ # "polars==1.24.0",
8
+ # ]
9
+ # ///
10
+
11
+ import marimo
12
+
13
+ __generated_with = "0.11.17"
14
+ app = marimo.App(width="medium")
15
+
16
+
17
+ @app.cell
18
+ def _(mo):
19
+ mo.md(
20
+ r"""
21
+ # Strings
22
+
23
+ _By [Péter Ferenc Gyarmati](http://github.com/peter-gy)_.
24
+
25
+ In this chapter we're going to dig into string manipulation. For a fun twist, we'll be mostly playing around with a dataset that every Polars user has bumped into without really thinking about it—the source code of the `polars` module itself. More precisely, we'll use a dataframe that pulls together all the Polars expressions and their docstrings, giving us a cool, hands-on way to explore the expression API in a truly data-driven manner.
26
+
27
+ We'll cover parsing, length calculation, case conversion, and much more, with practical examples and visualizations. Finally, we will combine various techniques you learned in prior chapters to build a fully interactive playground in which you can execute the official code examples of Polars expressions.
28
+ """
29
+ )
30
+ return
31
+
32
+
33
+ @app.cell
34
+ def _(mo):
35
+ mo.md(
36
+ r"""
37
+ ## 🛠️ Parsing & Conversion
38
+
39
+ Let's warm up with one of the most frequent use cases: parsing raw strings into various formats.
40
+ We'll take a tiny dataframe with metadata about Python packages represented as raw JSON strings and we'll use Polars string expressions to parse the attributes into their true data types.
41
+ """
42
+ )
43
+ return
44
+
45
+
46
+ @app.cell(hide_code=True)
47
+ def _(pl):
48
+ pip_metadata_raw_df = pl.DataFrame(
49
+ [
50
+ '{"package": "polars", "version": "1.24.0", "released_at": "2025-03-02T20:31:12+0000", "size_mb": "30.9"}',
51
+ '{"package": "marimo", "version": "0.11.14", "released_at": "2025-03-04T00:28:57+0000", "size_mb": "10.7"}',
52
+ ],
53
+ schema={"raw_json": pl.String},
54
+ )
55
+ pip_metadata_raw_df
56
+ return (pip_metadata_raw_df,)
57
+
58
+
59
+ @app.cell
60
+ def _(mo):
61
+ mo.md(r"""We can use the [`json_decode`](https://docs.pola.rs/api/python/stable/reference/series/api/polars.Series.str.json_decode.html) expression to parse the raw JSON strings into Polars-native structs and we can use the [unnest](https://docs.pola.rs/api/python/stable/reference/dataframe/api/polars.DataFrame.unnest.html) dataframe operation to have a dedicated column per parsed attribute.""")
62
+ return
63
+
64
+
65
+ @app.cell
66
+ def _(pip_metadata_raw_df, pl):
67
+ pip_metadata_df = pip_metadata_raw_df.select(json=pl.col('raw_json').str.json_decode()).unnest('json')
68
+ pip_metadata_df
69
+ return (pip_metadata_df,)
70
+
71
+
72
+ @app.cell
73
+ def _(mo):
74
+ mo.md(r"""This is already a much friendlier representation of the data we started out with, but note that since the JSON entries had only string attributes, all values are strings, even the temporal `released_at` and numerical `size_mb` columns.""")
75
+ return
76
+
77
+
78
+ @app.cell
79
+ def _(mo):
80
+ mo.md(r"""As we know that the `size_mb` column should have a decimal representation, we go ahead and use [`to_decimal`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_decimal.html#polars.Expr.str.to_decimal) to perform the conversion.""")
81
+ return
82
+
83
+
84
+ @app.cell
85
+ def _(pip_metadata_df, pl):
86
+ pip_metadata_df.select(
87
+ 'package',
88
+ 'version',
89
+ pl.col('size_mb').str.to_decimal(),
90
+ )
91
+ return
92
+
93
+
94
+ @app.cell
95
+ def _(mo):
96
+ mo.md(
97
+ r"""
98
+ Moving on to the `released_at` attribute which indicates the exact time when a given Python package got released, we have a bit more options to consider. We can convert to `Date`, `DateTime`, and `Time` types based on the desired temporal granularity. The [`to_date`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_date.html), [`to_datetime`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_datetime.html), and [`to_time`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_time.html) expressions are here to help us with the conversion, all we need is to provide the desired format string.
99
+
100
+ Since Polars uses Rust under the hood to implement all its expressions, we need to consult the [`chrono::format`](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) reference to come up with appropriate format strings.
101
+
102
+ Here's a quick reference:
103
+
104
+ | Specifier | Meaning |
105
+ |-----------|--------------------|
106
+ | `%Y` | Year (e.g., 2025) |
107
+ | `%m` | Month (01-12) |
108
+ | `%d` | Day (01-31) |
109
+ | `%H` | Hour (00-23) |
110
+ | `%z` | UTC offset |
111
+
112
+ The raw strings we are working with look like `"2025-03-02T20:31:12+0000"`. We can match this using the `"%Y-%m-%dT%H:%M:%S%z"` format string.
113
+ """
114
+ )
115
+ return
116
+
117
+
118
+ @app.cell
119
+ def _(pip_metadata_df, pl):
120
+ pip_metadata_df.select(
121
+ 'package',
122
+ 'version',
123
+ release_date=pl.col('released_at').str.to_date('%Y-%m-%dT%H:%M:%S%z'),
124
+ release_datetime=pl.col('released_at').str.to_datetime('%Y-%m-%dT%H:%M:%S%z'),
125
+ release_time=pl.col('released_at').str.to_time('%Y-%m-%dT%H:%M:%S%z'),
126
+ )
127
+ return
128
+
129
+
130
+ @app.cell
131
+ def _(mo):
132
+ mo.md(r"""Alternatively, instead of using three different functions to perform the conversion to date, we can use a single one, [`strptime`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strptime.html) which takes the desired temporal data type as its first parameter.""")
133
+ return
134
+
135
+
136
+ @app.cell
137
+ def _(pip_metadata_df, pl):
138
+ pip_metadata_df.select(
139
+ 'package',
140
+ 'version',
141
+ release_date=pl.col('released_at').str.strptime(pl.Date, '%Y-%m-%dT%H:%M:%S%z'),
142
+ release_datetime=pl.col('released_at').str.strptime(pl.Datetime, '%Y-%m-%dT%H:%M:%S%z'),
143
+ release_time=pl.col('released_at').str.strptime(pl.Time, '%Y-%m-%dT%H:%M:%S%z'),
144
+ )
145
+ return
146
+
147
+
148
+ @app.cell
149
+ def _(mo):
150
+ mo.md(r"""And to wrap up this section on parsing and conversion, let's consider a final scenario. What if we don't want to parse the entire raw JSON string, because we only need a subset of its attributes? Well, in this case we can leverage the [`json_path_match`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.json_path_match.html) expression to extract only the desired attributes using standard [JSONPath](https://goessner.net/articles/JsonPath/) syntax.""")
151
+ return
152
+
153
+
154
+ @app.cell
155
+ def _(pip_metadata_raw_df, pl):
156
+ pip_metadata_raw_df.select(
157
+ package=pl.col("raw_json").str.json_path_match("$.package"),
158
+ version=pl.col("raw_json").str.json_path_match("$.version"),
159
+ release_date=pl.col("raw_json")
160
+ .str.json_path_match("$.size_mb")
161
+ .str.to_decimal(),
162
+ )
163
+ return
164
+
165
+
166
+ @app.cell
167
+ def _(mo):
168
+ mo.md(
169
+ r"""
170
+ ## 📊 Dataset Overview
171
+
172
+ Now that we got our hands dirty, let's consider a somewhat wilder dataset for the subsequent sections: a dataframe of metadata about every single expression in your current Polars module.
173
+
174
+ At the risk of stating the obvious, in the previous section, when we typed `pl.col('raw_json').str.json_decode()`, we accessed the `json_decode` member of the `str` expression namespace through the `pl.col('raw_json')` expression *instance*. Under the hood, deep inside the Polars source code, there is a corresponding `def json_decode(...)` method with a carefully authored docstring explaining the purpose and signature of the member.
175
+
176
+ Since Python makes module introspection simple, we can easily enumerate all Polars expressions and organize their metadata in `expressions_df`, to be used for all the upcoming string manipulation examples.
177
+ """
178
+ )
179
+ return
180
+
181
+
182
+ @app.cell(hide_code=True)
183
+ def _(pl):
184
+ def list_members(expr, namespace) -> list[dict]:
185
+ """Iterates through the attributes of `expr` and returns their metadata"""
186
+ members = []
187
+ for attrname in expr.__dir__():
188
+ is_namespace = attrname in pl.Expr._accessors
189
+ is_private = attrname.startswith("_")
190
+ if is_namespace or is_private:
191
+ continue
192
+
193
+ attr = getattr(expr, attrname)
194
+ members.append(
195
+ {
196
+ "namespace": namespace,
197
+ "member": attrname,
198
+ "docstring": attr.__doc__,
199
+ }
200
+ )
201
+ return members
202
+
203
+
204
+ def list_expr_meta() -> list[dict]:
205
+ # Dummy expression instance to 'crawl'
206
+ expr = pl.lit("")
207
+ root_members = list_members(expr, "root")
208
+ namespaced_members: list[list[dict]] = [
209
+ list_members(getattr(expr, namespace), namespace)
210
+ for namespace in pl.Expr._accessors
211
+ ]
212
+ return sum(namespaced_members, root_members)
213
+
214
+
215
+ expressions_df = pl.from_dicts(list_expr_meta(), infer_schema_length=None).sort('namespace', 'member')
216
+ expressions_df
217
+ return expressions_df, list_expr_meta, list_members
218
+
219
+
220
+ @app.cell
221
+ def _(mo):
222
+ mo.md(r"""As the following visualization shows, `str` is one of the richest Polars expression namespaces with multiple dozens of functions in it.""")
223
+ return
224
+
225
+
226
+ @app.cell(hide_code=True)
227
+ def _(alt, expressions_df):
228
+ expressions_df.plot.bar(
229
+ x=alt.X("count(member):Q", title='Count of Expressions'),
230
+ y=alt.Y("namespace:N", title='Namespace').sort("-x"),
231
+ )
232
+ return
233
+
234
+
235
+ @app.cell
236
+ def _(mo):
237
+ mo.md(
238
+ r"""
239
+ ## 📏 Length Calculation
240
+
241
+ A common use case is to compute the length of a string. Most people associate string length exclusively with the number of characters the said string consists of; however, in certain scenarios it is useful to also know how much memory is required for storing, so how many bytes are required to represent the textual data.
242
+
243
+ The expressions [`len_chars`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.len_chars.html) and [`len_bytes`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.len_bytes.html) are here to help us with these calculations.
244
+
245
+ Below, we compute `docstring_len_chars` and `docstring_len_bytes` columns to see how many characters and bytes the documentation of each expression is made up of.
246
+ """
247
+ )
248
+ return
249
+
250
+
251
+ @app.cell
252
+ def _(expressions_df, pl):
253
+ docstring_length_df = expressions_df.select(
254
+ 'namespace',
255
+ 'member',
256
+ docstring_len_chars=pl.col("docstring").str.len_chars(),
257
+ docstring_len_bytes=pl.col("docstring").str.len_bytes(),
258
+ )
259
+ docstring_length_df
260
+ return (docstring_length_df,)
261
+
262
+
263
+ @app.cell
264
+ def _(mo):
265
+ mo.md(r"""As the dataframe preview above and the scatterplot below show, the docstring length measured in bytes is almost always bigger than the length expressed in characters. This is due to the fact that the docstrings include characters which require more than a single byte to represent, such as "╞" for displaying dataframe header and body separators.""")
266
+ return
267
+
268
+
269
+ @app.cell
270
+ def _(alt, docstring_length_df):
271
+ docstring_length_df.plot.point(
272
+ x=alt.X('docstring_len_chars', title='Docstring Length (Chars)'),
273
+ y=alt.Y('docstring_len_bytes', title='Docstring Length (Bytes)'),
274
+ tooltip=['namespace', 'member', 'docstring_len_chars', 'docstring_len_bytes'],
275
+ )
276
+ return
277
+
278
+
279
+ @app.cell
280
+ def _(mo):
281
+ mo.md(
282
+ r"""
283
+ ## 🔠 Case Conversion
284
+
285
+ Another frequent string transformation is lowercasing, uppercasing, and titlecasing. We can use [`to_lowercase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_lowercase.html), [`to_uppercase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_lowercase.html) and [`to_titlecase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_titlecase.html) for doing so.
286
+ """
287
+ )
288
+ return
289
+
290
+
291
+ @app.cell
292
+ def _(expressions_df, pl):
293
+ expressions_df.select(
294
+ member_lower=pl.col('member').str.to_lowercase(),
295
+ member_upper=pl.col('member').str.to_uppercase(),
296
+ member_title=pl.col('member').str.to_titlecase(),
297
+ )
298
+ return
299
+
300
+
301
+ @app.cell
302
+ def _(mo):
303
+ mo.md(
304
+ r"""
305
+ ## ➕ Padding
306
+
307
+ Sometimes we need to ensure that strings have a fixed-size character length. [`pad_start`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.pad_start.html) and [`pad_end`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.pad_end.html) can be used to fill the "front" or "back" of a string with a supplied character, while [`zfill`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.zfill.html) is a utility for padding the start of a string with `"0"` until it reaches a particular length. In other words, `zfill` is a more specific version of `pad_start`, where the `fill_char` parameter is explicitly set to `"0"`.
308
+
309
+ In the example below we take the unique Polars expression namespaces and pad them so that they have a uniform length which you can control via a slider.
310
+ """
311
+ )
312
+ return
313
+
314
+
315
+ @app.cell(hide_code=True)
316
+ def _(mo):
317
+ padding = mo.ui.slider(0, 16, step=1, value=8, label="Padding Size")
318
+ return (padding,)
319
+
320
+
321
+ @app.cell
322
+ def _(expressions_df, padding, pl):
323
+ padded_df = expressions_df.select("namespace").unique().select(
324
+ "namespace",
325
+ namespace_front_padded=pl.col("namespace").str.pad_start(padding.value, "_"),
326
+ namespace_back_padded=pl.col("namespace").str.pad_end(padding.value, "_"),
327
+ namespace_zfilled=pl.col("namespace").str.zfill(padding.value),
328
+ )
329
+ return (padded_df,)
330
+
331
+
332
+ @app.cell(hide_code=True)
333
+ def _(mo, padded_df, padding):
334
+ mo.vstack([
335
+ padding,
336
+ padded_df,
337
+ ])
338
+ return
339
+
340
+
341
+ @app.cell
342
+ def _(mo):
343
+ mo.md(
344
+ r"""
345
+ ## 🔄 Replacing
346
+
347
+ Let's say we want to convert from `snake_case` API member names to `kebab-case`, that is, we need to replace the underscore character with a hyphen. For operations like that, we can use [`replace`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace.html) and [`replace_all`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace_all.html).
348
+
349
+ As the example below demonstrates, `replace` stops after the first occurrence of the to-be-replaced pattern, while `replace_all` goes all the way through and changes all underscores to hyphens resulting in the `kebab-case` representation we were looking for.
350
+ """
351
+ )
352
+ return
353
+
354
+
355
+ @app.cell
356
+ def _(expressions_df, pl):
357
+ expressions_df.select(
358
+ "member",
359
+ member_kebab_case_partial=pl.col("member").str.replace("_", "-"),
360
+ member_kebab_case=pl.col("member").str.replace_all("_", "-"),
361
+ ).sort(pl.col("member").str.len_chars(), descending=True)
362
+ return
363
+
364
+
365
+ @app.cell
366
+ def _(mo):
367
+ mo.md(
368
+ r"""
369
+ A related expression is [`replace_many`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace_many.html), which accepts *many* pairs of to-be-matched patterns and corresponding replacements and uses the [Aho–Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) to carry out the operation with great performance.
370
+
371
+ In the example below we replace all instances of `"min"` with `"minimum"` and `"max"` with `"maximum"` using a single expression.
372
+ """
373
+ )
374
+ return
375
+
376
+
377
+ @app.cell
378
+ def _(expressions_df, pl):
379
+ expressions_df.select(
380
+ "member",
381
+ member_modified=pl.col("member").str.replace_many(
382
+ {
383
+ "min": "minimum",
384
+ "max": "maximum",
385
+ }
386
+ ),
387
+ )
388
+ return
389
+
390
+
391
+ @app.cell
392
+ def _(mo):
393
+ mo.md(
394
+ r"""
395
+ ## 🔍 Searching & Matching
396
+
397
+ A common need when working with strings is to determine whether their content satisfies some condition: whether it starts or ends with a particular substring or contains a certain pattern.
398
+
399
+ Let's suppose we want to determine whether a member of the Polars expression API is a "converter", such as `to_decimal`, identified by its `"to_"` prefix. We can use [`starts_with`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.starts_with.html) to perform this check.
400
+ """
401
+ )
402
+ return
403
+
404
+
405
+ @app.cell
406
+ def _(expressions_df, pl):
407
+ expressions_df.select(
408
+ "namespace",
409
+ "member",
410
+ is_converter=pl.col("member").str.starts_with("to_"),
411
+ ).sort(-pl.col("is_converter").cast(pl.Int8))
412
+ return
413
+
414
+
415
+ @app.cell
416
+ def _(mo):
417
+ mo.md(
418
+ r"""
419
+ Throughout this course as you have gained familiarity with the expression API you might have noticed that some members end with an underscore such as `or_`, since their "body" is a reserved Python keyword.
420
+
421
+ Let's use [`ends_with`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.ends_with.html) to find all the members which are named after such keywords.
422
+ """
423
+ )
424
+ return
425
+
426
+
427
+ @app.cell
428
+ def _(expressions_df, pl):
429
+ expressions_df.select(
430
+ "namespace",
431
+ "member",
432
+ is_escaped_keyword=pl.col("member").str.ends_with("_"),
433
+ ).sort(-pl.col("is_escaped_keyword").cast(pl.Int8))
434
+ return
435
+
436
+
437
+ @app.cell
438
+ def _(mo):
439
+ mo.md(
440
+ r"""
441
+ Now let's move on to analyzing the docstrings in a bit more detail. Based on their content we can determine whether a member is deprecated, accepts parameters, comes with examples, or references external URL(s) & related members.
442
+
443
+ As demonstrated below, we can compute all these boolean attributes using [`contains`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.contains.html) to check whether the docstring includes a particular substring.
444
+ """
445
+ )
446
+ return
447
+
448
+
449
+ @app.cell
450
+ def _(expressions_df, pl):
451
+ expressions_df.select(
452
+ 'namespace',
453
+ 'member',
454
+ is_deprecated=pl.col('docstring').str.contains('.. deprecated', literal=True),
455
+ has_parameters=pl.col('docstring').str.contains('Parameters'),
456
+ has_examples=pl.col('docstring').str.contains('Examples'),
457
+ has_related_members=pl.col('docstring').str.contains('See Also'),
458
+ has_url=pl.col('docstring').str.contains('https?://'),
459
+ )
460
+ return
461
+
462
+
463
+ @app.cell
464
+ def _(mo):
465
+ mo.md(r"""For scenarios where we want to combine multiple substrings to check for, we can use the [`contains`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.contains.html) expression to check for the presence of various patterns.""")
466
+ return
467
+
468
+
469
+ @app.cell
470
+ def _(expressions_df, pl):
471
+ expressions_df.select(
472
+ 'namespace',
473
+ 'member',
474
+ has_reference=pl.col('docstring').str.contains_any(['See Also', 'https://'])
475
+ )
476
+ return
477
+
478
+
479
+ @app.cell
480
+ def _(mo):
481
+ mo.md(
482
+ r"""
483
+ From the above analysis we could see that almost all the members come with code examples. It would be interesting to know how many variable assignments are going on within each of these examples, right? That's not as simple as checking for a pre-defined literal string containment though, because variables can have arbitrary names - any valid Python identifier is allowed. While the `contains` function supports checking for regular expressions instead of literal strings too, it would not suffice for this exercise because it only tells us whether there is at least a single occurrence of the sought pattern rather than telling us the exact number of matches.
484
+
485
+ Fortunately, we can take advantage of [`count_matches`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.count_matches.html) to achieve exactly what we want. We specify the regular expression `r'[a-zA-Z_][a-zA-Z0-9_]* = '` according to the [`regex` Rust crate](https://docs.rs/regex/latest/regex/) to match Python identifiers and we leave the rest to Polars.
486
+
487
+ In `count_matches(r'[a-zA-Z_][a-zA-Z0-9_]* = ')`:
488
+
489
+ - `[a-zA-Z_]` matches a letter or underscore (start of a Python identifier).
490
+ - `[a-zA-Z0-9_]*` matches zero or more letters, digits, or underscores.
491
+ - ` = ` matches a space, equals sign, and space (indicating assignment).
492
+
493
+ This finds variable assignments like `x = ` or `df_result = ` in docstrings.
494
+ """
495
+ )
496
+ return
497
+
498
+
499
+ @app.cell
500
+ def _(expressions_df, pl):
501
+ expressions_df.select(
502
+ 'namespace',
503
+ 'member',
504
+ variable_assignment_count=pl.col('docstring').str.count_matches(r'[a-zA-Z_][a-zA-Z0-9_]* = '),
505
+ )
506
+ return
507
+
508
+
509
+ @app.cell
510
+ def _(mo):
511
+ mo.md(r"""A related application example is to *find* the first index where a particular pattern is present, so that it can be used for downstream processing such as slicing. Below we use the [`find`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.find.html) expression to determine the index at which a code example starts in the docstring - identified by the Python shell substring `">>>"`.""")
512
+ return
513
+
514
+
515
+ @app.cell
516
+ def _(expressions_df, pl):
517
+ expressions_df.select(
518
+ 'namespace',
519
+ 'member',
520
+ code_example_start=pl.col('docstring').str.find('>>>'),
521
+ )
522
+ return
523
+
524
+
525
+ @app.cell
526
+ def _(mo):
527
+ mo.md(
528
+ r"""
529
+ ## ✂️ Slicing and Substrings
530
+
531
+ Sometimes we are only interested in a particular substring. We can use [`head`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.head.html), [`tail`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.tail.html) and [`slice`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.slice.html) to extract a substring from the start, end, or between arbitrary indices.
532
+ """
533
+ )
534
+ return
535
+
536
+
537
+ @app.cell
538
+ def _(mo):
539
+ slice = mo.ui.slider(1, 50, step=1, value=25, label="Slice Size")
540
+ return (slice,)
541
+
542
+
543
+ @app.cell
544
+ def _(expressions_df, pl, slice):
545
+ sliced_df = expressions_df.select(
546
+ # First 25 chars
547
+ docstring_head=pl.col("docstring").str.head(slice.value),
548
+ # 50 chars after the first 25 chars
549
+ docstring_slice=pl.col("docstring").str.slice(slice.value, 2*slice.value),
550
+ # Last 25 chars
551
+ docstring_tail=pl.col("docstring").str.tail(slice.value),
552
+ )
553
+ return (sliced_df,)
554
+
555
+
556
+ @app.cell
557
+ def _(mo, slice, sliced_df):
558
+ mo.vstack([
559
+ slice,
560
+ sliced_df,
561
+ ])
562
+ return
563
+
564
+
565
+ @app.cell
566
+ def _(mo):
567
+ mo.md(
568
+ r"""
569
+ ## ➗ Splitting
570
+
571
+ Certain strings follow a well-defined structure and we might be only interested in some parts of them. For example, when dealing with `snake_cased_expression` member names we might be curious to get only the first, second, or $n^{\text{th}}$ word before an underscore. We would need to *split* the string at a particular pattern for downstream processing.
572
+
573
+ The [`split`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.split.html), [`split_exact`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.split_exact.html) and [`splitn`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.splitn.html) expressions enable us to achieve this.
574
+
575
+ The primary difference between these string splitting utilities is that `split` produces a list of variadic length based on the number of resulting segments, `splitn` returns a struct with at least `0` and at most `n` fields while `split_exact` returns a struct of exactly `n` fields.
576
+ """
577
+ )
578
+ return
579
+
580
+
581
+ @app.cell
582
+ def _(expressions_df, pl):
583
+ expressions_df.select(
584
+ 'member',
585
+ member_name_parts=pl.col('member').str.split('_'),
586
+ member_name_parts_n=pl.col('member').str.splitn('_', n=2),
587
+ member_name_parts_exact=pl.col('member').str.split_exact('_', n=2),
588
+ )
589
+ return
590
+
591
+
592
+ @app.cell
593
+ def _(mo):
594
+ mo.md(r"""As a more practical example, we can use the `split` expression with some aggregation to count the number of times a particular word occurs in member names across all namespaces. This enables us to create a word cloud of the API members' constituents!""")
595
+ return
596
+
597
+
598
+ @app.cell(hide_code=True)
599
+ def _(mo, wordcloud, wordcloud_height, wordcloud_width):
600
+ mo.vstack([
601
+ wordcloud_width,
602
+ wordcloud_height,
603
+ wordcloud,
604
+ ])
605
+ return
606
+
607
+
608
+ @app.cell(hide_code=True)
609
+ def _(mo):
610
+ wordcloud_width = mo.ui.slider(0, 64, step=1, value=32, label="Word Cloud Width")
611
+ wordcloud_height = mo.ui.slider(0, 32, step=1, value=16, label="Word Cloud Height")
612
+ return wordcloud_height, wordcloud_width
613
+
614
+
615
+ @app.cell(hide_code=True)
616
+ def _(alt, expressions_df, pl, random, wordcloud_height, wordcloud_width):
617
+ wordcloud_df = (
618
+ expressions_df.select(pl.col("member").str.split("_"))
619
+ .explode("member")
620
+ .group_by("member")
621
+ .agg(pl.len())
622
+ # Generating random x and y coordinates to distribute the words in the 2D space
623
+ .with_columns(
624
+ x=pl.col("member").map_elements(
625
+ lambda e: random.randint(0, wordcloud_width.value),
626
+ return_dtype=pl.UInt8,
627
+ ),
628
+ y=pl.col("member").map_elements(
629
+ lambda e: random.randint(0, wordcloud_height.value),
630
+ return_dtype=pl.UInt8,
631
+ ),
632
+ )
633
+ )
634
+
635
+ wordcloud = alt.Chart(wordcloud_df).mark_text(baseline="middle").encode(
636
+ x=alt.X("x:O", axis=None),
637
+ y=alt.Y("y:O", axis=None),
638
+ text="member:N",
639
+ color=alt.Color("len:Q", scale=alt.Scale(scheme="bluepurple")),
640
+ size=alt.Size("len:Q", legend=None),
641
+ tooltip=["member", "len"],
642
+ ).configure_view(strokeWidth=0)
643
+ return wordcloud, wordcloud_df
644
+
645
+
646
+ @app.cell
647
+ def _(mo):
648
+ mo.md(
649
+ r"""
650
+ ## 🔗 Concatenation & Joining
651
+
652
+ Often we would like to create longer strings from strings we already have. We might want to create a formatted, sentence-like string or join multiple existing strings in our dataframe into a single one.
653
+
654
+ The top-level [`concat_str`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.concat_str.html) expression enables us to combine strings *horizontally* in a dataframe. As the example below shows, we can take the `member` and `namespace` column of each row and construct a `description` column in which each row will correspond to the value ``f"- Expression `{member}` belongs to namespace `{namespace}`"``.
655
+ """
656
+ )
657
+ return
658
+
659
+
660
+ @app.cell
661
+ def _(expressions_df, pl):
662
+ descriptions_df = expressions_df.sample(5).select(
663
+ description=pl.concat_str(
664
+ [
665
+ pl.lit("- Expression "),
666
+ pl.lit("`"),
667
+ "member",
668
+ pl.lit("`"),
669
+ pl.lit(" belongs to namespace "),
670
+ pl.lit("`"),
671
+ "namespace",
672
+ pl.lit("`"),
673
+ ],
674
+ )
675
+ )
676
+ descriptions_df
677
+ return (descriptions_df,)
678
+
679
+
680
+ @app.cell
681
+ def _(mo):
682
+ mo.md(
683
+ r"""
684
+ Now that we have constructed these bullet points through *horizontal* concatenation of strings, we can perform a *vertical* one so that we end up with a single string in which we have a bullet point on each line.
685
+
686
+ We will use the [`join`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.join.html) expression to do so.
687
+ """
688
+ )
689
+ return
690
+
691
+
692
+ @app.cell
693
+ def _(descriptions_df, pl):
694
+ descriptions_df.select(pl.col('description').str.join('\n'))
695
+ return
696
+
697
+
698
+ @app.cell(hide_code=True)
699
+ def _(descriptions_df, mo, pl):
700
+ mo.md(f"""In fact, since the string we constructed dynamically is valid markdown, we can display it dynamically using Marimo's `mo.md` utility!
701
+
702
+ ---
703
+
704
+ {descriptions_df.select(pl.col('description').str.join('\n')).to_numpy().squeeze().tolist()}
705
+ """)
706
+ return
707
+
708
+
709
+ @app.cell
710
+ def _(mo):
711
+ mo.md(
712
+ r"""
713
+ ## 🔍 Pattern-based Extraction
714
+
715
+ In the vast majority of the cases, when dealing with unstructured text data, all we really want is to extract something structured from it. A common use case is to extract URLs from text to get a better understanding of related content.
716
+
717
+ In the example below that's exactly what we do. We scan the `docstring` of each API member and extract URLs from them using [`extract`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract.html) and [`extract_all`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract_all.html) using a simple regular expression to match http and https URLs.
718
+
719
+ Note that `extract` stops after a first match and returns a scalar result (or `null` if there was no match) while `extract_all` returns a - potentially empty - list of matches.
720
+ """
721
+ )
722
+ return
723
+
724
+
725
+ @app.cell
726
+ def _(expressions_df, pl):
727
+ url_pattern = r'(https?://[^\s>]+)'
728
+ expressions_df.select(
729
+ 'namespace',
730
+ 'member',
731
+ url_match=pl.col('docstring').str.extract(url_pattern),
732
+ url_matches=pl.col('docstring').str.extract_all(url_pattern),
733
+ ).filter(pl.col('url_match').is_not_null())
734
+ return (url_pattern,)
735
+
736
+
737
+ @app.cell
738
+ def _(mo):
739
+ mo.md(
740
+ r"""
741
+ Note that in each `docstring` where a code example involving dataframes is present, we will see an output such as "shape: (5, 2)" indicating the number of rows and columns of the dataframe produced by the sample code. Let's say we would like to *capture* this information in a structured way.
742
+
743
+ [`extract_groups`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract_groups.html) is a really powerful expression allowing us to achieve exactly that.
744
+
745
+ Below we define the regular expression `r"shape:\s*\((?<height>\S+),\s*(?<width>\S+)\)"` with two capture groups, named `height` and `width` and pass it as the parameter of `extract_groups`. After execution, for each `docstring`, we end up with fully structured data we can further process downstream!
746
+ """
747
+ )
748
+ return
749
+
750
+
751
+ @app.cell
752
+ def _(expressions_df, pl):
753
+ expressions_df.select(
754
+ 'namespace',
755
+ 'member',
756
+ example_df_shape=pl.col('docstring').str.extract_groups(r"shape:\s*\((?<height>\S+),\s*(?<width>\S+)\)"),
757
+ )
758
+ return
759
+
760
+
761
+ @app.cell
762
+ def _(mo):
763
+ mo.md(
764
+ r"""
765
+ ## 🧹 Stripping
766
+
767
+ Strings might require some cleaning before further processing, such as the removal of some characters from the beginning or end of the text. [`strip_chars_start`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars_start.html), [`strip_chars_end`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars_end.html) and [`strip_chars`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars.html) are here to facilitate this.
768
+
769
+ All we need to do is to specify a set of characters we would like to get rid of and Polars handles the rest for us.
770
+ """
771
+ )
772
+ return
773
+
774
+
775
+ @app.cell
776
+ def _(expressions_df, pl):
777
+ expressions_df.select(
778
+ "member",
779
+ member_front_stripped=pl.col("member").str.strip_chars_start("a"),
780
+ member_back_stripped=pl.col("member").str.strip_chars_end("n"),
781
+ member_fully_stripped=pl.col("member").str.strip_chars("na"),
782
+ )
783
+ return
784
+
785
+
786
+ @app.cell
787
+ def _(mo):
788
+ mo.md(
789
+ r"""
790
+ Note that when using the above expressions, the specified characters do not need to form a sequence; they are handled as a set. However, in certain use cases we only want to strip complete substrings, so we would need our input to be strictly treated as a sequence rather than as a set.
791
+
792
+ That's exactly the rationale behind [`strip_prefix`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_prefix.html) and [`strip_suffix`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_suffix.html).
793
+
794
+ Below we use these to remove the `"to_"` prefixes and `"_with"` suffixes from each member name.
795
+ """
796
+ )
797
+ return
798
+
799
+
800
+ @app.cell
801
+ def _(expressions_df, pl):
802
+ expressions_df.select(
803
+ "member",
804
+ member_prefix_stripped=pl.col("member").str.strip_prefix("to_"),
805
+ member_suffix_stripped=pl.col("member").str.strip_suffix("_with"),
806
+ ).slice(20)
807
+ return
808
+
809
+
810
+ @app.cell
811
+ def _(mo):
812
+ mo.md(
813
+ r"""
814
+ ## 🔑 Encoding & Decoding
815
+
816
+ Should you find yourself in the need of encoding your strings into [base64](https://en.wikipedia.org/wiki/Base64) or [hexadecimal](https://en.wikipedia.org/wiki/Hexadecimal) format, then Polars has your back with its [`encode`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.encode.html) expression.
817
+ """
818
+ )
819
+ return
820
+
821
+
822
+ @app.cell
823
+ def _(expressions_df, pl):
824
+ encoded_df = expressions_df.select(
825
+ "member",
826
+ member_base64=pl.col('member').str.encode('base64'),
827
+ member_hex=pl.col('member').str.encode('hex'),
828
+ )
829
+ encoded_df
830
+ return (encoded_df,)
831
+
832
+
833
+ @app.cell
834
+ def _(mo):
835
+ mo.md(r"""And of course, you can convert back into a human-readable representation using the [`decode`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.decode.html) expression.""")
836
+ return
837
+
838
+
839
+ @app.cell
840
+ def _(encoded_df, pl):
841
+ encoded_df.with_columns(
842
+ member_base64_decoded=pl.col('member_base64').str.decode('base64').cast(pl.String),
843
+ member_hex_decoded=pl.col('member_hex').str.decode('hex').cast(pl.String),
844
+ )
845
+ return
846
+
847
+
848
+ @app.cell
849
+ def _(mo):
850
+ mo.md(
851
+ r"""
852
+ ## 🚀 Application: Dynamic Execution of Polars Examples
853
+
854
+ Now that we are familiar with string expressions, we can combine them with other Polars operations to build a fully interactive playground where code examples of Polars expressions can be explored.
855
+
856
+ We make use of string expressions to extract the raw Python source code of examples from the docstrings and we leverage the interactive Marimo environment to enable the selection of expressions via a searchable dropdown and a fully functional code editor whose output is rendered with Marimo's rich display utilities.
857
+
858
+ In other words, we will use Polars to execute Polars. ❄️ How cool is that?
859
+
860
+ ---
861
+ """
862
+ )
863
+ return
864
+
865
+
866
+ @app.cell(hide_code=True)
867
+ def _(
868
+ example_editor,
869
+ execution_result,
870
+ expression,
871
+ expression_description,
872
+ expression_docs_link,
873
+ mo,
874
+ ):
875
+ mo.vstack(
876
+ [
877
+ mo.md(f'### {expression.value}'),
878
+ expression,
879
+ mo.hstack([expression_description, expression_docs_link]),
880
+ example_editor,
881
+ execution_result,
882
+ ]
883
+ )
884
+ return
885
+
886
+
887
+ @app.cell(hide_code=True)
888
+ def _(mo, selected_expression_record):
889
+ expression_description = mo.md(selected_expression_record["description"])
890
+ expression_docs_link = mo.md(
891
+ f"🐻‍❄️ [Official Docs](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.{selected_expression_record['expr']}.html)"
892
+ )
893
+ return expression_description, expression_docs_link
894
+
895
+
896
+ @app.cell(hide_code=True)
897
+ def _(example_editor, execute_code):
898
+ execution_result = execute_code(example_editor.value)
899
+ return (execution_result,)
900
+
901
+
902
+ @app.cell(hide_code=True)
903
+ def _(code_df, mo):
904
+ expression = mo.ui.dropdown(code_df.get_column('expr'), value='arr.all', searchable=True)
905
+ return (expression,)
906
+
907
+
908
+ @app.cell(hide_code=True)
909
+ def _(code_df, expression):
910
+ selected_expression_record = code_df.filter(expr=expression.value).to_dicts()[0]
911
+ return (selected_expression_record,)
912
+
913
+
914
+ @app.cell(hide_code=True)
915
+ def _(mo, selected_expression_record):
916
+ example_editor = mo.ui.code_editor(value=selected_expression_record["code"])
917
+ return (example_editor,)
918
+
919
+
920
+ @app.cell(hide_code=True)
921
+ def _(expressions_df, pl):
922
+ code_df = (
923
+ expressions_df.select(
924
+ expr=pl.when(pl.col("namespace") == "root")
925
+ .then("member")
926
+ .otherwise(pl.concat_str(["namespace", "member"], separator=".")),
927
+ description=pl.col("docstring")
928
+ .str.split("\n\n")
929
+ .list.get(0)
930
+ .str.slice(9),
931
+ docstring_lines=pl.col("docstring").str.split("\n"),
932
+ )
933
+ .with_row_index()
934
+ .explode("docstring_lines")
935
+ .rename({"docstring_lines": "docstring_line"})
936
+ .with_columns(pl.col("docstring_line").str.strip_chars(" "))
937
+ .filter(pl.col("docstring_line").str.contains_any([">>> ", "... "]))
938
+ .with_columns(pl.col("docstring_line").str.slice(4))
939
+ .group_by(pl.exclude("docstring_line"), maintain_order=True)
940
+ .agg(code=pl.col("docstring_line").str.join("\n"))
941
+ .drop("index")
942
+ )
943
+ return (code_df,)
944
+
945
+
946
+ @app.cell(hide_code=True)
947
+ def _():
948
+ def execute_code(code: str):
949
+ import ast
950
+
951
+ # Create a new local namespace for execution
952
+ local_namespace = {}
953
+
954
+ # Parse the code into an AST to identify the last expression
955
+ parsed_code = ast.parse(code)
956
+
957
+ # Check if there's at least one statement
958
+ if not parsed_code.body:
959
+ return None
960
+
961
+ # If the last statement is an expression, we'll need to get its value
962
+ last_is_expr = isinstance(parsed_code.body[-1], ast.Expr)
963
+
964
+ if last_is_expr:
965
+ # Split the code: everything except the last statement, and the last statement
966
+ last_expr = ast.Expression(parsed_code.body[-1].value)
967
+
968
+ # Remove the last statement from the parsed code
969
+ parsed_code.body = parsed_code.body[:-1]
970
+
971
+ # Execute everything except the last statement
972
+ if parsed_code.body:
973
+ exec(
974
+ compile(parsed_code, "<string>", "exec"),
975
+ globals(),
976
+ local_namespace,
977
+ )
978
+
979
+ # Execute the last statement and get its value
980
+ result = eval(
981
+ compile(last_expr, "<string>", "eval"), globals(), local_namespace
982
+ )
983
+ return result
984
+ else:
985
+ # If the last statement is not an expression (e.g., an assignment),
986
+ # execute the entire code and return None
987
+ exec(code, globals(), local_namespace)
988
+ return None
989
+ return (execute_code,)
990
+
991
+
992
+ @app.cell(hide_code=True)
993
+ def _():
994
+ import polars as pl
995
+ import marimo as mo
996
+ import altair as alt
997
+ import random
998
+
999
+ random.seed(42)
1000
+ return alt, mo, pl, random
1001
+
1002
+
1003
+ if __name__ == "__main__":
1004
+ app.run()