{ "cells": [ { "cell_type": "markdown", "id": "4b4adc2a-bf0c-4ace-87be-dbaf90be0125", "metadata": {}, "source": [ "# Pre-processing" ] }, { "cell_type": "code", "execution_count": 1, "id": "f7e6298c-d886-432a-a1b7-c3fee914c24f", "metadata": {}, "outputs": [], "source": [ "import ibis\n", "from ibis import _\n", "\n", "conn = ibis.duckdb.connect(\"tmp\", extensions=[\"spatial\"])\n", "ca_parquet = \"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\"\n", "# or use local copy:\n", "ca_parquet = \"/home/rstudio/source.coop/cboettig/ca30x30/ca_areas.parquet\"\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "a0cb34b1-8d70-49bf-80c6-244ecc8ddf84", "metadata": {}, "outputs": [], "source": [ "buffer = -2\n", "\n", "tbl = (\n", " conn.read_parquet(ca_parquet)\n", " .cast({\"SHAPE\": \"geometry\"})\n", " .rename(geom = \"SHAPE\")\n", "# .filter(_.UNIT_NAME == \"Angeles National Forest\")\n", " .filter(_.reGAP < 3) \n", ")\n", "tbl_2023 = tbl.filter(_.Release_Year == 2023).mutate(geom=_.geom.buffer(buffer))\n", "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n", "intersects = tbl_2024.anti_join(tbl_2023, _.geom.intersects(tbl_2023.geom))\n", "\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "a0b75637-e015-4be4-86e1-c9757ac43d0f", "metadata": {}, "outputs": [], "source": [ "## Testing, run only on subset data\n", "if False:\n", " gdf = intersects.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n", " gdf_2023 = tbl_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n", " gdf_2024 = tbl_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n", " # gdf = ca2024\n", " established = {'property': 'established',\n", " 'type': 'categorical',\n", " 'stops': [\n", " [2023, \"#26542C80\"], \n", " [2024, \"#F3AB3D80\"]]\n", " }\n", " inter = {\"fill-color\": \"#F3AB3D\"}\n", " p2024 = {\"fill-color\": \"#26542C\"}\n", " p2023 = {\"fill-color\": \"#8B0A1A\"}\n", " \n", " m = leafmap.Map(style=\"positron\")\n", " m.add_gdf(gdf_2024,layer_type=\"fill\", name = \"2024\", paint = p2024)\n", " m.add_gdf(gdf_2023,layer_type=\"fill\", name = \"2023\", paint = p2023)\n", " m.add_gdf(gdf,layer_type=\"fill\", name = \"intersects\", paint = inter)\n", " \n", " m.add_layer_control()\n", " m" ] }, { "cell_type": "code", "execution_count": 4, "id": "275c171a-f82f-4ee8-991c-1e34eb83a33d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f1616caa5ca54678a00caa974721de2b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 55min 28s, sys: 2.94 s, total: 55min 31s\n", "Wall time: 22min 6s\n" ] } ], "source": [ "%%time\n", "\n", "new2024 = intersects.select(\"OBJECTID\").mutate(established = 2024)\n", "\n", "ca = (conn\n", " .read_parquet(ca_parquet)\n", " .cast({\"SHAPE\": \"geometry\"})\n", " .mutate(area = _.SHAPE.area())\n", " .filter(_.Release_Year == 2024)\n", " .filter(_.reGAP < 3)\n", " .left_join(new2024, \"OBJECTID\")\n", " .mutate(established=_.established.fill_null(2023))\n", " .mutate(geom = _.SHAPE.convert(\"epsg:3310\",\"epsg:4326\"))\n", " .rename(name = \"cpad_PARK_NAME\", access_type = \"cpad_ACCESS_TYP\", manager = \"cpad_MNG_AGENCY\",\n", " manager_type = \"cpad_MNG_AG_LEV\", id = \"OBJECTID\", type = \"TYPE\")\n", " .select(_.established, _.reGAP, _.name, _.access_type, _.manager, _.manager_type,\n", " _.Easement, _.Acres, _.id, _.type, _.geom)\n", " )\n", "ca2024 = ca.execute()\n", "\n", "ca2024.to_parquet(\"ca2024.parquet\")" ] }, { "cell_type": "code", "execution_count": 1, "id": "cfac7aa4-e418-4d7c-91e0-04ff8eae804c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n", "Token is valid (permission: write).\n", "Your token has been saved to /home/jovyan/.cache/huggingface/token\n", "Login successful\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "9e680dc3991c4c9b808447f04d056f53", "version_major": 2, "version_minor": 0 }, "text/plain": [ "ca2024.parquet: 0%| | 0.00/137M [00:00\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
managermanager_typearea
0California Department of Fish and WildlifeState54853.556568
1California Department of Parks and RecreationState21439.451269
2California Tahoe ConservancyState6119.753048
3California Department of Water ResourcesState4033.217739
4California State University SonomaState3842.054169
5University of CaliforniaState2050.549176
6California Department of Forestry and Fire Pro...State1212.712394
7Coachella Valley Mountains ConservancyState167.224090
8California State Lands CommissionState113.344073
9California State Coastal ConservancyState97.314705
10Other StateState55.542241
11California Department of TransportationState1.957490
12San Joaquin River ConservancyState1.531470
13California State UniversityState0.021589
\n", "" ], "text/plain": [ " manager manager_type \\\n", "0 California Department of Fish and Wildlife State \n", "1 California Department of Parks and Recreation State \n", "2 California Tahoe Conservancy State \n", "3 California Department of Water Resources State \n", "4 California State University Sonoma State \n", "5 University of California State \n", "6 California Department of Forestry and Fire Pro... State \n", "7 Coachella Valley Mountains Conservancy State \n", "8 California State Lands Commission State \n", "9 California State Coastal Conservancy State \n", "10 Other State State \n", "11 California Department of Transportation State \n", "12 San Joaquin River Conservancy State \n", "13 California State University State \n", "\n", " area \n", "0 54853.556568 \n", "1 21439.451269 \n", "2 6119.753048 \n", "3 4033.217739 \n", "4 3842.054169 \n", "5 2050.549176 \n", "6 1212.712394 \n", "7 167.224090 \n", "8 113.344073 \n", "9 97.314705 \n", "10 55.542241 \n", "11 1.957490 \n", "12 1.531470 \n", "13 0.021589 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# compute some summary tables:\n", "\n", "(ca2024\n", " .filter(_.established == 2024)\n", " .filter(_.manager_type == \"State\")\n", " .group_by(_.manager, _.manager_type)\n", " .agg(area = _.Acres.sum())\n", " .order_by(_.area.desc())\n", " .execute()\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "c62854f6-1456-4207-8c69-53af17970102", "metadata": {}, "outputs": [], "source": [ "\n", "gdf = ca2024.execute()\n", "established = {'property': 'established',\n", " 'type': 'categorical',\n", " 'stops': [\n", " [2023, \"#26542C80\"], \n", " [2024, \"#F3AB3D80\"]]}\n", "paint = {\"fill-color\": established}\n", "\n", "\n", "m = leafmap.Map(style=\"positron\")\n", "m.add_gdf(gdf,layer_type=\"fill\", name = \"intersects\", paint = paint)\n", "\n", "m.add_layer_control()\n", "m.to_html(\"ca2024.html\")\n", "m" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" } }, "nbformat": 4, "nbformat_minor": 5 }