first push

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9-slim
+EXPOSE 8501
+WORKDIR /app
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    software-properties-common \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+COPY . /app
+RUN pip3 install -r requirements.txt
+ENTRYPOINT ["streamlit", "run", "streamlit.py", "--server.port=8501", "--server.address=0.0.0.0"]

Metadata_Country_API_Download_DS2_en_csv_v2_5657328.csv

@@ -0,0 +1,9 @@
+"Country Code","Region","IncomeGroup","SpecialNotes","TableName",
+"COD","Sub-Saharan Africa","Low income","The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1999-2004. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.","Congo, Dem. Rep.",
+"GHA","Sub-Saharan Africa","Lower middle income","The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 1974-1987. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.","Ghana",
+"KEN","Sub-Saharan Africa","Lower middle income","Fiscal year end: June 30; reporting period for national accounts data: CY.","Kenya",
+"NGA","Sub-Saharan Africa","Lower middle income","The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to 1970-2020. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.","Nigeria",
+"ZAF","Sub-Saharan Africa","Upper middle income","Fiscal year end: March 31; reporting period for national accounts data: CY.","South Africa",
+"ZWE","Sub-Saharan Africa","Lower middle income","National Accounts data are reported in Zimbabwean Dollar (ZWL). Before 2017, one ZWL is set to be equal to one USD.
+
+The World Bank systematically assesses the appropriateness of official exchange rates as conversion factors. In this country, multiple or dual exchange rate activity exists and must be accounted for appropriately in underlying statistics. An alternative estimate (“alternative conversion factor” - PA.NUS.ATLS) is thus calculated as a weighted average of the different exchange rates in use in the country. Doing so better reflects economic reality and leads to more accurate cross-country comparisons and country classifications by income level. For this country, this applies to the period 2017-2022. Alternative conversion factors are used in the Atlas methodology and elsewhere in World Development Indicators as single-year conversion factors.","Zimbabwe",

README.md

@@ -1,11 +1,107 @@
 ---
-title: Docker Streamlit
-emoji: 👁
-colorFrom: purple
-colorTo: indigo
+title: Streamlit Docker
+emoji: 🐨
+colorFrom: indigo
+colorTo: red
 sdk: docker
 pinned: false
 license: apache-2.0
+app_port: 8501
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+## Introduction to Data Science with Python
+
+## Overview
+
+Location: Accra, Ghana When: July 31 and August 1, 2023
+
+This material focuses on [Polars](https://pola-rs.github.io/polars-book/user-guide/), [Parquet files](https://parquet.apache.org/docs/), [Plotly Express](https://plotly.com/python/plotly-express/), and [Streamlit](https://streamlit.io/) to introduce the data science process.
+
+## Installing the tools
+
+We will need [Visual Studio Code](https://code.visualstudio.com/download) and [Python](https://www.python.org/downloads/) installed for this short course. Each tool has additional packages/extensions that we will need to install as well.
+
+
+### Visual Studio Code Extensions
+
+You can use [Managing Extensions in Visual Studio Code](https://code.visualstudio.com/docs/editor/extension-marketplace) to learn about how to install extensions. We will use [Python - Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=ms-python.python) extension heavily. [Managing Extensions in Visual Studio Code](https://code.visualstudio.com/docs/editor/extension-marketplace) provides more background on extensions if needed.
+
+#### VS Code Interactive Python Window
+
+An open-source project called [Jupyter](http://jupyter-notebook.readthedocs.io/en/latest/) is the standard method for interactive Python use for data science or scientific computing. However, there are [some issues](https://towardsdatascience.com/5-reasons-why-jupyter-notebooks-suck-4dc201e27086) with its use in a development environment. VS Code provides a way for us to have the best of Python and Jupyter Notebooks with their [Python Interactive Window](https://code.visualstudio.com/docs/python/jupyter-support-py).
+
+VS Code is fairly intelligent in responding to your needs. If you open a `.py` file it should ask pop up a window asking you if you would like prepare your Python experience. You will need to install the [jupyter python package](https://jupyter.readthedocs.io/en/latest/install.html). If VS Code doesn't install it it, you can use `pip` or `pip3` for the interactive Python window to work.
+
+Using the VS Code functionality, you will work with a standard `.py` file instead of the `.ipynb` extension typically used with jupyter notebooks. The Python extension in VS Code will recognize `# %%` as a cell or chunk of python code and add notebook options to ‘Run Cell’ as well as other actions. You can see the code example bellow with the image of the view in VS Code as an example. [Microsoft’s documentation](https://code.visualstudio.com/docs/python/jupyter-support-py) goes into more detail (https://code.visualstudio.com/docs/python/jupyter-support-py).
+
+To make the interactive window use more functional you can `ctrl + ,` or `cmd + ,` on a mac to open the settings. From there you can search **‘Send Selection to Interactive Window’** and make sure the box is checked. Now you will be able to use `shift + return` to send a selected chunk of code or an entire cell.
+
+```python
+# %%
+msg = "Hello World"
+print(msg)
+
+# %%
+msg = "Hello again"
+print(msg)
+```
+
+![img](img/vscode-code-cells-01.png)
+
+### Python Packages
+
+#### `pip` overview
+
+*The standard command* - `pip install polars[all] plotly streamlit` is executed in your Terminal, Command Window, or by using the `New Terminal` under `Terminal` in VS Code. If you are using a Mac you most likely will use `pip3 install polars[all] plotly streamlit`. In your interactive Python environment in VS Code (Jupyter server) you can run `!pip install polars[all] plotly streamlit` as explained [here](https://jakevdp.github.io/blog/2017/12/05/installing-python-packages-from-jupyter/#How-to-use-Pip-from-the-Jupyter-Notebook). Finally, you could use the following Python code snippet.
+
+The two commands that can be used in the interactive python window in VS Code to install packages.
+
+```python
+!pip install polars[all] plotly streamlit
+```
+
+or
+
+```python
+import sys
+!{sys.executable} -m pip install polars[all] plotly streamlit
+```
+
+#### `pip` commands
+
+- `pip install polars[all] plotly streamlit` should install all needed packages.
+
+You could install them individually using the following commands.
+
+- `pip install polars[all]` for [Polars](https://pola-rs.github.io/polars-book/user-guide/installation/)
+- `pip install streamlit` for [Streamlit](https://docs.streamlit.io/library/get-started/main-concepts)
+- `pip install plotly` for [plotly in Python](https://plotly.com/python/getting-started/)
+
+## Repo Navigation
+
+### `guides` folder
+
+The `guides` folder will allow us to explore these packages if the internet connection is down during our course.
+
+- PDF Files: The pdf files should have most of the commands we will need during the course. The `polars_website.pdf` is a full pdf build of their website guide as of July 2023.
+- `streamlit_md` folder: This folder has the markdown files used to build their website guide. It is a little harder to navigate.
+- `polars_site` folder: This folder has the fully built website for the polars package as of July 2023. From your OS file explorer open the `index.html` file to see the full site.
+
+### `data` folder
+
+This folder has the data we will be using for the short course. Read more about [the data folder](https://file+.vscode-resource.vscode-cdn.net/Users/hathawayj/git/hathawayj/ghana_datascience/data/readme.md).
+
+### Scripts folder
+
+The scripts folder has the starting scripts for each of the activities we will complete during the short course.
+
+### Markdown links
+
+- [plotly.md](https://file+.vscode-resource.vscode-cdn.net/Users/hathawayj/git/hathawayj/ghana_datascience/plotly.md): links to the primary functions we will use as we create charts with Plotly Express
+- [polars.md](https://file+.vscode-resource.vscode-cdn.net/Users/hathawayj/git/hathawayj/ghana_datascience/polars.md): links to the key methods we will leverage for data import and munging.
+- [streamlit.md](https://file+.vscode-resource.vscode-cdn.net/Users/hathawayj/git/hathawayj/ghana_datascience/streamlit.md): links to the dashboard functions and concepts we will use with Streamlit
+
+## Slides
+
+The [HTML Slides](https://hathawayj.github.io/ghana_datascience/) and [pdf slides](https://github.com/hathawayj/ghana_datascience/blob/slides/slides.pdf)

docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.9'
+services:
+  streamlit:
+    build:
+      dockerfile: Dockerfile
+      context: .
+    container_name: streamlit-example
+    cpus: 2
+    mem_limit: 2048m
+    ports:
+      - "8501:8501"
+    volumes:
+      - ".:/app:rw"

remark_slides.md

@@ -0,0 +1,15 @@
+# GitHub Pages Slideshow with [Remark](https://github.com/gnab/remark)
+
+This template is made from [Remark](https://github.com/gnab/remark), an open source tool to help create and display slideshows from markdown. For questions, see [Remark's documentation](https://github.com/gnab/remark). I have added a Github action to convert the slides to a pdf in the `slides` branch.
+
+The most important things to know are:
+- Enable GitHub Pages from `master` for the slides to work
+- Once enabled, the slides will be visible at `https://USERNAME.github.io/REPOSITORY-NAME/#1`, like https://brianamarie.github.io/slideshow-on-pages/#1
+- Edit the `index.html` file to edit the slides
+- Slides are separated by `----`
+- Presenter notes after `???` within one slide
+- Toggle presenter notes during presentation with `P`
+- Read the full guide to [remark markdown](https://github.com/gnab/remark/wiki)
+- Press `C` to clone a display; then press `P` to switch to presenter mode. Open help menu with `h`
+
+Fork this repository to get started!

requirements.txt

@@ -0,0 +1,8 @@
+altair
+polars
+pandas
+streamlit
+scikit-learn
+numpy
+plotly
+lets-plot

scripts_build/01_read.py

@@ -0,0 +1,49 @@
+# %%
+import polars as pl
+# Notice that the world health leaves missing as blanks in the csv. We need to explain that blanks aren't strings but missing values.
+
+dat = pl.read_csv("../data/API_Download_DS2_en_csv_v2_5657328.csv", skip_rows=4, null_values = "")
+# We don't like the World Banks wide format.  Let's clean it upt to long format.
+dat_long = dat.melt(id_vars=["Country Name", "Country Code", "Indicator Name", "Indicator Code"])
+# no we need to fix the year column and give it a better name.
+# we could have fixed the name as an argument in `.melt()` as well.
+# https://docs.pola.rs/user-guide/expressions/casting/#overflow
+dat_long = dat_long\
+  .with_columns(
+    pl.col("variable").cast(pl.Int64, strict=False).alias("variable"),
+    pl.col("value").cast(pl.Float32, strict=False).alias("value"))\
+  .rename({"variable":"year"})\
+  .filter(pl.col("value").is_not_null())
+
+
+# %%
+# Can we split out the information in the indicator Code
+indicator_columns = dat_long\
+  .select(
+    pl.col("Indicator Code"),
+    pl.col("Indicator Code")\
+    # split string: example VC.IDP.TOCV into list object ["VC", "IDP", "TOCV"]
+        .str.split_exact(".", 6).alias("split")).unnest("split")\
+  .unique()
+      
+
+# What should we call these columns?
+# https://datahelpdesk.worldbank.org/knowledgebase/articles/201175-how-does-the-world-bank-code-its-indicators
+new_names = {"field_0":"topic", "field_1":"general_subj", "field_2":"specific_subj",
+        "field_3":"ext_1", "field_4":"ext_2", "field_5":"ext_3", "field_6":"ext_4"}
+indicator_columns = indicator_columns.rename(new_names)
+
+# %%
+# now we need to finalize our munge and write our data
+dat_final = dat_long.join(indicator_columns, how="left", on="Indicator Code")
+# Now I want to reorder the columns
+names = dat_final.columns
+new_order = [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3, 0]
+name_order = [names[i] for i in new_order]
+dat_final = dat_final.select(name_order)
+
+# %%
+# write data
+dat_final.write_parquet("../data/dat_munged.parquet")
+
+

scripts_build/02_explore.py

@@ -0,0 +1,82 @@
+# %%
+import polars as pl
+import plotly.express as px
+import plotly.io as pio
+pio.templates.default = "simple_white"
+
+from lets_plot import *
+LetsPlot.setup_html()
+# ["plotly", "plotly_white", "plotly_dark", "ggplot2", "seaborn", "simple_white", "none"]
+
+dat = pl.read_parquet("../data/dat_munged.parquet")
+info = pl.read_csv("../data/Metadata_Indicator_API_Download_DS2_en_csv_v2_5657328.csv").rename({"INDICATOR_CODE":"Indicator Code", "INDICATOR_NAME":"Indicator Name"})
+
+# %%
+dat_vars = dat\
+    .group_by("Indicator Name", "Indicator Code", "Country Code").len()\
+    .pivot(values="len", index=["Indicator Name", "Indicator Code"], columns="Country Code", aggregate_function="first")\
+    .with_columns((pl.col("COD").fill_null(0) + pl.col("GHA").fill_null(0) +
+                   pl.col("KEN").fill_null(0) + pl.col("NGA").fill_null(0) +
+                   pl.col("ZAF").fill_null(0) + pl.col("ZWE").fill_null(0)).alias("total"))\
+    .sort(pl.col("total"), descending=True)\
+    .filter(pl.col("total") > 25)
+
+dat_vars.write_parquet("../data/dat_vars.parquet")
+
+# %%
+ggplot(
+    dat.filter(pl.col("Indicator Code").is_in(["NY.GDP.PCAP.PP.KD"])),
+    aes(x="year", y="value", color="Country Code")) +\
+  geom_line() +\
+  geom_point(size=1.2)
+
+# %%
+# Access to fuels
+# Access to internet
+# GDP
+
+drop_country =  ["ZAF"]
+indicator_code = "NY.GDP.PCAP.PP.KD"
+title_text = dat_vars\
+  .filter(pl.col("Indicator Code") == indicator_code)\
+  .select("Indicator Name")\
+  .to_series()[0]
+subtitle_text = info\
+  .filter(pl.col("Indicator Code") == indicator_code)\
+  .select("SOURCE_NOTE")\
+  .to_series()[0]
+subtitle_text = subtitle_text[1:100] + "..."
+chart_title = title_text + "<br><sup>" + subtitle_text + "</sup>"
+y_axis_title = chart_title[chart_title.find("(")+1:chart_title.find(")")]
+chart_dat = dat\
+  .filter(
+    (pl.col("Indicator Code").is_in([indicator_code])) &
+    (~pl.col("Country Code").is_in(drop_country)))
+
+chart_plotly = px.line(chart_dat,
+    x="year", y="value", color="Country Code", markers=True,
+    labels = {"year":"Year", "value":y_axis_title},
+    title = chart_title)
+
+chart_lp = ggplot(chart_dat, aes(x="year", y="value", color="Country Code")) +\
+  geom_point(shape=21, size=1.25, tooltips="none") +\
+  geom_line(tooltips=layer_tooltips()\
+            .format('value', '{.0f}')) +\
+  labs(
+    x="Year", y=y_axis_title,
+    title=title_text,
+    subtitle=subtitle_text) +\
+  scale_x_continuous(format='.0f') +\
+  theme(legend_position="bottom")
+chart_lp
+
+# %%
+# https://2001-2009.state.gov/r/pa/ho/time/pcw/98678.htm#:~:text=Apartheid%2C%20the%20Afrikaans%20name%20given,a%20democratic%20government%20in%201994.
+# What happened in South Africa in the 1990s?
+# What happened in Nigeria in early 2000?
+# What happened in Ghana in 2010s?
+# sp.add_annotation(
+#         x=1994, y=4100,
+#         text="Democratic Government",
+#         showarrow=True,
+#         yshift=10)

scripts_build/03_holoviews.py

@@ -0,0 +1,54 @@
+# %%
+import polars as pl
+import holoviews as hv
+import panel as pn
+from holoviews import opts
+hv.extension('bokeh')
+dat = pl.read_parquet("../data/dat_munged.parquet")
+info = pl.read_csv("../data/Metadata_Indicator_API_Download_DS2_en_csv_v2_5657328.csv").rename({"INDICATOR_CODE":"Indicator Code", "INDICATOR_NAME":"Indicator Name"})
+
+dat_vars = dat\
+    .group_by("Indicator Name", "Indicator Code", "Country Code").len()\
+    .pivot(values="len", index=["Indicator Name", "Indicator Code"], columns="Country Code", aggregate_function="first")\
+    .with_columns((pl.col("COD").fill_null(0) + pl.col("GHA").fill_null(0) +
+                   pl.col("KEN").fill_null(0) + pl.col("NGA").fill_null(0) +
+                   pl.col("ZAF").fill_null(0) + pl.col("ZWE").fill_null(0)).alias("total"))\
+    .sort(pl.col("total"), descending=True)\
+    .filter(pl.col("total") > 25)
+
+# %%
+hv.Scatter(dat.filter(pl.col("Indicator Code").is_in(["NY.GDP.PCAP.PP.KD"])).to_pandas(), "year", "value")\
+  .opts(width=500)
+
+# %%
+def select_row(row=0):
+  return dat.slice(row, 1)
+
+app = pn.interact(select_row, row=(0, dat.select(pl.len()).item()))
+print(app)
+# %%
+pn.Column("## Choose a row", pn.Row(app[0], app[1]))
+# %%
+color_picker = pn.widgets.ColorPicker()
+color_picker
+# %%
+html = pn.pane.HTML('', width=200, height=200, styles={'background-color': color_picker.value})
+color_picker.link(html, value="background-color")# %%
+html
+
+# %%
+x = pn.widgets.IntSlider(name='x', start=0, end=100)
+background = pn.widgets.ColorPicker(name='Background', value='lightgray')
+
+def square(x):
+    return f'{x} squared is {x**2}'
+
+def styles(background):
+    return {'background-color': background, 'padding': '0 10px'}
+
+pn.Column(
+    x,
+    background,
+    pn.pane.Markdown(pn.bind(square, x), styles=pn.bind(styles, background))
+)
+# %%

streamlit.py

@@ -0,0 +1,82 @@
+# %%
+# packages
+import streamlit as st
+import polars as pl
+import plotly.express as px
+import plotly.io as pio
+pio.templates.default = "simple_white"
+# %%
+# Data
+dat = pl.read_parquet("data/dat_munged.parquet")
+info = pl.read_csv("data/Metadata_Indicator_API_Download_DS2_en_csv_v2_5657328.csv").rename({"INDICATOR_CODE":"Indicator Code", "INDICATOR_NAME":"Indicator Name"})
+dat_vars = pl.read_parquet("data/dat_vars.parquet")
+
+# %%
+# Example Chart
+# drop_country =  ["ZAF"]
+# indicator_code = "NY.GDP.PCAP.PP.KD"
+list_name = dat_vars.select("Indicator Name").to_series().to_list()
+list_code = dat_vars.select("Indicator Code").to_series().to_list()
+list_country_code = ["ZAF", "ZWE", "KEN", "NGA", "GHA", "COD"]
+list_country_name = ["South Africa", "Zimbabwe", "Kenya", "Nigeria", "Ghana", "Congo, Dem. Rep."]
+
+
+drop_country = st.sidebar.multiselect("Remove Country (Country Code)", list_country_code)
+
+checked_var = st.sidebar.checkbox("Use Variable Name")
+
+
+if checked_var:
+    indicator_name = st.sidebar.selectbox("Select your variable", list_name)
+    indicator_code = dat_vars.filter(pl.col("Indicator Name") == indicator_name).select("Indicator Code").to_series()[0]
+else:
+    indicator_code = st.sidebar.selectbox("Select your variable", list_code)
+    indicator_name = dat_vars.filter(pl.col("Indicator Code") == indicator_code).select("Indicator Name").to_series()[0]
+
+title_text = indicator_name
+subtitle_text = info.filter(pl.col("Indicator Code") == indicator_code).select("SOURCE_NOTE").to_series()[0]
+
+y_axis_title = indicator_name[indicator_name.find("(")+1:indicator_name.find(")")]
+
+use_dat = dat.filter((pl.col("Indicator Code").is_in([str(indicator_code)])) & (~pl.col("Country Code").is_in(drop_country)) & (pl.col("value").is_not_null()))
+
+sp = px.line(use_dat.to_pandas(),
+    x="year", y="value", color="Country Name", markers=True,
+    labels = {"year":"Year", "value":y_axis_title},
+    title = title_text)
+
+st.markdown("## Country performance over time")
+
+st.markdown("__" + title_text + "__")
+
+st.markdown(subtitle_text)
+
+st.markdown("### Chart")
+st.markdown("_Use the expand arrows visible when you hover over the upper right corner of the chart to see it in full screen._")
+
+sp
+
+st.markdown("### Table: " + title_text)
+
+display_dat = use_dat.select("Country Code", "Indicator Name", "year", "value")
+
+st.dataframe(
+  display_dat\
+    .pivot(index="year", on="Country Code", values="value", aggregate_function="first")\
+    .sort(pl.col("year"),descending=True), hide_index=True,
+    use_container_width=True,
+    column_config={
+        "value": y_axis_title,
+        "year": st.column_config.NumberColumn(
+            "Year",
+            help="Year of data",
+            format="%.0f"
+        )})
+
+
+def convert_df(df):
+    return df.write_csv().encode('utf-8')
+
+csv = convert_df(display_dat)
+
+st.download_button("Download Data", data = csv, file_name = "data.csv", mime="text/csv")