us_demographics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# US Demographics Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:31.323061Z",
     "start_time": "2020-05-12T10:08:29.743366Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import ipyleaflet as ipyl\n",
    "import geopandas as gpd\n",
    "import matplotlib\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import HTML\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "from mpl_toolkits.axes_grid1.inset_locator import inset_axes\n",
    "from shapely.geometry import Polygon\n",
    "from matplotlib.colors import LogNorm\n",
    "from shapely.geometry import mapping, MultiPolygon, Polygon\n",
    "from branca.colormap import linear, LinearColormap\n",
    "import folium\n",
    "from folium import plugins\n",
    "from io import BytesIO\n",
    "import binascii\n",
    "from matplotlib.cm import get_cmap\n",
    "from PIL import Image\n",
    "from typing import Iterable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:31.326961Z",
     "start_time": "2020-05-12T10:08:31.324520Z"
    }
   },
   "outputs": [],
   "source": [
    "out_dir = 'out/'\n",
    "data_dir = 'data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Demographics Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Taken from the National Bureau of Economic Research Census data: \n",
    "\n",
    "https://data.nber.org/census/popest/countypopmonthasrh.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:33.529423Z",
     "start_time": "2020-05-12T10:08:31.328482Z"
    }
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(data_dir+'countypopmonthasrh.csv')\n",
    "data = data[data.yearref > 2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load County Shapes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Taken from the 2017 census TIGER files:\n",
    "\n",
    "https://www2.census.gov/geo/tiger/TIGER2017/COUNTY/tl_2017_us_county.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:45.423723Z",
     "start_time": "2020-05-12T10:08:33.530850Z"
    }
   },
   "outputs": [],
   "source": [
    "county_shapes = gpd.read_file(data_dir+'tl_2017_us_county').dropna(subset=['GEOID'])\n",
    "county_shapes.GEOID = county_shapes.GEOID.astype(np.int64)\n",
    "county_shapes['area'] = (county_shapes.ALAND + county_shapes.AWATER) / 1E6\n",
    "county_shapes.geometry = county_shapes.geometry.simplify(.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:45.621257Z",
     "start_time": "2020-05-12T10:08:45.425474Z"
    }
   },
   "outputs": [],
   "source": [
    "data = gpd.GeoDataFrame(data.merge(county_shapes[['GEOID', 'geometry', 'area']], \n",
    "                                   left_on='county', right_on='GEOID'))\n",
    "data.crs = county_shapes.crs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:45.803722Z",
     "start_time": "2020-05-12T10:08:45.622339Z"
    }
   },
   "outputs": [],
   "source": [
    "data_agg = data[data.agegrp == 0].copy()\n",
    "data = data[data.agegrp > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:45.809210Z",
     "start_time": "2020-05-12T10:08:45.805099Z"
    }
   },
   "outputs": [],
   "source": [
    "age_groups= {1: 0,\n",
    " 2: 5,\n",
    " 3: 10,\n",
    " 4: 15,\n",
    " 5: 20,\n",
    " 6: 25,\n",
    " 7: 30,\n",
    " 8: 35,\n",
    " 9: 40,\n",
    " 10: 45,\n",
    " 11: 50,\n",
    " 12: 55,\n",
    " 13: 60,\n",
    " 14: 65,\n",
    " 15: 70,\n",
    " 16: 75,\n",
    " 17: 80,\n",
    " 18: 85}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:08:45.838067Z",
     "start_time": "2020-05-12T10:08:45.812043Z"
    }
   },
   "outputs": [],
   "source": [
    "data.agegrp = data.agegrp.replace(age_groups)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing Data for 2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:50:47.449535Z",
     "start_time": "2020-05-11T22:50:47.413672Z"
    }
   },
   "outputs": [],
   "source": [
    "data_2016 = data[data.year == 2016].copy()\n",
    "data_2016_agg = data_agg[data_agg.year == 2016].copy()\n",
    "data_2016_agg['female_percentage'] = np.round(data_2016_agg.tot_female / data_2016_agg.tot_pop * 100, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Plotting Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:50:47.455781Z",
     "start_time": "2020-05-11T22:50:47.450832Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def bbox_to_polyon(min_x, min_y, max_x, max_y):\n",
    "    return Polygon([(min_x, min_y), (max_x, min_y), (max_x, max_y), (min_x, max_y)])\n",
    "\n",
    "def plot_geo_df(geo_df, value_col, cmap='RdBu', vmin=0, vmax=1, ax=None, cax=None, legend=False,\n",
    "               plot_kwargs = {}):\n",
    "    if ax is None:\n",
    "        fig, ax = plt.subplots()\n",
    "        \n",
    "    if legend and cax is None:\n",
    "        divider = make_axes_locatable(ax)\n",
    "        cax = divider.append_axes(\"right\", size=\"2%\", pad=0.1)\n",
    "\n",
    "    geo_df.plot(value_col, cmap=cmap, vmin=vmin, vmax=vmax, ax=ax, legend=legend, cax=cax, **plot_kwargs)\n",
    "    geo_df.boundary.plot(ax=ax, color='black', linewidth=.25, alpha = .25)\n",
    "    ax.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:50:47.463955Z",
     "start_time": "2020-05-11T22:50:47.456785Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_us_data(geo_df, column, title, center=50, cmap='RdBu_r', logarithmic = False):\n",
    "    fig, ax1 = plt.subplots(1, figsize=(20,20), dpi=200)\n",
    "    fig.patch.set_facecolor('#f5f5f5'), fig.patch.set_alpha(1)\n",
    "    ax1.set_title(title)\n",
    "    divider = make_axes_locatable(ax1)\n",
    "    cax = divider.append_axes(\"right\", size=\"1.5%\", pad=0.1)\n",
    "    ax2 = inset_axes(ax1, width=\"30%\", height=\"30%\", loc=3)\n",
    "\n",
    "    alaska = geo_df[geo_df.geometry.within(bbox_to_polyon(-179, 50, -125, 72))]\n",
    "    continental_us = geo_df[geo_df.geometry.within(bbox_to_polyon(-126, 23, -66,  50))]\n",
    "\n",
    "    if center is None:\n",
    "        vmin, vmax = geo_df[column].min(), geo_df[column].max()\n",
    "    else:\n",
    "        rng = np.max(np.abs(geo_df[column] - center))\n",
    "        vmin, vmax = center-rng, center+rng\n",
    "    \n",
    "    plot_kwargs = dict(norm=LogNorm(vmin=vmin, vmax=vmax)) if logarithmic else {}\n",
    "    plot_geo_df(continental_us, column, ax=ax1, legend=True, cax=cax, vmin=vmin, vmax=vmax, cmap=cmap,\n",
    "               plot_kwargs=plot_kwargs)\n",
    "    plot_geo_df(alaska, column, ax=ax2, vmin=vmin, vmax=vmax, cmap=cmap,\n",
    "               plot_kwargs=plot_kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Percentage of female population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:50:51.340413Z",
     "start_time": "2020-05-11T22:50:47.464982Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plot_us_data(data_2016_agg, 'female_percentage', 'Percentage of female population in US counties')\n",
    "plt.savefig(out_dir + 'female_percentage.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:50:55.057265Z",
     "start_time": "2020-05-11T22:50:51.341586Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plot_us_data(data_2016_agg, 'tot_pop', 'Population per US county', cmap='viridis', center=None, logarithmic=True)\n",
    "plt.savefig(out_dir + 'population.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interactive Plotting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tooltip Heatmap Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:12:51.946214Z",
     "start_time": "2020-05-12T10:12:51.919502Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_choro_heatmap(df, ax, cmap, vmin, vmax, choro_col='female_percentage', x_col='year', y_col='agegrp'):\n",
    "    heatmap_data = df.pivot(columns=x_col, index=y_col, values=choro_col)\n",
    "    sns.heatmap(heatmap_data, cmap=cmap, vmin=vmin, vmax=vmax, ax=ax)\n",
    "    plt.tight_layout()\n",
    "    plt.ylim(len(df[y_col].unique()), 0)\n",
    "    \n",
    "def get_ethnicity_df(df):\n",
    "    ethnicities = {'wa': 'white', 'ba':'black', 'ia':'native', 'aa':'asian', 'na':'pacific', 'h': 'hisp.'}\n",
    "    eth_df = df[df.year == df.year.max()][['agegrp']].copy()\n",
    "    for eth, name in ethnicities.items():\n",
    "        eth_df[name] = (df[eth+'_female'] + df[eth+'_male'])\n",
    "    return eth_df.groupby(['agegrp']).sum().reset_index()\n",
    "\n",
    "def plot_ethnicities_heatmap(df, ax, cmap='viridis', cbar_ax = None, **_):\n",
    "    eth_df = get_ethnicity_df(df)\n",
    "    eth_df_melted = eth_df.melt(id_vars=('agegrp', ), var_name='ethnicity')\n",
    "    heatmap_data=eth_df_melted.pivot(columns='ethnicity', index='agegrp', values='value')\n",
    "    sns.heatmap(heatmap_data, cmap=cmap, ax=ax, cbar_ax=cbar_ax)\n",
    "    plt.ylim(heatmap_data.shape[0], 0)\n",
    "    \n",
    "\n",
    "def get_heatmap_plot(df, plotting_fun, vmin=None, vmax=None, cmap='viridis', svg=False, figsize=(3.25, 2.75), \n",
    "                     colorspace_compression=True, **kwargs):\n",
    "    def encode_plot_png(fig: plt.Figure):\n",
    "        if colorspace_compression:\n",
    "            with BytesIO() as buf:\n",
    "                plt.savefig(buf, format='png')\n",
    "                buf.seek(0)\n",
    "                im = Image.open(buf)\n",
    "                # convert to 8bit\n",
    "                im2 = im.convert('RGB').convert('P', palette=Image.ADAPTIVE)\n",
    "            with BytesIO() as buf2:\n",
    "                im2.save(buf2 , format='PNG')\n",
    "                img_data = binascii.b2a_base64(buf2.getvalue()).decode()\n",
    "        else:\n",
    "            with BytesIO() as buf:\n",
    "                fig.canvas.print_png(buf, transparent=True)\n",
    "                img_data = binascii.b2a_base64(buf.getvalue()).decode()\n",
    "        \n",
    "        img_html = '<img src=\"data:image/png;base64,{}\">'.format(img_data)\n",
    "        return img_html\n",
    "    \n",
    "    def encode_plot_svg(fig: plt.Figure):\n",
    "        with BytesIO() as buf:\n",
    "            plt.savefig(buf, format='svg')\n",
    "            img_data = buf.getvalue().decode()\n",
    "        return img_data\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=figsize) \n",
    "    plotting_fun(df, ax=ax, cmap=cmap, vmin=vmin, vmax=vmax, **kwargs)\n",
    "    plt.tight_layout()\n",
    "    img_data = encode_plot_svg(fig) if svg else encode_plot_png(fig)\n",
    "    plt.close(fig)\n",
    "    return img_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T10:44:33.904985Z",
     "start_time": "2020-05-12T10:44:33.885876Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_choropleth_layer(geo_df: gpd.GeoDataFrame, choro_col: str, name_col: str, colormap, name: str = '', \n",
    "                         vmin=0, vmax=1, id_col: str='county', show_layer=True):\n",
    "\n",
    "    def create_geo_data(df: gpd.GeoDataFrame):\n",
    "        return ipyl.GeoData(geo_dataframe=df).data\n",
    "    \n",
    "    def get_style(feature):\n",
    "        return {'fillColor': colormap(feature['properties'][choro_col]), 'color': 'grey', 'weight': .1, \n",
    "                'lineOpacity': .4, 'fillOpacity': .8}\n",
    "    \n",
    "    def get_hover_style(feature):\n",
    "        return {'fillColor': colormap(feature['properties'][choro_col]), \n",
    "                'lineOpacity': 1, 'fillOpacity': 1}\n",
    "\n",
    "    geo_json_data = create_geo_data(geo_df)\n",
    "    tooltip_cols = [name_col, 'population', choro_col]\n",
    "    tooltip_aliases = [c.replace('_percentage', ' %').replace('_', '<br/>') for c in tooltip_cols]\n",
    "    if 'img' in geo_df.columns:\n",
    "        tooltip_aliases, tooltip_cols = [*tooltip_aliases, ''], [*tooltip_cols, 'img']\n",
    "    \n",
    "    tooltip=folium.features.GeoJsonTooltip(fields= tooltip_cols, aliases= tooltip_aliases)\n",
    "    \n",
    "    layer = folium.GeoJson(ipyl.GeoData(geo_dataframe=geo_df).data, highlight_function=get_hover_style,\n",
    "        name=name, smooth_factor=0, tooltip=tooltip, style_function=get_style, show=show_layer)\n",
    "\n",
    "    return layer\n",
    "\n",
    "def plot_us_data_map(df, choro_column = 'female_percentage', layers_column = 'year', m=None, name:str=None,\n",
    "                    vmin = 0, vmax=1, layer_val = 2016, name_column = 'ctyname', choro_fun=None, cmap='viridis',\n",
    "                    plotting_fun=plot_choro_heatmap, log_scaling=False, add_colorbar=True, add_controls=True,\n",
    "                     show_layer=True, **plotting_fun_kwargs):\n",
    "    def add_plots():\n",
    "        for county in df.county.unique():\n",
    "            plots[county]=[]\n",
    "            for fun in plotting_fun:\n",
    "                indexer = df.county == county\n",
    "                county_df = df[indexer]\n",
    "                img = get_heatmap_plot(county_df, vmin=vmin, vmax=vmax, cmap=cmap, plotting_fun=fun,\n",
    "                                      choro_col=choro_column, **plotting_fun_kwargs)\n",
    "                plots[county].append(img)\n",
    "            plots[county] = '<br/>'.join(plots[county])\n",
    "    \n",
    "    if m is None:\n",
    "        m = folium.Map(location=[48, -102], zoom_start=3)\n",
    "        \n",
    "    mp_colormap = get_cmap(cmap)\n",
    "    samples = np.linspace(vmin, vmax, 20)\n",
    "    samples_scaled = np.logspace(np.log10(vmin), np.log10(vmax), 20) if log_scaling else samples\n",
    "    colors = [mp_colormap((s-vmin)/(vmax-vmin)) for s in samples]\n",
    "    colormap = LinearColormap(colors=colors, index=samples_scaled, vmin=vmin,vmax=vmax)\n",
    "    \n",
    "    if plotting_fun is not None:\n",
    "        if choro_fun != None:\n",
    "            df[choro_column] = choro_fun(df)\n",
    "        plots = {}\n",
    "        if not isinstance(plotting_fun, Iterable):\n",
    "            plotting_fun = [plotting_fun]\n",
    "        add_plots()\n",
    "        \n",
    "\n",
    "    layer_df = df[df[layers_column] == layer_val]\n",
    "    grouped = layer_df.groupby('county')\n",
    "    layer_df_agg = gpd.GeoDataFrame(grouped.sum())\n",
    "    layer_df_agg['ctyname'] = grouped.ctyname.first()\n",
    "    layer_df_agg['geometry'] = grouped.geometry.first()\n",
    "    layer_df_agg['population'] = layer_df_agg['tot_pop']\n",
    "    layer_df_agg['area'] = grouped.area.first()\n",
    "    \n",
    "    if choro_fun != None:\n",
    "        layer_df_agg[choro_column] = choro_fun(layer_df_agg)\n",
    "\n",
    "    layer_df_agg.crs = df.crs\n",
    "    layer_df_agg.reset_index(inplace=True)\n",
    "    \n",
    "    columns = ['geometry', choro_column, name_column, 'population']\n",
    "    \n",
    "    if plotting_fun is not None:\n",
    "        layer_df_agg['img'] = layer_df_agg.county.apply(lambda c: plots[c])\n",
    "        columns.append('img')\n",
    "\n",
    "    if name in (None, ''):\n",
    "        name = '{} {}'.format(layers_column, layer_val).replace('_', ' ')\n",
    "        \n",
    "    layer = get_choropleth_layer(layer_df_agg[columns], choro_col=choro_column, name_col = name_column,\n",
    "                                name=name, colormap=colormap, vmin=vmin, vmax=vmax, show_layer=show_layer)\n",
    "    layer.add_to(m)\n",
    "        \n",
    "    if add_colorbar:\n",
    "        m.add_child(colormap)\n",
    "    \n",
    "    if add_controls:\n",
    "        folium.LayerControl().add_to(m)\n",
    "        plugins.Fullscreen().add_to(m)\n",
    "    \n",
    "    return m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Gender Balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:59:36.747609Z",
     "start_time": "2020-05-11T22:50:55.093060Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_female_percentage(df):\n",
    "    return round(df.tot_female / df.tot_pop * 100, 2)\n",
    "\n",
    "m_gender_balance = plot_us_data_map(data, vmin=25, vmax=75, choro_column = 'female_percentage',\n",
    "                                   choro_fun = get_female_percentage, cmap='RdBu_r', name='female percentage',\n",
    "                                   plotting_fun = plot_choro_heatmap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:59:38.568858Z",
     "start_time": "2020-05-11T22:59:36.748654Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "m_gender_balance.save(out_dir + 'gender_balance_2016.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T22:59:39.627536Z",
     "start_time": "2020-05-11T22:59:38.571847Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "grouped = data.groupby(['year', 'agegrp'])\n",
    "overall_gender_balance_df = get_female_percentage(grouped.sum()).reset_index().rename({0: 'female_percentage'}, \n",
    "                                                                                      axis='columns')\n",
    "rng = np.max(np.abs(overall_gender_balance_df.female_percentage - 50))\n",
    "plt.figure(figsize=(17,6), dpi = 150)\n",
    "plt.title('Percent of female Population')\n",
    "sns.heatmap(overall_gender_balance_df.pivot(index='year', columns='agegrp', values='female_percentage'),\n",
    "           annot=True, cmap='RdBu_r', vmin = 50-rng, vmax=50+rng, fmt='.3g', linewidths=1)\n",
    "\n",
    "b, t = plt.ylim()\n",
    "plt.ylim(b+.5, t-.5)\n",
    "plt.savefig(out_dir + 'gender_balance_overall.svg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Population Ethnicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T11:16:31.079182Z",
     "start_time": "2020-05-12T11:07:27.680858Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_ethnic_percentage(df):\n",
    "    return round((1 - (df.wa_male + df.wa_female) / df.tot_pop) * 100, 2)\n",
    "\n",
    "m_ethnic = folium.Map(location=[48, -102], zoom_start=3)\n",
    "m_ethnic = plot_us_data_map(data, vmin=0, vmax=100, choro_column = 'ethnic_percentage', \n",
    "                            name='overall ethnic %', choro_fun = get_ethnic_percentage, cmap='magma_r', \n",
    "                            plotting_fun=plot_ethnicities_heatmap, add_controls=False, m = m_ethnic)\n",
    "\n",
    "ethnicities = {'wa': 'white', 'ba':'black', 'ia':'native', 'aa':'asian', 'na':'pacific', 'h': 'hisp.'}\n",
    "for abbrev, name in ethnicities.items():\n",
    "    def get_ethnicity(df):\n",
    "        return round((df[abbrev+'_female'] + df[abbrev+'_male']) / df.tot_pop*100, 1)\n",
    "    \n",
    "    m_ethnic = plot_us_data_map(data, choro_column = name + ' %', m=m_ethnic, add_controls=False,\n",
    "                                name=name+ ' %', choro_fun = get_ethnicity, cmap='magma_r', add_colorbar=False,\n",
    "                                plotting_fun=None, vmin=0, vmax=100, show_layer=False)\n",
    "    \n",
    "folium.LayerControl().add_to(m_ethnic)\n",
    "plugins.Fullscreen().add_to(m_ethnic);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T11:16:35.894849Z",
     "start_time": "2020-05-12T11:16:31.080563Z"
    }
   },
   "outputs": [],
   "source": [
    "m_ethnic.save(out_dir + 'ethnic_pop_2016.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T23:08:26.560137Z",
     "start_time": "2020-05-11T23:08:25.491562Z"
    }
   },
   "outputs": [],
   "source": [
    "grouped = data.groupby(['year', 'agegrp'])\n",
    "overall_ethnic_df = get_ethnic_percentage(grouped.sum()).reset_index().rename({0: 'ethnic'}, \n",
    "                                                                                      axis='columns')\n",
    "plt.figure(figsize=(17,6), dpi = 150)\n",
    "plt.title('Percent of ethnic Population')\n",
    "sns.heatmap(overall_ethnic_df.pivot(index='year', columns='agegrp', values='ethnic'),\n",
    "           annot=True, cmap='magma_r', fmt='.3g', linewidths=1)\n",
    "\n",
    "b, t = plt.ylim()\n",
    "plt.ylim(b+.5, t-.5)\n",
    "plt.savefig(out_dir + 'ethnic_pop_overall.svg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Population Density"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T23:15:13.237487Z",
     "start_time": "2020-05-11T23:15:06.797094Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_pop_density(df):\n",
    "    return round(df.tot_pop / df['area'], 1)\n",
    "\n",
    "grouped = data[data.year == 2016].groupby('ctyname')\n",
    "densities = grouped.tot_pop.sum() / grouped['area'].first()\n",
    "vmin, vmax = densities.min(), densities.max()\n",
    "\n",
    "m_pop = plot_us_data_map(data, choro_column = 'population_density', vmin=vmin, vmax=vmax,\n",
    "                            name='population density<br>log scaled', choro_fun = get_pop_density, cmap='magma_r', \n",
    "                            plotting_fun=None, log_scaling=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-11T23:15:13.768646Z",
     "start_time": "2020-05-11T23:15:13.239302Z"
    }
   },
   "outputs": [],
   "source": [
    "m_pop.save(out_dir + 'population_density_2016.html')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (data-explorer)",
   "language": "python",
   "name": "data-explorer"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}