# The following lines were added to get the Jupyter Notebook working in Colab
!pip install datashader
!pip install pyproj
# Be sure to update IPython otherwise GeoJSON will not work
# Also be sure to restart the runtime environment once when this section of code runs
!pip install -U IPython
from shapely.geometry import box
# Download of the data set from Github
!wget https://github.com/logicalschema/Fall-2021/raw/main/DATA608/module2/nyc_pluto_21v2_csv.zip
!unzip nyc_pluto_21v2_csv.zip
import datashader as ds
import datashader.transfer_functions as tf
import datashader.glyphs
from datashader import reductions
from datashader.core import bypixel
from datashader.utils import lnglat_to_meters as webm, export_image
from datashader.colors import colormap_select, Greys9, viridis, inferno
import copy
from pyproj import Proj, transform
import numpy as np
import pandas as pd
import urllib
import json
import datetime
import colorlover as cl
import plotly.offline as py
# Added by Sung Lee
import plotly.express as px
import plotly.graph_objs as go
from plotly import tools
# from shapely.geometry import Point, Polygon, shape
# In order to get shapley, you'll need to run [pip install shapely.geometry] from your terminal
from functools import partial
from IPython.display import GeoJSON
# This line was added by Sung Lee for HTML function
from IPython.display import HTML
py.init_notebook_mode()
For module 2 we'll be looking at techniques for dealing with big data. In particular binning strategies and the datashader library (which possibly proves we'll never need to bin large data for visualization ever again.)
To demonstrate these concepts we'll be looking at the PLUTO dataset put out by New York City's department of city planning. PLUTO contains data about every tax lot in New York City.
PLUTO data can be downloaded from here. Unzip them to the same directory as this notebook, and you should be able to read them in using this (or very similar) code. Also take note of the data dictionary, it'll come in handy for this assignment.
# Code to read in v17, column names have been updated (without upper case letters) for v18
# bk = pd.read_csv('PLUTO17v1.1/BK2017V11.csv')
# bx = pd.read_csv('PLUTO17v1.1/BX2017V11.csv')
# mn = pd.read_csv('PLUTO17v1.1/MN2017V11.csv')
# qn = pd.read_csv('PLUTO17v1.1/QN2017V11.csv')
# si = pd.read_csv('PLUTO17v1.1/SI2017V11.csv')
# ny = pd.concat([bk, bx, mn, qn, si], ignore_index=True)
# ny = pd.read_csv('nyc_pluto_21v2_csv/pluto_21v2.csv')
# This line was edited to use the data file via Github download
ny = pd.read_csv('pluto_21v2.csv')
# Getting rid of some outliers
ny = ny[(ny['yearbuilt'] > 1850) & (ny['yearbuilt'] < 2020) & (ny['numfloors'] != 0)]
I'll also do some prep for the geographic component of this data, which we'll be relying on for datashader.
You're not required to know how I'm retrieving the lattitude and longitude here, but for those interested: this dataset uses a flat x-y projection (assuming for a small enough area that the world is flat for easier calculations), and this needs to be projected back to traditional lattitude and longitude.
# wgs84 = Proj("+proj=longlat +ellps=GRS80 +datum=NAD83 +no_defs")
# nyli = Proj("+proj=lcc +lat_1=40.66666666666666 +lat_2=41.03333333333333 +lat_0=40.16666666666666 +lon_0=-74 +x_0=300000 +y_0=0 +ellps=GRS80 +datum=NAD83 +to_meter=0.3048006096012192 +no_defs")
# ny['xcoord'] = 0.3048*ny['xcoord']
# ny['ycoord'] = 0.3048*ny['ycoord']
# ny['lon'], ny['lat'] = transform(nyli, wgs84, ny['xcoord'].values, ny['ycoord'].values)
# ny = ny[(ny['lon'] < -60) & (ny['lon'] > -100) & (ny['lat'] < 60) & (ny['lat'] > 20)]
#Defining some helper functions for DataShader
background = "black"
export = partial(export_image, background = background, export_path="export")
cm = partial(colormap_select, reverse=(background!="black"))
Part 1: Binning and Aggregation¶
Binning is a common strategy for visualizing large datasets. Binning is inherent to a few types of visualizations, such as histograms and 2D histograms (also check out their close relatives: 2D density plots and the more general form: heatmaps.
While these visualization types explicitly include binning, any type of visualization used with aggregated data can be looked at in the same way. For example, lets say we wanted to look at building construction over time. This would be best viewed as a line graph, but we can still think of our results as being binned by year:
trace = go.Scatter(
# I'm choosing BBL here because I know it's a unique key.
x = ny.groupby('yearbuilt').count()['bbl'].index,
y = ny.groupby('yearbuilt').count()['bbl']
)
layout = go.Layout(
xaxis = dict(title = 'Year Built'),
yaxis = dict(title = 'Number of Lots Built')
)
fig = go.FigureWidget(data = [trace], layout = layout)
# This line was changed to the line that follows to allow display
#fig
# Write the figure as a html file
fig.write_html('question1a.html')
HTML(fig.to_html())
Something looks off... You're going to have to deal with this imperfect data to answer this first question.
But first: some notes on pandas. Pandas dataframes are a different beast than R dataframes, here are some tips to help you get up to speed:
Hello all, here are some pandas tips to help you guys through this homework:
Indexing and Selecting: .loc and .iloc are the analogs for base R subsetting, or filter() in dplyr
Group By: This is the pandas analog to group_by() and the appended function the analog to summarize(). Try out a few examples of this, and display the results in Jupyter. Take note of what's happening to the indexes, you'll notice that they'll become hierarchical. I personally find this more of a burden than a help, and this sort of hierarchical indexing leads to a fundamentally different experience compared to R dataframes. Once you perform an aggregation, try running the resulting hierarchical datafrome through a reset_index().
Reset_index: I personally find the hierarchical indexes more of a burden than a help, and this sort of hierarchical indexing leads to a fundamentally different experience compared to R dataframes. reset_index() is a way of restoring a dataframe to a flatter index style. Grouping is where you'll notice it the most, but it's also useful when you filter data, and in a few other split-apply-combine workflows. With pandas indexes are more meaningful, so use this if you start getting unexpected results.
Indexes are more important in Pandas than in R. If you delve deeper into the using python for data science, you'll begin to see the benefits in many places (despite the personal gripes I highlighted above.) One place these indexes come in handy is with time series data. The pandas docs have a huge section on datetime indexing. In particular, check out resample, which provides time series specific aggregation.
Merging, joining, and concatenation: There's some overlap between these different types of merges, so use this as your guide. Concat is a single function that replaces cbind and rbind in R, and the results are driven by the indexes. Read through these examples to get a feel on how these are performed, but you will have to manage your indexes when you're using these functions. Merges are fairly similar to merges in R, similarly mapping to SQL joins.
Apply: This is explained in the "group by" section linked above. These are your analogs to the plyr library in R. Take note of the lambda syntax used here, these are anonymous functions in python. Rather than predefining a custom function, you can just define it inline using lambda.
Browse through the other sections for some other specifics, in particular reshaping and categorical data (pandas' answer to factors.) Pandas can take a while to get used to, but it is a pretty strong framework that makes more advanced functions easier once you get used to it. Rolling functions for example follow logically from the apply workflow (and led to the best google results ever when I first tried to find this out and googled "pandas rolling")
Google Wes Mckinney's book "Python for Data Analysis," which is a cookbook style intro to pandas. It's an O'Reilly book that should be pretty available out there.
Question¶
After a few building collapses, the City of New York is going to begin investigating older buildings for safety. The city is particularly worried about buildings that were unusually tall when they were built, since best-practices for safety hadn’t yet been determined. Create a graph that shows how many buildings of a certain number of floors were built in each year (note: you may want to use a log scale for the number of buildings). Find a strategy to bin buildings (It should be clear 20-29-story buildings, 30-39-story buildings, and 40-49-story buildings were first built in large numbers, but does it make sense to continue in this way as you get taller?)
# Start your answer here, inserting more cells as you go along
# A look at data
ny.head(5)
A look at the bldgclass for the data set. Additional information can be found in Appendix C of this document.
There are no NA values.
print(ny['bldgclass'].unique())
print("\n\nThe minimum number of floors is " + str(ny['numfloors'].min()))
print("The maximum number of floors is " + str(ny['numfloors'].max()))
For the data set, records between 1850 and 2020 exclusive were selected for buildings with numfloors not equal to zero. Visualizing the data by making the bins the number of floors (ex: 0 to 10) and grouping by year yields the following graph.
Histogram¶
Bins¶
I decided to make the yearbuiltbin variable for the data set to make a categorical variable for grouping. Besides the binning of the number of floors, the yearbuilt will be binned into 5 bins. I first used pandas.qcut to determine the interval labels. More information about the function is here.
minYear = min(ny['yearbuilt']) # The min value is 1851
maxYear = max(ny['yearbuilt']) # The max value is 2019
yearLabels = ['1851 to 1920', '1921 to 1930', '1931 to 1940', '1941 to 1965', '1966 to 2019']
ny['yearbuiltbin'] = pd.qcut(ny['yearbuilt'], 5, labels = yearLabels)
ny[['yearbuilt', 'yearbuiltbin']].head()
# https://plotly.com/python/histograms/#horizontal-histogram
fig = px.histogram(ny, x="numfloors",
histfunc="count",
title='Histogram of Floors',
labels={'numfloors':'Number of Floors'}, # can specify one label per df column
width=800, height=600,
opacity=0.7,
log_y=True, # represent bars with log scale
color = 'yearbuiltbin',
nbins=15
)
fig.update_layout({
'plot_bgcolor': 'rgba(0, 0, 0, 0)',
})
# Write the figure as a html file
fig.write_html('question1b.html')
HTML(fig.to_html())
The plot of the graph gets complicated with the years and as the buildings get taller. Besides adjusting the opacity and using pattern markers, the next logical step would be to make this interactive with Dash. I envision having a widget that would allow you to select by checkbox years. Remember the years are from 1850 to 2020, so being able to adjust the visualization of the year groups is helpful.
Scatter¶
I decided to also make a scatter graph to see if it would help with the original premise of the problem.
scatter_df = ny.loc[:, ('numfloors', 'yearbuilt')]
# Create bins
bins = np.arange(0, 110, 10)
# Place values of numfloors into bins
inds = np.digitize(scatter_df['numfloors'], bins, right=True)
# Add the bin location to each of the rows for scatter_df
scatter_df['bin'] = inds.tolist()
print(scatter_df.head(5))
plotxaxis = dict(
tickmode = 'array',
tickvals = list(range(1,12)),
ticktext = ['[0-10]',
'(10-20]',
'(20-30]',
'(30-40]',
'(40-50]',
'(50-60]',
'(60-70]',
'(70-80]',
'(80-90]',
'(90-100]',
'(100-110]'
]
)
fig = px.scatter(
scatter_df, x='bin', y='yearbuilt',
labels={'bin': 'Number of Floors',
'yearbuilt': 'Year Built'
}
)
# Change the tick marks for the x axis
fig.update_layout(
xaxis = plotxaxis
)
# Remove the background
fig.update_layout({
'plot_bgcolor': 'rgba(0, 0, 0, 0)',
})
# Write the figure as a html file
fig.write_html('question1c.html')
HTML(fig.to_html())
