Code Along 1: SF Rental Market
Jordan Lanowy
2024-09-10
Goal: to predict the rental prices in the SF rental market Click here for the data.
Import Data
rent <- read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2022/2022-07-05/rent.csv')## Rows: 200796 Columns: 17
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (8): post_id, nhood, city, county, address, title, descr, details
## dbl (9): date, year, price, beds, baths, sqft, room_in_apt, lat, lon
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.skimr::skim(rent)| Name | rent |
| Number of rows | 200796 |
| Number of columns | 17 |
| _______________________ | |
| Column type frequency: | |
| character | 8 |
| numeric | 9 |
| ________________________ | |
| Group variables | None |
Variable type: character
| skim_variable | n_missing | complete_rate | min | max | empty | n_unique | whitespace |
|---|---|---|---|---|---|---|---|
| post_id | 0 | 1.00 | 9 | 14 | 0 | 200796 | 0 |
| nhood | 0 | 1.00 | 4 | 43 | 0 | 167 | 0 |
| city | 0 | 1.00 | 5 | 19 | 0 | 104 | 0 |
| county | 1394 | 0.99 | 4 | 13 | 0 | 10 | 0 |
| address | 196888 | 0.02 | 1 | 38 | 0 | 2869 | 0 |
| title | 2517 | 0.99 | 2 | 298 | 0 | 184961 | 0 |
| descr | 197542 | 0.02 | 13 | 16975 | 0 | 3025 | 0 |
| details | 192780 | 0.04 | 4 | 595 | 0 | 7667 | 0 |
Variable type: numeric
| skim_variable | n_missing | complete_rate | mean | sd | p0 | p25 | p50 | p75 | p100 | hist |
|---|---|---|---|---|---|---|---|---|---|---|
| date | 0 | 1.00 | 20095718.38 | 44694.07 | 20000902.00 | 20050227.00 | 20110924.00 | 20120805.0 | 20180717.00 | ▁▇▁▆▃ |
| year | 0 | 1.00 | 2009.51 | 4.48 | 2000.00 | 2005.00 | 2011.00 | 2012.0 | 2018.00 | ▁▇▁▆▃ |
| price | 0 | 1.00 | 2135.36 | 1427.75 | 220.00 | 1295.00 | 1800.00 | 2505.0 | 40000.00 | ▇▁▁▁▁ |
| beds | 6608 | 0.97 | 1.89 | 1.08 | 0.00 | 1.00 | 2.00 | 3.0 | 12.00 | ▇▂▁▁▁ |
| baths | 158121 | 0.21 | 1.68 | 0.69 | 1.00 | 1.00 | 2.00 | 2.0 | 8.00 | ▇▁▁▁▁ |
| sqft | 136117 | 0.32 | 1201.83 | 5000.22 | 80.00 | 750.00 | 1000.00 | 1360.0 | 900000.00 | ▇▁▁▁▁ |
| room_in_apt | 0 | 1.00 | 0.00 | 0.04 | 0.00 | 0.00 | 0.00 | 0.0 | 1.00 | ▇▁▁▁▁ |
| lat | 193145 | 0.04 | 37.67 | 0.35 | 33.57 | 37.40 | 37.76 | 37.8 | 40.43 | ▁▁▅▇▁ |
| lon | 196484 | 0.02 | -122.21 | 0.78 | -123.20 | -122.42 | -122.26 | -122.0 | -74.20 | ▇▁▁▁▁ |
data <- rent %>%
# treat missing values
select(-address, -descr, -details, -lat, -lon, -date, -year, -room_in_apt) %>%
na.omit() %>%
# log tansform variables with pos-skewed distribution
mutate(price = log(price))
skimr::skim(data)| Name | data |
| Number of rows | 14394 |
| Number of columns | 9 |
| _______________________ | |
| Column type frequency: | |
| character | 5 |
| numeric | 4 |
| ________________________ | |
| Group variables | None |
Variable type: character
| skim_variable | n_missing | complete_rate | min | max | empty | n_unique | whitespace |
|---|---|---|---|---|---|---|---|
| post_id | 0 | 1 | 10 | 14 | 0 | 14394 | 0 |
| nhood | 0 | 1 | 4 | 38 | 0 | 143 | 0 |
| city | 0 | 1 | 5 | 14 | 0 | 85 | 0 |
| county | 0 | 1 | 4 | 13 | 0 | 10 | 0 |
| title | 0 | 1 | 6 | 298 | 0 | 14075 | 0 |
Variable type: numeric
| skim_variable | n_missing | complete_rate | mean | sd | p0 | p25 | p50 | p75 | p100 | hist |
|---|---|---|---|---|---|---|---|---|---|---|
| price | 0 | 1 | 7.82 | 0.45 | 5.86 | 7.52 | 7.8 | 8.08 | 10.09 | ▁▃▇▁▁ |
| beds | 0 | 1 | 2.32 | 1.03 | 0.00 | 2.00 | 2.0 | 3.00 | 8.00 | ▂▇▁▁▁ |
| baths | 0 | 1 | 1.77 | 0.74 | 1.00 | 1.00 | 2.0 | 2.00 | 8.00 | ▇▂▁▁▁ |
| sqft | 0 | 1 | 1273.44 | 698.09 | 110.00 | 887.00 | 1100.0 | 1500.00 | 22000.00 | ▇▁▁▁▁ |
Explore Data
Identify good predictors
sqft
data %>%
ggplot(aes(price, sqft)) +
scale_y_log10()+
geom_point()
beds
data %>%
ggplot(aes(price, as.factor(beds))) +
geom_boxplot()
title
data %>%
# tokenize title
unnest_tokens(output = word, input = title) %>%
# calculate avg rent per word
group_by(word) %>%
summarise(price = mean(price),
n = n()) %>%
ungroup() %>%
filter(n > 10, !str_detect(word, "\\d")) %>%
slice_max(order_by = price, n = 20) %>%
# plot
ggplot(aes(price, fct_reorder(word, price))) +
geom_point() +
labs(y = "Words in Title")
EDA shortcut
# Step 1: Prepare data
data_binarized_tbl <- data %>%
select(-post_id, -title) %>%
binarize
data_binarized_tbl %>% glimpse## Rows: 14,394
## Columns: 85
## $ nhood__campbell <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__concord_/_pleasant_hill_/_martinez` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__cupertino <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__daly_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__danville_/_san_ramon` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__dublin_/_pleasanton` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__fairfield_/_vacaville` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__foster_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__hayward_/_castro_valley` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__milpitas <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__mountain_view <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__napa_county <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__palo_alto <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__petaluma <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__pittsburg_/_antioch` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__rohnert_pk_/_cotati` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_francisco <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_jose_central <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_jose_east <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_jose_north <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_jose_south <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_jose_west <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_mateo <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__san_rafael <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__santa_clara <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__santa_cruz <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__santa_rosa <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__SOMA_/_south_beach` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__sunnyvale <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ nhood__union_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__vallejo_/_benicia` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__willow_glen_/_cambrian` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `nhood__-OTHER` <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1,…
## $ city__cambrian <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__campbell <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__concord <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__cupertino <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__daly_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__dublin <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__fairfield <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__foster_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__hayward <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__milpitas <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__mountain_view <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__napa_county <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__oakland <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__palo_alto <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__petaluma <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__pittsburg <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__rohnert_park <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__san_francisco <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__san_jose <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__san_mateo <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__san_rafael <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__san_ramon <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__santa_clara <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__santa_cruz <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__santa_rosa <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__sunnyvale <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__union_city <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ city__vallejo <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `city__-OTHER` <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1,…
## $ county__alameda <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1,…
## $ county__contra_costa <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__marin <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__napa <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__san_francisco <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__san_mateo <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__santa_clara <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__santa_cruz <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__solano <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ county__sonoma <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `price__-Inf_7.52294091807237` <dbl> 0, 1, 0, 1, 0, 1, 1, 0, 0,…
## $ price__7.52294091807237_7.80384330353877 <dbl> 0, 0, 1, 0, 0, 0, 0, 1, 1,…
## $ price__7.80384330353877_8.07868822922987 <dbl> 1, 0, 0, 0, 1, 0, 0, 0, 0,…
## $ price__8.07868822922987_Inf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `beds__-Inf_2` <dbl> 0, 1, 0, 1, 1, 1, 0, 0, 1,…
## $ beds__2_3 <dbl> 0, 0, 1, 0, 0, 0, 1, 1, 0,…
## $ beds__3_Inf <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0,…
## $ `baths__-Inf_2` <dbl> 0, 1, 1, 1, 1, 1, 0, 0, 1,…
## $ baths__2_Inf <dbl> 1, 0, 0, 0, 0, 0, 1, 1, 0,…
## $ `sqft__-Inf_887` <dbl> 0, 1, 0, 0, 0, 0, 0, 0, 1,…
## $ sqft__887_1100 <dbl> 0, 0, 0, 1, 0, 0, 0, 1, 0,…
## $ sqft__1100_1500 <dbl> 0, 0, 1, 0, 1, 1, 0, 0, 0,…
## $ sqft__1500_Inf <dbl> 1, 0, 0, 0, 0, 0, 1, 0, 0,…# step 2: correlate
data_correlate_tbl <- data_binarized_tbl %>%
correlate(price__8.07868822922987_Inf)
# step 3: Plot
data_correlate_tbl %>%
plot_correlation_funnel()## Warning: ggrepel: 69 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps