Boston_EDA
Danielle Megafu
9/14/2026
Boston Housing Data
Load the libraries and the data:
library(MASS)
data(Boston)
# Avoid attach() in teaching examples so variable names always come from Boston$...And load the package for data visualization:
library(ggplot2)Exploratory Data Analysis (EDA)
Read the data document again
?BostonExamine data and variable types
head(Boston)## crim zn indus chas nox rm age dis rad tax ptratio black lstat
## 1 0.00632 18 2.31 0 0.538 6.575 65.2 4.0900 1 296 15.3 396.90 4.98
## 2 0.02731 0 7.07 0 0.469 6.421 78.9 4.9671 2 242 17.8 396.90 9.14
## 3 0.02729 0 7.07 0 0.469 7.185 61.1 4.9671 2 242 17.8 392.83 4.03
## 4 0.03237 0 2.18 0 0.458 6.998 45.8 6.0622 3 222 18.7 394.63 2.94
## 5 0.06905 0 2.18 0 0.458 7.147 54.2 6.0622 3 222 18.7 396.90 5.33
## 6 0.02985 0 2.18 0 0.458 6.430 58.7 6.0622 3 222 18.7 394.12 5.21
## medv
## 1 24.0
## 2 21.6
## 3 34.7
## 4 33.4
## 5 36.2
## 6 28.7str(Boston)## 'data.frame': 506 obs. of 14 variables:
## $ crim : num 0.00632 0.02731 0.02729 0.03237 0.06905 ...
## $ zn : num 18 0 0 0 0 0 12.5 12.5 12.5 12.5 ...
## $ indus : num 2.31 7.07 7.07 2.18 2.18 2.18 7.87 7.87 7.87 7.87 ...
## $ chas : int 0 0 0 0 0 0 0 0 0 0 ...
## $ nox : num 0.538 0.469 0.469 0.458 0.458 0.458 0.524 0.524 0.524 0.524 ...
## $ rm : num 6.58 6.42 7.18 7 7.15 ...
## $ age : num 65.2 78.9 61.1 45.8 54.2 58.7 66.6 96.1 100 85.9 ...
## $ dis : num 4.09 4.97 4.97 6.06 6.06 ...
## $ rad : int 1 2 2 3 3 3 5 5 5 5 ...
## $ tax : num 296 242 242 222 222 222 311 311 311 311 ...
## $ ptratio: num 15.3 17.8 17.8 18.7 18.7 18.7 15.2 15.2 15.2 15.2 ...
## $ black : num 397 397 393 395 397 ...
## $ lstat : num 4.98 9.14 4.03 2.94 5.33 ...
## $ medv : num 24 21.6 34.7 33.4 36.2 28.7 22.9 27.1 16.5 18.9 ...dim(Boston)## [1] 506 14Summary statistics
summary(Boston)## crim zn indus chas
## Min. : 0.00632 Min. : 0.00 Min. : 0.46 Min. :0.00000
## 1st Qu.: 0.08205 1st Qu.: 0.00 1st Qu.: 5.19 1st Qu.:0.00000
## Median : 0.25651 Median : 0.00 Median : 9.69 Median :0.00000
## Mean : 3.61352 Mean : 11.36 Mean :11.14 Mean :0.06917
## 3rd Qu.: 3.67708 3rd Qu.: 12.50 3rd Qu.:18.10 3rd Qu.:0.00000
## Max. :88.97620 Max. :100.00 Max. :27.74 Max. :1.00000
## nox rm age dis
## Min. :0.3850 Min. :3.561 Min. : 2.90 Min. : 1.130
## 1st Qu.:0.4490 1st Qu.:5.886 1st Qu.: 45.02 1st Qu.: 2.100
## Median :0.5380 Median :6.208 Median : 77.50 Median : 3.207
## Mean :0.5547 Mean :6.285 Mean : 68.57 Mean : 3.795
## 3rd Qu.:0.6240 3rd Qu.:6.623 3rd Qu.: 94.08 3rd Qu.: 5.188
## Max. :0.8710 Max. :8.780 Max. :100.00 Max. :12.127
## rad tax ptratio black
## Min. : 1.000 Min. :187.0 Min. :12.60 Min. : 0.32
## 1st Qu.: 4.000 1st Qu.:279.0 1st Qu.:17.40 1st Qu.:375.38
## Median : 5.000 Median :330.0 Median :19.05 Median :391.44
## Mean : 9.549 Mean :408.2 Mean :18.46 Mean :356.67
## 3rd Qu.:24.000 3rd Qu.:666.0 3rd Qu.:20.20 3rd Qu.:396.23
## Max. :24.000 Max. :711.0 Max. :22.00 Max. :396.90
## lstat medv
## Min. : 1.73 Min. : 5.00
## 1st Qu.: 6.95 1st Qu.:17.02
## Median :11.36 Median :21.20
## Mean :12.65 Mean :22.53
## 3rd Qu.:16.95 3rd Qu.:25.00
## Max. :37.97 Max. :50.00table(Boston$rad)##
## 1 2 3 4 5 6 7 8 24
## 20 24 38 110 115 26 17 24 132Some visualization
Visual exploration is an important step in understanding any dataset.
And instead of using basic R functions, we will be using
ggplot this time!
ggplot(Boston, aes(x=medv)) +
geom_histogram(binwidth=5, fill="blue", alpha=0.7) +
ggtitle("Distribution of Median House Value") +
xlab("Median House Value in $1000s") +
ylab("Frequency")
ggplot(Boston, aes(x = rm, y = medv)) +
geom_point() +
labs(title = "Rooms vs Median Value", x = "Average Rooms", y = "Median Value")
ggplot(Boston, aes(x=lstat, y=medv)) +
geom_point(aes(color=rad)) +
ggtitle("Median House Value vs. % Lower Status of the Population") +
xlab("% Lower Status of the Population") +
ylab("Median House Value in $1000s") +
scale_color_continuous(name="Index of Accessibility")
ggplot(Boston, aes(x=factor(rad), y=medv)) +
geom_boxplot(aes(fill=factor(rad))) +
ggtitle("Median House Value by Accessibility to Radial Highways") +
xlab("Index of Accessibility to Radial Highways") +
ylab("Median House Value in $1000s")
Correlation Matrix visualization
corr_matrix <- cor(Boston)
print(corr_matrix)## crim zn indus chas nox
## crim 1.00000000 -0.20046922 0.40658341 -0.055891582 0.42097171
## zn -0.20046922 1.00000000 -0.53382819 -0.042696719 -0.51660371
## indus 0.40658341 -0.53382819 1.00000000 0.062938027 0.76365145
## chas -0.05589158 -0.04269672 0.06293803 1.000000000 0.09120281
## nox 0.42097171 -0.51660371 0.76365145 0.091202807 1.00000000
## rm -0.21924670 0.31199059 -0.39167585 0.091251225 -0.30218819
## age 0.35273425 -0.56953734 0.64477851 0.086517774 0.73147010
## dis -0.37967009 0.66440822 -0.70802699 -0.099175780 -0.76923011
## rad 0.62550515 -0.31194783 0.59512927 -0.007368241 0.61144056
## tax 0.58276431 -0.31456332 0.72076018 -0.035586518 0.66802320
## ptratio 0.28994558 -0.39167855 0.38324756 -0.121515174 0.18893268
## black -0.38506394 0.17552032 -0.35697654 0.048788485 -0.38005064
## lstat 0.45562148 -0.41299457 0.60379972 -0.053929298 0.59087892
## medv -0.38830461 0.36044534 -0.48372516 0.175260177 -0.42732077
## rm age dis rad tax ptratio
## crim -0.21924670 0.35273425 -0.37967009 0.625505145 0.58276431 0.2899456
## zn 0.31199059 -0.56953734 0.66440822 -0.311947826 -0.31456332 -0.3916785
## indus -0.39167585 0.64477851 -0.70802699 0.595129275 0.72076018 0.3832476
## chas 0.09125123 0.08651777 -0.09917578 -0.007368241 -0.03558652 -0.1215152
## nox -0.30218819 0.73147010 -0.76923011 0.611440563 0.66802320 0.1889327
## rm 1.00000000 -0.24026493 0.20524621 -0.209846668 -0.29204783 -0.3555015
## age -0.24026493 1.00000000 -0.74788054 0.456022452 0.50645559 0.2615150
## dis 0.20524621 -0.74788054 1.00000000 -0.494587930 -0.53443158 -0.2324705
## rad -0.20984667 0.45602245 -0.49458793 1.000000000 0.91022819 0.4647412
## tax -0.29204783 0.50645559 -0.53443158 0.910228189 1.00000000 0.4608530
## ptratio -0.35550149 0.26151501 -0.23247054 0.464741179 0.46085304 1.0000000
## black 0.12806864 -0.27353398 0.29151167 -0.444412816 -0.44180801 -0.1773833
## lstat -0.61380827 0.60233853 -0.49699583 0.488676335 0.54399341 0.3740443
## medv 0.69535995 -0.37695457 0.24992873 -0.381626231 -0.46853593 -0.5077867
## black lstat medv
## crim -0.38506394 0.4556215 -0.3883046
## zn 0.17552032 -0.4129946 0.3604453
## indus -0.35697654 0.6037997 -0.4837252
## chas 0.04878848 -0.0539293 0.1752602
## nox -0.38005064 0.5908789 -0.4273208
## rm 0.12806864 -0.6138083 0.6953599
## age -0.27353398 0.6023385 -0.3769546
## dis 0.29151167 -0.4969958 0.2499287
## rad -0.44441282 0.4886763 -0.3816262
## tax -0.44180801 0.5439934 -0.4685359
## ptratio -0.17738330 0.3740443 -0.5077867
## black 1.00000000 -0.3660869 0.3334608
## lstat -0.36608690 1.0000000 -0.7376627
## medv 0.33346082 -0.7376627 1.0000000library(corrplot)## corrplot 0.95 loadedcorrplot(corr_matrix, method="circle", type="upper", order="hclust",
tl.col="black", tl.srt=45)
Pre-processing
The Boston Housing dataset is a well-studied and cleaned data set that often does not require much pre-processing. However, we can always double check!
Note: This historical dataset includes variables that require careful interpretation and should be used here only for teaching EDA and preprocessing techniques.
Check whether there is missing Value for each column
colSums(is.na(Boston))## crim zn indus chas nox rm age dis rad tax
## 0 0 0 0 0 0 0 0 0 0
## ptratio black lstat medv
## 0 0 0 0Outliers, Data Truncation/Winsorization
boxplot(Boston$zn, las=2, cex.axis=0.6)
Looks like some of them are pretty skewed. Let’s truncate
crim and winsorize zn.
Truncate crim at 20, which means removing observations
with crim greater than 20
However, truncation of data usually leads to reduced size, losing information!
# Truncate data based on a specific threshold (20 in this case)
Boston_truncated <- Boston[Boston$crim <= 20, ]
dim(Boston_truncated)## [1] 488 14Let’s double check whether truncation worked.
summary(Boston_truncated$crim)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.00632 0.07937 0.22901 2.34765 2.39617 19.60910Winsorize zn at the 1st percentile and 99th percentile.
In other words, we replace extreme values with less extreme values. This
usually keeps all rows.
# Calculate the 1st and 99th percentiles for 'zn'
lower_bound_zn <- quantile(Boston_truncated$zn, 0.01, na.rm = TRUE)
upper_bound_zn <- quantile(Boston_truncated$zn, 0.99, na.rm = TRUE)
# Winsorize the data, replacing values beyond the thresholds with the thresholds.
Boston_truncated$zn[Boston_truncated$zn < lower_bound_zn] <- lower_bound_zn
Boston_truncated$zn[Boston_truncated$zn > upper_bound_zn] <- upper_bound_znDouble check is a good habit!!
summary(Boston_truncated[,c("crim","zn")])## crim zn
## Min. : 0.00632 Min. : 0.00
## 1st Qu.: 0.07937 1st Qu.: 0.00
## Median : 0.22901 Median : 0.00
## Mean : 2.34765 Mean :11.73
## 3rd Qu.: 2.39617 3rd Qu.:18.50
## Max. :19.60910 Max. :90.65Please pay attention to the data.frame name, we are now working
on Boston_truncated.
Your turn!
Please create a new data.frame named Boston_truncated2
with medv truncated at 45.
# Truncate data based on a specific threshold (45 in this case)
Boston_truncated2 <- Boston_truncated[Boston_truncated$medv <= 45, ]
dim(Boston_truncated2)## [1] 466 14Please winsorize tax at 5 percentile and 95 percentile
on Boston_truncated2 and save it as
Boston_truncated3.
# Start from Boston_truncated2
Boston_truncated3 <- Boston_truncated2
# Calculate the 5th and 95th percentiles for 'tax'
lower_bound_tax <- quantile(Boston_truncated3$tax, 0.05, na.rm = TRUE)
upper_bound_tax <- quantile(Boston_truncated3$tax, 0.95, na.rm = TRUE)
# Winsorize the data, replacing values beyond the thresholds with the thresholds
Boston_truncated3$tax[Boston_truncated3$tax < lower_bound_tax] <- lower_bound_tax
Boston_truncated3$tax[Boston_truncated3$tax > upper_bound_tax] <- upper_bound_taxStandardization
Range Standardization
Scales features to a fixed range, typically [0, 1], which preserves zero values and doesn’t center the data.
Formula: \((x - min(x)) / (max(x) - min(x))\)
Range Standardization is often preferred when you need bounded values, like in neural network inputs or when comparing scores that have different scales but a natural minimum and maximum.
#let's work on the original Boston data again
summary(Boston$crim)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.00632 0.08205 0.25651 3.61352 3.67708 88.97620Add the range standardized crim as a new column.
#Range Standardization for `crim` and saved the range standardized `crim` as a new column
min_crim <- min(Boston$crim)
max_crim <- max(Boston$crim)
Boston$crim_range_standardized <- (Boston$crim - min_crim) / (max_crim - min_crim)
summary(Boston$crim_range_standardized)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.0000000 0.0008511 0.0028121 0.0405441 0.0412585 1.0000000Z-Standardization
Transforms data to have a mean of 0 and standard deviation of 1.
Formula: \((x - mean(x)) / std(x)\)
Z-Standardization is commonly used in statistical analyses and machine learning models where the assumption of normally distributed data is important, or when dealing with outliers.
# Z-Standardization for 'crim'and saved the Z-standardized `crim` as a new column
mean_crim <- mean(Boston$crim)
sd_crim <- sd(Boston$crim)
Boston$crim_z_standardized <- (Boston$crim - mean_crim) / sd_crim
summary(Boston$crim_z_standardized)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -0.419367 -0.410563 -0.390280 0.000000 0.007389 9.924110sd(Boston$crim_z_standardized)## [1] 1Your turn!
Please apply Range standardization on zn and add it as a
new column zn_rangestd.
min_zn <- min(Boston$zn)
max_zn <- max(Boston$zn)
Boston$zn_rangestd <- (Boston$zn - min_zn) / (max_zn - min_zn)
summary(Boston$zn_rangestd)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 0.0000 0.0000 0.0000 0.1136 0.1250 1.0000Please apply Z-standardization on zn and add it as a new
column zn_zstd.
mean_zn <- mean(Boston$zn)
sd_zn <- sd(Boston$zn)
Boston$zn_zstd <- (Boston$zn - mean_zn) / sd_zn
summary(Boston$zn_zstd)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -0.48724 -0.48724 -0.48724 0.00000 0.04872 3.80047