library(dplyr)
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
## filter, lag
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
library(ggplot2)
head(USArrests)
## Murder Assault UrbanPop Rape
## Alabama 13.2 236 58 21.2
## Alaska 10.0 263 48 44.5
## Arizona 8.1 294 80 31.0
## Arkansas 8.8 190 50 19.5
## California 9.0 276 91 40.6
## Colorado 7.9 204 78 38.7
We can see that these features are on different scales. For example, assault tends to be more common than murder, and so it may be beneficial to scale each feature to have unit variance.
We can perform PCA and summarize the results with the commands
pca <- prcomp (USArrests)
summary(pca)
## Importance of components:
## PC1 PC2 PC3 PC4
## Standard deviation 83.7324 14.21240 6.4894 2.48279
## Proportion of Variance 0.9655 0.02782 0.0058 0.00085
## Cumulative Proportion 0.9655 0.99335 0.9991 1.00000
PC1 explains over 95% of the variation in the dataset. PC 1, 2 , 3 and 4 corresponded to each variable
We can obtain the projections of the input data on the PCs as
Z <- pca$x
head(Z)
## PC1 PC2 PC3 PC4
## Alabama 64.80216 -11.448007 -2.4949328 -2.4079009
## Alaska 92.82745 -17.982943 20.1265749 4.0940470
## Arizona 124.06822 8.830403 -1.6874484 4.3536852
## Arkansas 18.34004 -16.703911 0.2101894 0.5209936
## California 107.42295 22.520070 6.7458730 2.8118259
## Colorado 34.97599 13.719584 12.2793628 1.7214637
We can obtain the variance explained by each PC (lambda) as The proportion of variance explained (PVE) for PC 𝑚 is defined as The cumulative PVE (CPVE) by applying summary()
lambda <- diag(var(Z))
lambda
## PC1 PC2 PC3 PC4
## 7011.114851 201.992366 42.112651 6.164246
var(Z) #covariance matrix of the projections
## PC1 PC2 PC3 PC4
## PC1 7.011115e+03 -4.609827e-13 -6.931417e-14 -1.337705e-13
## PC2 -4.609827e-13 2.019924e+02 -2.552153e-14 3.806479e-16
## PC3 -6.931417e-14 -2.552153e-14 4.211265e+01 -5.501268e-15
## PC4 -1.337705e-13 3.806479e-16 -5.501268e-15 6.164246e+00
PVE <- lambda / sum(lambda)
PVE #which is identical to that obtained by applying summary()
## PC1 PC2 PC3 PC4
## 0.9655342206 0.0278173366 0.0057995349 0.0008489079
CPVE <- cumsum(PVE)
CPVE
## PC1 PC2 PC3 PC4
## 0.9655342 0.9933516 0.9991511 1.0000000
Build some graphs
tibble(PC = seq(1, ncol(Z)), PVE = c(PVE)) %>%
ggplot(mapping = aes(x = PC, y = PVE)) + geom_col()
tibble(PC = seq(1, ncol(Z)), CPVE = c(CPVE)) %>%
ggplot(mapping = aes(x = PC, y = CPVE)) + geom_col()
as_tibble(Z) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) + geom_point()
biplot(pca)
IMportance of scaling
pca_scaled <- prcomp(USArrests, scale = TRUE)
summary(pca)
## Importance of components:
## PC1 PC2 PC3 PC4
## Standard deviation 83.7324 14.21240 6.4894 2.48279
## Proportion of Variance 0.9655 0.02782 0.0058 0.00085
## Cumulative Proportion 0.9655 0.99335 0.9991 1.00000
summary(pca_scaled)
## Importance of components:
## PC1 PC2 PC3 PC4
## Standard deviation 1.5749 0.9949 0.59713 0.41645
## Proportion of Variance 0.6201 0.2474 0.08914 0.04336
## Cumulative Proportion 0.6201 0.8675 0.95664 1.00000
which is substantially less skewed toward PC1 than without scaling With scaling, it takes three PCs to explain at least 95% of the variation in the data, whereas it takes only one PC without scaling.
Z_scaled <- pca_scaled$x #projections, different in the scaling than in the data without scaling
head(Z)
## PC1 PC2 PC3 PC4
## Alabama 64.80216 -11.448007 -2.4949328 -2.4079009
## Alaska 92.82745 -17.982943 20.1265749 4.0940470
## Arizona 124.06822 8.830403 -1.6874484 4.3536852
## Arkansas 18.34004 -16.703911 0.2101894 0.5209936
## California 107.42295 22.520070 6.7458730 2.8118259
## Colorado 34.97599 13.719584 12.2793628 1.7214637
head(Z_scaled)
## PC1 PC2 PC3 PC4
## Alabama -0.9756604 -1.1220012 0.43980366 0.154696581
## Alaska -1.9305379 -1.0624269 -2.01950027 -0.434175454
## Arizona -1.7454429 0.7384595 -0.05423025 -0.826264240
## Arkansas 0.1399989 -1.1085423 -0.11342217 -0.180973554
## California -2.4986128 1.5274267 -0.59254100 -0.338559240
## Colorado -1.4993407 0.9776297 -1.08400162 0.001450164
lambda_scaled <- diag(var(Z_scaled))
lambda_scaled
## PC1 PC2 PC3 PC4
## 2.4802416 0.9897652 0.3565632 0.1734301
lambda
## PC1 PC2 PC3 PC4
## 7011.114851 201.992366 42.112651 6.164246
PVE: proportion of variance explained by each PC.
PVE_scaled <- lambda_scaled / sum(lambda_scaled)
PVE_scaled #PC1 and PC2 explain 86% of variance.
## PC1 PC2 PC3 PC4
## 0.62006039 0.24744129 0.08914080 0.04335752
PVE #without scaling the weight of variance is suported in PC1(96%)
## PC1 PC2 PC3 PC4
## 0.9655342206 0.0278173366 0.0057995349 0.0008489079
CPVE: cumulative variance explained
CPVE_scaled <- cumsum(PVE_scaled)
CPVE_scaled # untill PC3 accummulated 95% of explain variance.
## PC1 PC2 PC3 PC4
## 0.6200604 0.8675017 0.9566425 1.0000000
CPVE
## PC1 PC2 PC3 PC4
## 0.9655342 0.9933516 0.9991511 1.0000000
you can see the accumulative PVE in the scaled data and the not scaled. Scaling help to considered more than only one PC to explain the variance.
tibble(PC = seq(1, ncol(Z)), PVE = c(PVE)) %>%
ggplot(mapping = aes(x = PC, y = PVE)) +
geom_col()
tibble(PC = seq(1, ncol(Z_scaled)), PVE = c(PVE_scaled)) %>%
ggplot(mapping = aes(x = PC, y = PVE_scaled)) +
geom_col()
# Consider scaling ….
tibble(PC = seq(1, ncol(Z)), CPVE = c(CPVE)) %>%
ggplot(mapping = aes(x = PC, y = CPVE)) +
geom_col()
tibble(PC = seq(1, ncol(Z_scaled)), CPVE = c(CPVE_scaled)) %>%
ggplot(mapping = aes(x = PC, y = CPVE_scaled)) +
geom_col()
# Visualizing
as_tibble(Z) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) +
geom_point()
as_tibble(Z_scaled) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) +
geom_point() #different to the exaple in th PDF. But we sill seeng he imporatance of sclaing
biplot(pca)
biplot(pca_scaled)
library(ISLR)
head(College)
## Private Apps Accept Enroll Top10perc Top25perc
## Abilene Christian University Yes 1660 1232 721 23 52
## Adelphi University Yes 2186 1924 512 16 29
## Adrian College Yes 1428 1097 336 22 50
## Agnes Scott College Yes 417 349 137 60 89
## Alaska Pacific University Yes 193 146 55 16 44
## Albertson College Yes 587 479 158 38 62
## F.Undergrad P.Undergrad Outstate Room.Board Books
## Abilene Christian University 2885 537 7440 3300 450
## Adelphi University 2683 1227 12280 6450 750
## Adrian College 1036 99 11250 3750 400
## Agnes Scott College 510 63 12960 5450 450
## Alaska Pacific University 249 869 7560 4120 800
## Albertson College 678 41 13500 3335 500
## Personal PhD Terminal S.F.Ratio perc.alumni Expend
## Abilene Christian University 2200 70 78 18.1 12 7041
## Adelphi University 1500 29 30 12.2 16 10527
## Adrian College 1165 53 66 12.9 30 8735
## Agnes Scott College 875 92 97 7.7 37 19016
## Alaska Pacific University 1500 76 72 11.9 2 10922
## Albertson College 675 67 73 9.4 11 9727
## Grad.Rate
## Abilene Christian University 60
## Adelphi University 56
## Adrian College 54
## Agnes Scott College 59
## Alaska Pacific University 15
## Albertson College 55
pca2_scaled <- College %>%
select(-Private) %>%
prcomp(scale = TRUE)
summary(pca2_scaled)
## Importance of components:
## PC1 PC2 PC3 PC4 PC5 PC6 PC7
## Standard deviation 2.3331 2.1161 1.08312 1.00345 0.96593 0.92054 0.77782
## Proportion of Variance 0.3202 0.2634 0.06901 0.05923 0.05488 0.04985 0.03559
## Cumulative Proportion 0.3202 0.5836 0.65262 0.71185 0.76673 0.81658 0.85217
## PC8 PC9 PC10 PC11 PC12 PC13 PC14
## Standard deviation 0.76623 0.72796 0.63544 0.55950 0.46939 0.40936 0.37920
## Proportion of Variance 0.03454 0.03117 0.02375 0.01841 0.01296 0.00986 0.00846
## Cumulative Proportion 0.88670 0.91788 0.94163 0.96004 0.97300 0.98286 0.99132
## PC15 PC16 PC17
## Standard deviation 0.29650 0.19152 0.15165
## Proportion of Variance 0.00517 0.00216 0.00135
## Cumulative Proportion 0.99649 0.99865 1.00000
#First two PCs explain over four times the PVE compared to the third PC, and it takes 11 PCs to explain the proportion at least 95% CPVE
pca2 <- College %>%
select(-Private) %>%
prcomp()
summary(pca2)
## Importance of components:
## PC1 PC2 PC3 PC4 PC5
## Standard deviation 6560.3306 6148.6285 2.499e+03 1.708e+03 1.201e+03
## Proportion of Variance 0.4636 0.4072 6.727e-02 3.144e-02 1.553e-02
## Cumulative Proportion 0.4636 0.8708 9.381e-01 9.695e-01 9.851e-01
## PC6 PC7 PC8 PC9 PC10 PC11
## Standard deviation 788.28658 610.36148 571.94216 198.19578 159.11274 20.64
## Proportion of Variance 0.00669 0.00401 0.00352 0.00042 0.00027 0.00
## Cumulative Proportion 0.99176 0.99577 0.99929 0.99972 0.99999 1.00
## PC12 PC13 PC14 PC15 PC16 PC17
## Standard deviation 14.48 12.55 8.994 5.999 5.195 2.905
## Proportion of Variance 0.00 0.00 0.000 0.000 0.000 0.000
## Cumulative Proportion 1.00 1.00 1.000 1.000 1.000 1.000
Z2_scaled <- pca2_scaled$x
head(Z2_scaled)
## PC1 PC2 PC3 PC4
## Abilene Christian University -1.5918301 -0.76683957 0.1010085 -0.92115606
## Adelphi University -2.1909905 0.57845739 -2.2773312 3.58660802
## Adrian College -1.4300426 1.09211544 0.4378108 0.67680458
## Agnes Scott College 2.8537193 2.62891904 -0.1416309 -1.29465208
## Alaska Pacific University -2.2105837 -0.02161689 -2.3854932 -1.11382101
## Albertson College -0.5712973 1.49536146 -0.0243384 0.06690048
## PC5 PC6 PC7 PC8
## Abilene Christian University 0.7434965 -0.298114059 -0.6380325 0.87882007
## Adelphi University -1.0593143 -0.177023284 -0.2366009 -0.04689512
## Adrian College 0.3693754 -0.959973348 0.2481163 -0.30854175
## Agnes Scott College 0.1837188 -1.058826459 1.2485522 0.14759865
## Alaska Pacific University -0.6840101 0.004914805 2.1578296 0.62401120
## Albertson College 0.3760192 -0.667913283 1.6087992 0.52904990
## PC9 PC10 PC11 PC12
## Abilene Christian University -0.09302403 0.04856147 -0.3994899 0.08963200
## Adelphi University -1.11306351 0.96453235 0.2123721 -0.09717669
## Adrian College 0.10538437 0.64024745 0.1548936 0.34450881
## Agnes Scott College -0.37875338 0.46094732 0.4203800 -0.68670100
## Alaska Pacific University 0.16027963 0.36319428 0.1532407 0.05051983
## Albertson College -0.26335712 -0.69406192 0.1002334 1.09306406
## PC13 PC14 PC15 PC16
## Abilene Christian University 0.05206491 0.180023730 -0.001751267 -0.093080115
## Adelphi University 0.24336085 -0.743725183 -0.103643044 -0.050223339
## Adrian College -0.09748838 0.227380823 0.022543766 -0.004050074
## Agnes Scott College 0.07541215 -0.003377972 0.073133407 -0.191422815
## Alaska Pacific University -0.26703492 -0.614013525 0.273816736 0.010646132
## Albertson College 0.13113350 -0.094391027 -0.220863555 0.056987451
## PC17
## Abilene Christian University 0.093491982
## Adelphi University -0.173945012
## Adrian College 0.003756339
## Agnes Scott College -0.175141641
## Alaska Pacific University 0.048312998
## Albertson College -0.038559813
Z2 <- pca2$x
head(Z2)
## PC1 PC2 PC3 PC4
## Abilene Christian University -2551.8379 -3445.9472 1250.662 -234.1463
## Adelphi University -743.7295 2227.3636 -1009.434 526.9017
## Adrian College -3373.5562 674.2525 -782.013 -358.8200
## Agnes Scott College -1436.8268 9838.7203 3967.336 652.3482
## Alaska Pacific University -4413.8395 546.4212 3663.809 -464.5684
## Albertson College -3935.9789 2868.2670 -1650.577 666.7683
## PC5 PC6 PC7 PC8
## Abilene Christian University -346.9513 468.8914 717.55579 175.40914
## Adelphi University 866.9539 -1610.0110 230.96239 511.32005
## Adrian College -293.5338 665.3054 -69.91512 44.76759
## Agnes Scott College -262.8231 -720.4029 -463.73374 365.56495
## Alaska Pacific University 522.8917 -236.0222 13.07293 232.84488
## Albertson College -517.7020 1338.7965 -435.39931 -305.27532
## PC9 PC10 PC11 PC12
## Abilene Christian University 119.839807 -114.56165 -8.207299 -3.140683
## Adelphi University -72.824411 147.05876 63.861849 38.880976
## Adrian College -3.918174 -116.26003 17.326433 7.563394
## Agnes Scott College -75.626478 -112.91344 -35.133918 -2.170740
## Alaska Pacific University -52.331076 246.18217 11.740005 -27.865705
## Albertson College -80.867390 22.56715 6.861264 -1.489958
## PC13 PC14 PC15 PC16
## Abilene Christian University 0.396452 7.709364 -2.597207 0.3622332
## Adelphi University -14.621668 -2.200664 13.080968 1.0499687
## Adrian College -12.029746 -8.690244 -3.015789 0.1628214
## Agnes Scott College -20.428906 -7.018801 1.118292 2.6271054
## Alaska Pacific University -31.951414 5.219380 7.492396 -6.5180323
## Albertson College -21.798920 15.738840 1.512080 3.2664258
## PC17
## Abilene Christian University 2.74805786
## Adelphi University 0.04335824
## Adrian College -0.19199117
## Agnes Scott College -1.74512531
## Alaska Pacific University -2.47945293
## Albertson College -3.86510389
lambda2_scaled <- diag(var(Z2_scaled))
lambda2_scaled
## PC1 PC2 PC3 PC4 PC5 PC6 PC7
## 5.44350679 4.47783645 1.17315581 1.00690817 0.93302887 0.84739916 0.60500815
## PC8 PC9 PC10 PC11 PC12 PC13 PC14
## 0.58711563 0.52992973 0.40378256 0.31304247 0.22032704 0.16757820 0.14379320
## PC15 PC16 PC17
## 0.08791135 0.03667818 0.02299823
lambda2 <- diag(var(Z2))
lambda2
## PC1 PC2 PC3 PC4 PC5 PC6
## 4.303794e+07 3.780563e+07 6.244929e+06 2.918527e+06 1.442148e+06 6.213957e+05
## PC7 PC8 PC9 PC10 PC11 PC12
## 3.725411e+05 3.271178e+05 3.928157e+04 2.531686e+04 4.262041e+02 2.095555e+02
## PC13 PC14 PC15 PC16 PC17
## 1.573910e+02 8.089572e+01 3.598291e+01 2.698442e+01 8.438281e+00
PVE2_scaled <- lambda2_scaled / sum(lambda2_scaled)
PVE2_scaled
## PC1 PC2 PC3 PC4 PC5 PC6
## 0.320206282 0.263402144 0.069009166 0.059229892 0.054884051 0.049847010
## PC7 PC8 PC9 PC10 PC11 PC12
## 0.035588715 0.034536213 0.031172337 0.023751915 0.018414263 0.012960414
## PC13 PC14 PC15 PC16 PC17
## 0.009857541 0.008458423 0.005171256 0.002157540 0.001352837
PVE2<- lambda2 / sum(lambda2)
PVE2
## PC1 PC2 PC3 PC4 PC5 PC6
## 4.635922e-01 4.072313e-01 6.726856e-02 3.143752e-02 1.553440e-02 6.693494e-03
## PC7 PC8 PC9 PC10 PC11 PC12
## 4.012905e-03 3.523618e-03 4.231297e-04 2.727059e-04 4.590947e-06 2.257271e-06
## PC13 PC14 PC15 PC16 PC17
## 1.695370e-06 8.713853e-07 3.875975e-07 2.906683e-07 9.089471e-08
CPVE2_scaled <- cumsum(PVE2_scaled)
CPVE2_scaled
## PC1 PC2 PC3 PC4 PC5 PC6 PC7 PC8
## 0.3202063 0.5836084 0.6526176 0.7118475 0.7667315 0.8165785 0.8521673 0.8867035
## PC9 PC10 PC11 PC12 PC13 PC14 PC15 PC16
## 0.9178758 0.9416277 0.9600420 0.9730024 0.9828599 0.9913184 0.9964896 0.9986472
## PC17
## 1.0000000
CPVE2 <- cumsum(PVE2)
CPVE2
## PC1 PC2 PC3 PC4 PC5 PC6 PC7 PC8
## 0.4635922 0.8708235 0.9380920 0.9695296 0.9850640 0.9917575 0.9957704 0.9992940
## PC9 PC10 PC11 PC12 PC13 PC14 PC15 PC16
## 0.9997171 0.9999898 0.9999944 0.9999967 0.9999984 0.9999992 0.9999996 0.9999999
## PC17
## 1.0000000
tibble(PC = seq(1, ncol(Z2_scaled)), PVE = c(PVE2_scaled)) %>%
ggplot(mapping = aes(x = PC, y = PVE2_scaled)) +
geom_col()
tibble(PC = seq(1, ncol(Z2)), PVE = c(PVE2)) %>%
ggplot(mapping = aes(x = PC, y = PVE2)) +
geom_col()
tibble(PC = seq(1, ncol(Z2_scaled)), CPVE = c(CPVE2_scaled)) %>%
ggplot(mapping = aes(x = PC, y = CPVE2_scaled)) +
geom_col()
tibble(PC = seq(1, ncol(Z2)), CPVE = c(CPVE2)) %>%
ggplot(mapping = aes(x = PC, y = CPVE2)) +
geom_col()
“Z” is the projections of the input data in the PCA
Z.withPrivate_scaled <- as_tibble(Z2_scaled) %>%
mutate(Private = College$Private) %>%
select(Private, everything())
Z.withPrivate_scaled %>%
ggplot(mapping = aes(x = PC1, y = PC2, color=Private)) +
geom_point()
Z.withPrivate_not_scaled <- as_tibble(Z2) %>%
mutate(Private = College$Private) %>%
select(Private, everything())
Z.withPrivate_not_scaled %>%
ggplot(mapping = aes(x = PC1, y = PC2, color=Private)) +
geom_point()
Comparing scaling and not scaling
biplot(pca2_scaled)
biplot(pca2)