Important packages

library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(ggplot2)

EXAMPLE 1

Data

head(USArrests)
##            Murder Assault UrbanPop Rape
## Alabama      13.2     236       58 21.2
## Alaska       10.0     263       48 44.5
## Arizona       8.1     294       80 31.0
## Arkansas      8.8     190       50 19.5
## California    9.0     276       91 40.6
## Colorado      7.9     204       78 38.7

We can see that these features are on different scales. For example, assault tends to be more common than murder, and so it may be beneficial to scale each feature to have unit variance.

We can perform PCA and summarize the results with the commands

pca <- prcomp (USArrests)
summary(pca)
## Importance of components:
##                            PC1      PC2    PC3     PC4
## Standard deviation     83.7324 14.21240 6.4894 2.48279
## Proportion of Variance  0.9655  0.02782 0.0058 0.00085
## Cumulative Proportion   0.9655  0.99335 0.9991 1.00000

PC1 explains over 95% of the variation in the dataset. PC 1, 2 , 3 and 4 corresponded to each variable

We can obtain the projections of the input data on the PCs as

Z <- pca$x
head(Z)
##                  PC1        PC2        PC3        PC4
## Alabama     64.80216 -11.448007 -2.4949328 -2.4079009
## Alaska      92.82745 -17.982943 20.1265749  4.0940470
## Arizona    124.06822   8.830403 -1.6874484  4.3536852
## Arkansas    18.34004 -16.703911  0.2101894  0.5209936
## California 107.42295  22.520070  6.7458730  2.8118259
## Colorado    34.97599  13.719584 12.2793628  1.7214637

We can obtain the variance explained by each PC (lambda) as The proportion of variance explained (PVE) for PC 𝑚 is defined as The cumulative PVE (CPVE) by applying summary()

lambda <- diag(var(Z)) 
lambda
##         PC1         PC2         PC3         PC4 
## 7011.114851  201.992366   42.112651    6.164246
var(Z) #covariance matrix of the projections
##               PC1           PC2           PC3           PC4
## PC1  7.011115e+03 -4.609827e-13 -6.931417e-14 -1.337705e-13
## PC2 -4.609827e-13  2.019924e+02 -2.552153e-14  3.806479e-16
## PC3 -6.931417e-14 -2.552153e-14  4.211265e+01 -5.501268e-15
## PC4 -1.337705e-13  3.806479e-16 -5.501268e-15  6.164246e+00
PVE <- lambda / sum(lambda) 
PVE #which is identical to that obtained by applying summary()
##          PC1          PC2          PC3          PC4 
## 0.9655342206 0.0278173366 0.0057995349 0.0008489079
CPVE <- cumsum(PVE)
CPVE
##       PC1       PC2       PC3       PC4 
## 0.9655342 0.9933516 0.9991511 1.0000000

Build some graphs

tibble(PC = seq(1, ncol(Z)), PVE = c(PVE)) %>% 
  ggplot(mapping = aes(x = PC, y = PVE)) + geom_col()

tibble(PC = seq(1, ncol(Z)), CPVE = c(CPVE)) %>% 
  ggplot(mapping = aes(x = PC, y = CPVE)) + geom_col()

as_tibble(Z) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) + geom_point()

biplot(pca)

IMportance of scaling

pca_scaled <- prcomp(USArrests, scale = TRUE) 
summary(pca)
## Importance of components:
##                            PC1      PC2    PC3     PC4
## Standard deviation     83.7324 14.21240 6.4894 2.48279
## Proportion of Variance  0.9655  0.02782 0.0058 0.00085
## Cumulative Proportion   0.9655  0.99335 0.9991 1.00000
summary(pca_scaled)
## Importance of components:
##                           PC1    PC2     PC3     PC4
## Standard deviation     1.5749 0.9949 0.59713 0.41645
## Proportion of Variance 0.6201 0.2474 0.08914 0.04336
## Cumulative Proportion  0.6201 0.8675 0.95664 1.00000

which is substantially less skewed toward PC1 than without scaling With scaling, it takes three PCs to explain at least 95% of the variation in the data, whereas it takes only one PC without scaling.

Z_scaled <- pca_scaled$x #projections, different in the scaling than in the data without scaling
head(Z)
##                  PC1        PC2        PC3        PC4
## Alabama     64.80216 -11.448007 -2.4949328 -2.4079009
## Alaska      92.82745 -17.982943 20.1265749  4.0940470
## Arizona    124.06822   8.830403 -1.6874484  4.3536852
## Arkansas    18.34004 -16.703911  0.2101894  0.5209936
## California 107.42295  22.520070  6.7458730  2.8118259
## Colorado    34.97599  13.719584 12.2793628  1.7214637
head(Z_scaled)
##                   PC1        PC2         PC3          PC4
## Alabama    -0.9756604 -1.1220012  0.43980366  0.154696581
## Alaska     -1.9305379 -1.0624269 -2.01950027 -0.434175454
## Arizona    -1.7454429  0.7384595 -0.05423025 -0.826264240
## Arkansas    0.1399989 -1.1085423 -0.11342217 -0.180973554
## California -2.4986128  1.5274267 -0.59254100 -0.338559240
## Colorado   -1.4993407  0.9776297 -1.08400162  0.001450164
lambda_scaled <- diag(var(Z_scaled))
lambda_scaled
##       PC1       PC2       PC3       PC4 
## 2.4802416 0.9897652 0.3565632 0.1734301
lambda
##         PC1         PC2         PC3         PC4 
## 7011.114851  201.992366   42.112651    6.164246

PVE: proportion of variance explained by each PC.

PVE_scaled <- lambda_scaled / sum(lambda_scaled)
PVE_scaled #PC1 and PC2 explain 86% of variance. 
##        PC1        PC2        PC3        PC4 
## 0.62006039 0.24744129 0.08914080 0.04335752
PVE #without scaling the weight of variance is suported in PC1(96%)
##          PC1          PC2          PC3          PC4 
## 0.9655342206 0.0278173366 0.0057995349 0.0008489079

CPVE: cumulative variance explained

CPVE_scaled <- cumsum(PVE_scaled)
CPVE_scaled # untill PC3 accummulated 95% of explain variance. 
##       PC1       PC2       PC3       PC4 
## 0.6200604 0.8675017 0.9566425 1.0000000
CPVE
##       PC1       PC2       PC3       PC4 
## 0.9655342 0.9933516 0.9991511 1.0000000

PVE and CPVE

you can see the accumulative PVE in the scaled data and the not scaled. Scaling help to considered more than only one PC to explain the variance.

tibble(PC = seq(1, ncol(Z)), PVE = c(PVE)) %>% 
  ggplot(mapping = aes(x = PC, y = PVE)) + 
  geom_col()

tibble(PC = seq(1, ncol(Z_scaled)), PVE = c(PVE_scaled)) %>% 
  ggplot(mapping = aes(x = PC, y = PVE_scaled)) + 
  geom_col()

# Consider scaling ….

tibble(PC = seq(1, ncol(Z)), CPVE = c(CPVE)) %>% 
  ggplot(mapping = aes(x = PC, y = CPVE)) +
  geom_col()

tibble(PC = seq(1, ncol(Z_scaled)), CPVE = c(CPVE_scaled)) %>% 
  ggplot(mapping = aes(x = PC, y = CPVE_scaled)) +
  geom_col() 

# Visualizing

as_tibble(Z) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) + 
geom_point()

as_tibble(Z_scaled) %>%
ggplot(mapping = aes(x = PC1, y = PC2)) + 
geom_point() #different to the exaple in th PDF. But we sill seeng he imporatance of sclaing 

biplot(pca)

biplot(pca_scaled)

EXAMPLE 2

library(ISLR)
head(College)
##                              Private Apps Accept Enroll Top10perc Top25perc
## Abilene Christian University     Yes 1660   1232    721        23        52
## Adelphi University               Yes 2186   1924    512        16        29
## Adrian College                   Yes 1428   1097    336        22        50
## Agnes Scott College              Yes  417    349    137        60        89
## Alaska Pacific University        Yes  193    146     55        16        44
## Albertson College                Yes  587    479    158        38        62
##                              F.Undergrad P.Undergrad Outstate Room.Board Books
## Abilene Christian University        2885         537     7440       3300   450
## Adelphi University                  2683        1227    12280       6450   750
## Adrian College                      1036          99    11250       3750   400
## Agnes Scott College                  510          63    12960       5450   450
## Alaska Pacific University            249         869     7560       4120   800
## Albertson College                    678          41    13500       3335   500
##                              Personal PhD Terminal S.F.Ratio perc.alumni Expend
## Abilene Christian University     2200  70       78      18.1          12   7041
## Adelphi University               1500  29       30      12.2          16  10527
## Adrian College                   1165  53       66      12.9          30   8735
## Agnes Scott College               875  92       97       7.7          37  19016
## Alaska Pacific University        1500  76       72      11.9           2  10922
## Albertson College                 675  67       73       9.4          11   9727
##                              Grad.Rate
## Abilene Christian University        60
## Adelphi University                  56
## Adrian College                      54
## Agnes Scott College                 59
## Alaska Pacific University           15
## Albertson College                   55
pca2_scaled <- College %>% 
  select(-Private) %>% 
  prcomp(scale = TRUE)
summary(pca2_scaled) 
## Importance of components:
##                           PC1    PC2     PC3     PC4     PC5     PC6     PC7
## Standard deviation     2.3331 2.1161 1.08312 1.00345 0.96593 0.92054 0.77782
## Proportion of Variance 0.3202 0.2634 0.06901 0.05923 0.05488 0.04985 0.03559
## Cumulative Proportion  0.3202 0.5836 0.65262 0.71185 0.76673 0.81658 0.85217
##                            PC8     PC9    PC10    PC11    PC12    PC13    PC14
## Standard deviation     0.76623 0.72796 0.63544 0.55950 0.46939 0.40936 0.37920
## Proportion of Variance 0.03454 0.03117 0.02375 0.01841 0.01296 0.00986 0.00846
## Cumulative Proportion  0.88670 0.91788 0.94163 0.96004 0.97300 0.98286 0.99132
##                           PC15    PC16    PC17
## Standard deviation     0.29650 0.19152 0.15165
## Proportion of Variance 0.00517 0.00216 0.00135
## Cumulative Proportion  0.99649 0.99865 1.00000
#First two PCs explain over four times the PVE compared to the third PC, and it takes 11 PCs to explain the proportion at least 95% CPVE
pca2 <- College %>% 
  select(-Private) %>% 
  prcomp()
summary(pca2)
## Importance of components:
##                              PC1       PC2       PC3       PC4       PC5
## Standard deviation     6560.3306 6148.6285 2.499e+03 1.708e+03 1.201e+03
## Proportion of Variance    0.4636    0.4072 6.727e-02 3.144e-02 1.553e-02
## Cumulative Proportion     0.4636    0.8708 9.381e-01 9.695e-01 9.851e-01
##                              PC6       PC7       PC8       PC9      PC10  PC11
## Standard deviation     788.28658 610.36148 571.94216 198.19578 159.11274 20.64
## Proportion of Variance   0.00669   0.00401   0.00352   0.00042   0.00027  0.00
## Cumulative Proportion    0.99176   0.99577   0.99929   0.99972   0.99999  1.00
##                         PC12  PC13  PC14  PC15  PC16  PC17
## Standard deviation     14.48 12.55 8.994 5.999 5.195 2.905
## Proportion of Variance  0.00  0.00 0.000 0.000 0.000 0.000
## Cumulative Proportion   1.00  1.00 1.000 1.000 1.000 1.000
Z2_scaled <- pca2_scaled$x 
head(Z2_scaled)
##                                     PC1         PC2        PC3         PC4
## Abilene Christian University -1.5918301 -0.76683957  0.1010085 -0.92115606
## Adelphi University           -2.1909905  0.57845739 -2.2773312  3.58660802
## Adrian College               -1.4300426  1.09211544  0.4378108  0.67680458
## Agnes Scott College           2.8537193  2.62891904 -0.1416309 -1.29465208
## Alaska Pacific University    -2.2105837 -0.02161689 -2.3854932 -1.11382101
## Albertson College            -0.5712973  1.49536146 -0.0243384  0.06690048
##                                     PC5          PC6        PC7         PC8
## Abilene Christian University  0.7434965 -0.298114059 -0.6380325  0.87882007
## Adelphi University           -1.0593143 -0.177023284 -0.2366009 -0.04689512
## Adrian College                0.3693754 -0.959973348  0.2481163 -0.30854175
## Agnes Scott College           0.1837188 -1.058826459  1.2485522  0.14759865
## Alaska Pacific University    -0.6840101  0.004914805  2.1578296  0.62401120
## Albertson College             0.3760192 -0.667913283  1.6087992  0.52904990
##                                      PC9        PC10       PC11        PC12
## Abilene Christian University -0.09302403  0.04856147 -0.3994899  0.08963200
## Adelphi University           -1.11306351  0.96453235  0.2123721 -0.09717669
## Adrian College                0.10538437  0.64024745  0.1548936  0.34450881
## Agnes Scott College          -0.37875338  0.46094732  0.4203800 -0.68670100
## Alaska Pacific University     0.16027963  0.36319428  0.1532407  0.05051983
## Albertson College            -0.26335712 -0.69406192  0.1002334  1.09306406
##                                     PC13         PC14         PC15         PC16
## Abilene Christian University  0.05206491  0.180023730 -0.001751267 -0.093080115
## Adelphi University            0.24336085 -0.743725183 -0.103643044 -0.050223339
## Adrian College               -0.09748838  0.227380823  0.022543766 -0.004050074
## Agnes Scott College           0.07541215 -0.003377972  0.073133407 -0.191422815
## Alaska Pacific University    -0.26703492 -0.614013525  0.273816736  0.010646132
## Albertson College             0.13113350 -0.094391027 -0.220863555  0.056987451
##                                      PC17
## Abilene Christian University  0.093491982
## Adelphi University           -0.173945012
## Adrian College                0.003756339
## Agnes Scott College          -0.175141641
## Alaska Pacific University     0.048312998
## Albertson College            -0.038559813
Z2 <- pca2$x 
head(Z2)
##                                     PC1        PC2       PC3       PC4
## Abilene Christian University -2551.8379 -3445.9472  1250.662 -234.1463
## Adelphi University            -743.7295  2227.3636 -1009.434  526.9017
## Adrian College               -3373.5562   674.2525  -782.013 -358.8200
## Agnes Scott College          -1436.8268  9838.7203  3967.336  652.3482
## Alaska Pacific University    -4413.8395   546.4212  3663.809 -464.5684
## Albertson College            -3935.9789  2868.2670 -1650.577  666.7683
##                                    PC5        PC6        PC7        PC8
## Abilene Christian University -346.9513   468.8914  717.55579  175.40914
## Adelphi University            866.9539 -1610.0110  230.96239  511.32005
## Adrian College               -293.5338   665.3054  -69.91512   44.76759
## Agnes Scott College          -262.8231  -720.4029 -463.73374  365.56495
## Alaska Pacific University     522.8917  -236.0222   13.07293  232.84488
## Albertson College            -517.7020  1338.7965 -435.39931 -305.27532
##                                     PC9       PC10       PC11       PC12
## Abilene Christian University 119.839807 -114.56165  -8.207299  -3.140683
## Adelphi University           -72.824411  147.05876  63.861849  38.880976
## Adrian College                -3.918174 -116.26003  17.326433   7.563394
## Agnes Scott College          -75.626478 -112.91344 -35.133918  -2.170740
## Alaska Pacific University    -52.331076  246.18217  11.740005 -27.865705
## Albertson College            -80.867390   22.56715   6.861264  -1.489958
##                                    PC13      PC14      PC15       PC16
## Abilene Christian University   0.396452  7.709364 -2.597207  0.3622332
## Adelphi University           -14.621668 -2.200664 13.080968  1.0499687
## Adrian College               -12.029746 -8.690244 -3.015789  0.1628214
## Agnes Scott College          -20.428906 -7.018801  1.118292  2.6271054
## Alaska Pacific University    -31.951414  5.219380  7.492396 -6.5180323
## Albertson College            -21.798920 15.738840  1.512080  3.2664258
##                                     PC17
## Abilene Christian University  2.74805786
## Adelphi University            0.04335824
## Adrian College               -0.19199117
## Agnes Scott College          -1.74512531
## Alaska Pacific University    -2.47945293
## Albertson College            -3.86510389
lambda2_scaled <- diag(var(Z2_scaled)) 
lambda2_scaled
##        PC1        PC2        PC3        PC4        PC5        PC6        PC7 
## 5.44350679 4.47783645 1.17315581 1.00690817 0.93302887 0.84739916 0.60500815 
##        PC8        PC9       PC10       PC11       PC12       PC13       PC14 
## 0.58711563 0.52992973 0.40378256 0.31304247 0.22032704 0.16757820 0.14379320 
##       PC15       PC16       PC17 
## 0.08791135 0.03667818 0.02299823
lambda2 <- diag(var(Z2))
lambda2
##          PC1          PC2          PC3          PC4          PC5          PC6 
## 4.303794e+07 3.780563e+07 6.244929e+06 2.918527e+06 1.442148e+06 6.213957e+05 
##          PC7          PC8          PC9         PC10         PC11         PC12 
## 3.725411e+05 3.271178e+05 3.928157e+04 2.531686e+04 4.262041e+02 2.095555e+02 
##         PC13         PC14         PC15         PC16         PC17 
## 1.573910e+02 8.089572e+01 3.598291e+01 2.698442e+01 8.438281e+00
PVE2_scaled <- lambda2_scaled / sum(lambda2_scaled)
PVE2_scaled
##         PC1         PC2         PC3         PC4         PC5         PC6 
## 0.320206282 0.263402144 0.069009166 0.059229892 0.054884051 0.049847010 
##         PC7         PC8         PC9        PC10        PC11        PC12 
## 0.035588715 0.034536213 0.031172337 0.023751915 0.018414263 0.012960414 
##        PC13        PC14        PC15        PC16        PC17 
## 0.009857541 0.008458423 0.005171256 0.002157540 0.001352837
PVE2<- lambda2 / sum(lambda2)
PVE2
##          PC1          PC2          PC3          PC4          PC5          PC6 
## 4.635922e-01 4.072313e-01 6.726856e-02 3.143752e-02 1.553440e-02 6.693494e-03 
##          PC7          PC8          PC9         PC10         PC11         PC12 
## 4.012905e-03 3.523618e-03 4.231297e-04 2.727059e-04 4.590947e-06 2.257271e-06 
##         PC13         PC14         PC15         PC16         PC17 
## 1.695370e-06 8.713853e-07 3.875975e-07 2.906683e-07 9.089471e-08
CPVE2_scaled <- cumsum(PVE2_scaled)
CPVE2_scaled
##       PC1       PC2       PC3       PC4       PC5       PC6       PC7       PC8 
## 0.3202063 0.5836084 0.6526176 0.7118475 0.7667315 0.8165785 0.8521673 0.8867035 
##       PC9      PC10      PC11      PC12      PC13      PC14      PC15      PC16 
## 0.9178758 0.9416277 0.9600420 0.9730024 0.9828599 0.9913184 0.9964896 0.9986472 
##      PC17 
## 1.0000000
CPVE2 <- cumsum(PVE2)
CPVE2
##       PC1       PC2       PC3       PC4       PC5       PC6       PC7       PC8 
## 0.4635922 0.8708235 0.9380920 0.9695296 0.9850640 0.9917575 0.9957704 0.9992940 
##       PC9      PC10      PC11      PC12      PC13      PC14      PC15      PC16 
## 0.9997171 0.9999898 0.9999944 0.9999967 0.9999984 0.9999992 0.9999996 0.9999999 
##      PC17 
## 1.0000000
tibble(PC = seq(1, ncol(Z2_scaled)), PVE = c(PVE2_scaled)) %>% 
  ggplot(mapping = aes(x = PC, y = PVE2_scaled)) + 
  geom_col()

tibble(PC = seq(1, ncol(Z2)), PVE = c(PVE2)) %>% 
  ggplot(mapping = aes(x = PC, y = PVE2)) + 
  geom_col()

tibble(PC = seq(1, ncol(Z2_scaled)), CPVE = c(CPVE2_scaled)) %>% 
  ggplot(mapping = aes(x = PC, y = CPVE2_scaled)) + 
  geom_col()

tibble(PC = seq(1, ncol(Z2)), CPVE = c(CPVE2)) %>% 
  ggplot(mapping = aes(x = PC, y = CPVE2)) + 
  geom_col()

projection onto PCs1 and 2

“Z” is the projections of the input data in the PCA

Z.withPrivate_scaled <- as_tibble(Z2_scaled) %>% 
  mutate(Private = College$Private) %>% 
  select(Private, everything())
Z.withPrivate_scaled %>%
ggplot(mapping = aes(x = PC1, y = PC2, color=Private)) + 
  geom_point()

Z.withPrivate_not_scaled <- as_tibble(Z2) %>% 
  mutate(Private = College$Private) %>% 
  select(Private, everything())
Z.withPrivate_not_scaled %>%
ggplot(mapping = aes(x = PC1, y = PC2, color=Private)) + 
  geom_point()

Comparing scaling and not scaling

biplot(pca2_scaled)

biplot(pca2)