Main Goal: Practice using R to implement a statistical method to analyze multivariate data. This slides shows you how to use the R statistical software to carry out some simple multivariate analyses, with a focus on principal components analysis (PCA). PCA is used in exploratory data analysis and for making decisions in predictive models.
Multi -> Multiple
Variate -> Variables
Multivariate -> Multiple Variables
wine <- read.table("http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data", sep=",")
# library(tidyverse)
# library(gapminder)
data(gapminder)
names(gapminder)## [1] "country" "continent" "year" "lifeExp" "pop" "gdpPercap"
## [1] "mpg" "cyl" "disp" "hp" "drat" "wt" "qsec" "vs" "am" "gear"
## [11] "carb"
## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 23.60 48.20 60.71 59.47 70.85 82.60
## [1] 0.5837062
ggplot(data = gapminder,
aes(x = year, y = lifeExp, group = country, color = continent)) +
geom_line(size = 0.4) +
geom_smooth(aes(group = continent),
se = FALSE, size = 1.5,
color = "black",
method = "loess") +
facet_wrap(~ continent) ## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## `geom_smooth()` using formula = 'y ~ x'
# library(car)
# wine <- read.table("http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine # .data", sep=",")
head(wine) ## V1 V2 V3 V4 V5 V6 V7 V8 V9 V10 V11 V12 V13 V14
## 1 1 14.23 1.71 2.43 15.6 127 2.80 3.06 0.28 2.29 5.64 1.04 3.92 1065
## 2 1 13.20 1.78 2.14 11.2 100 2.65 2.76 0.26 1.28 4.38 1.05 3.40 1050
## 3 1 13.16 2.36 2.67 18.6 101 2.80 3.24 0.30 2.81 5.68 1.03 3.17 1185
## 4 1 14.37 1.95 2.50 16.8 113 3.85 3.49 0.24 2.18 7.80 0.86 3.45 1480
## 5 1 13.24 2.59 2.87 21.0 118 2.80 2.69 0.39 1.82 4.32 1.04 2.93 735
## 6 1 14.20 1.76 2.45 15.2 112 3.27 3.39 0.34 1.97 6.75 1.05 2.85 1450
## V2 V3 V4 V5 V6
## 1 14.23 1.71 2.43 15.6 127
## 2 13.20 1.78 2.14 11.2 100
## 3 13.16 2.36 2.67 18.6 101
## 4 14.37 1.95 2.50 16.8 113
## 5 13.24 2.59 2.87 21.0 118
## 6 14.20 1.76 2.45 15.2 112
## 7 14.39 1.87 2.45 14.6 96
## 8 14.06 2.15 2.61 17.6 121
## 9 14.83 1.64 2.17 14.0 97
## 10 13.86 1.35 2.27 16.0 98
## 11 14.10 2.16 2.30 18.0 105
## 12 14.12 1.48 2.32 16.8 95
## 13 13.75 1.73 2.41 16.0 89
## 14 14.75 1.73 2.39 11.4 91
## 15 14.38 1.87 2.38 12.0 102
## 16 13.63 1.81 2.70 17.2 112
## 17 14.30 1.92 2.72 20.0 120
## 18 13.83 1.57 2.62 20.0 115
## 19 14.19 1.59 2.48 16.5 108
## 20 13.64 3.10 2.56 15.2 116
## 21 14.06 1.63 2.28 16.0 126
## 22 12.93 3.80 2.65 18.6 102
## 23 13.71 1.86 2.36 16.6 101
## 24 12.85 1.60 2.52 17.8 95
## 25 13.50 1.81 2.61 20.0 96
## 26 13.05 2.05 3.22 25.0 124
## 27 13.39 1.77 2.62 16.1 93
## 28 13.30 1.72 2.14 17.0 94
## 29 13.87 1.90 2.80 19.4 107
## 30 14.02 1.68 2.21 16.0 96
## 31 13.73 1.50 2.70 22.5 101
## 32 13.58 1.66 2.36 19.1 106
## 33 13.68 1.83 2.36 17.2 104
## 34 13.76 1.53 2.70 19.5 132
## 35 13.51 1.80 2.65 19.0 110
## 36 13.48 1.81 2.41 20.5 100
## 37 13.28 1.64 2.84 15.5 110
## 38 13.05 1.65 2.55 18.0 98
## 39 13.07 1.50 2.10 15.5 98
## 40 14.22 3.99 2.51 13.2 128
## 41 13.56 1.71 2.31 16.2 117
## 42 13.41 3.84 2.12 18.8 90
## 43 13.88 1.89 2.59 15.0 101
## 44 13.24 3.98 2.29 17.5 103
## 45 13.05 1.77 2.10 17.0 107
## 46 14.21 4.04 2.44 18.9 111
## 47 14.38 3.59 2.28 16.0 102
## 48 13.90 1.68 2.12 16.0 101
## 49 14.10 2.02 2.40 18.8 103
## 50 13.94 1.73 2.27 17.4 108
## 51 13.05 1.73 2.04 12.4 92
## 52 13.83 1.65 2.60 17.2 94
## 53 13.82 1.75 2.42 14.0 111
## 54 13.77 1.90 2.68 17.1 115
## 55 13.74 1.67 2.25 16.4 118
## 56 13.56 1.73 2.46 20.5 116
## 57 14.22 1.70 2.30 16.3 118
## 58 13.29 1.97 2.68 16.8 102
## 59 13.72 1.43 2.50 16.7 108
## 60 12.37 0.94 1.36 10.6 88
## 61 12.33 1.10 2.28 16.0 101
## 62 12.64 1.36 2.02 16.8 100
## 63 13.67 1.25 1.92 18.0 94
## 64 12.37 1.13 2.16 19.0 87
## 65 12.17 1.45 2.53 19.0 104
## 66 12.37 1.21 2.56 18.1 98
## 67 13.11 1.01 1.70 15.0 78
## 68 12.37 1.17 1.92 19.6 78
## 69 13.34 0.94 2.36 17.0 110
## 70 12.21 1.19 1.75 16.8 151
## 71 12.29 1.61 2.21 20.4 103
## 72 13.86 1.51 2.67 25.0 86
## 73 13.49 1.66 2.24 24.0 87
## 74 12.99 1.67 2.60 30.0 139
## 75 11.96 1.09 2.30 21.0 101
## 76 11.66 1.88 1.92 16.0 97
## 77 13.03 0.90 1.71 16.0 86
## 78 11.84 2.89 2.23 18.0 112
## 79 12.33 0.99 1.95 14.8 136
## 80 12.70 3.87 2.40 23.0 101
## 81 12.00 0.92 2.00 19.0 86
## 82 12.72 1.81 2.20 18.8 86
## 83 12.08 1.13 2.51 24.0 78
## 84 13.05 3.86 2.32 22.5 85
## 85 11.84 0.89 2.58 18.0 94
## 86 12.67 0.98 2.24 18.0 99
## 87 12.16 1.61 2.31 22.8 90
## 88 11.65 1.67 2.62 26.0 88
## 89 11.64 2.06 2.46 21.6 84
## 90 12.08 1.33 2.30 23.6 70
## 91 12.08 1.83 2.32 18.5 81
## 92 12.00 1.51 2.42 22.0 86
## 93 12.69 1.53 2.26 20.7 80
## 94 12.29 2.83 2.22 18.0 88
## 95 11.62 1.99 2.28 18.0 98
## 96 12.47 1.52 2.20 19.0 162
## 97 11.81 2.12 2.74 21.5 134
## 98 12.29 1.41 1.98 16.0 85
## 99 12.37 1.07 2.10 18.5 88
## 100 12.29 3.17 2.21 18.0 88
## 101 12.08 2.08 1.70 17.5 97
## 102 12.60 1.34 1.90 18.5 88
## 103 12.34 2.45 2.46 21.0 98
## 104 11.82 1.72 1.88 19.5 86
## 105 12.51 1.73 1.98 20.5 85
## 106 12.42 2.55 2.27 22.0 90
## 107 12.25 1.73 2.12 19.0 80
## 108 12.72 1.75 2.28 22.5 84
## 109 12.22 1.29 1.94 19.0 92
## 110 11.61 1.35 2.70 20.0 94
## 111 11.46 3.74 1.82 19.5 107
## 112 12.52 2.43 2.17 21.0 88
## 113 11.76 2.68 2.92 20.0 103
## 114 11.41 0.74 2.50 21.0 88
## 115 12.08 1.39 2.50 22.5 84
## 116 11.03 1.51 2.20 21.5 85
## 117 11.82 1.47 1.99 20.8 86
## 118 12.42 1.61 2.19 22.5 108
## 119 12.77 3.43 1.98 16.0 80
## 120 12.00 3.43 2.00 19.0 87
## 121 11.45 2.40 2.42 20.0 96
## 122 11.56 2.05 3.23 28.5 119
## 123 12.42 4.43 2.73 26.5 102
## 124 13.05 5.80 2.13 21.5 86
## 125 11.87 4.31 2.39 21.0 82
## 126 12.07 2.16 2.17 21.0 85
## 127 12.43 1.53 2.29 21.5 86
## 128 11.79 2.13 2.78 28.5 92
## 129 12.37 1.63 2.30 24.5 88
## 130 12.04 4.30 2.38 22.0 80
## 131 12.86 1.35 2.32 18.0 122
## 132 12.88 2.99 2.40 20.0 104
## 133 12.81 2.31 2.40 24.0 98
## 134 12.70 3.55 2.36 21.5 106
## 135 12.51 1.24 2.25 17.5 85
## 136 12.60 2.46 2.20 18.5 94
## 137 12.25 4.72 2.54 21.0 89
## 138 12.53 5.51 2.64 25.0 96
## 139 13.49 3.59 2.19 19.5 88
## 140 12.84 2.96 2.61 24.0 101
## 141 12.93 2.81 2.70 21.0 96
## 142 13.36 2.56 2.35 20.0 89
## 143 13.52 3.17 2.72 23.5 97
## 144 13.62 4.95 2.35 20.0 92
## 145 12.25 3.88 2.20 18.5 112
## 146 13.16 3.57 2.15 21.0 102
## 147 13.88 5.04 2.23 20.0 80
## 148 12.87 4.61 2.48 21.5 86
## 149 13.32 3.24 2.38 21.5 92
## 150 13.08 3.90 2.36 21.5 113
## 151 13.50 3.12 2.62 24.0 123
## 152 12.79 2.67 2.48 22.0 112
## 153 13.11 1.90 2.75 25.5 116
## 154 13.23 3.30 2.28 18.5 98
## 155 12.58 1.29 2.10 20.0 103
## 156 13.17 5.19 2.32 22.0 93
## 157 13.84 4.12 2.38 19.5 89
## 158 12.45 3.03 2.64 27.0 97
## 159 14.34 1.68 2.70 25.0 98
## 160 13.48 1.67 2.64 22.5 89
## 161 12.36 3.83 2.38 21.0 88
## 162 13.69 3.26 2.54 20.0 107
## 163 12.85 3.27 2.58 22.0 106
## 164 12.96 3.45 2.35 18.5 106
## 165 13.78 2.76 2.30 22.0 90
## 166 13.73 4.36 2.26 22.5 88
## 167 13.45 3.70 2.60 23.0 111
## 168 12.82 3.37 2.30 19.5 88
## 169 13.58 2.58 2.69 24.5 105
## 170 13.40 4.60 2.86 25.0 112
## 171 12.20 3.03 2.32 19.0 96
## 172 12.77 2.39 2.28 19.5 86
## 173 14.16 2.51 2.48 20.0 91
## 174 13.71 5.65 2.45 20.5 95
## 175 13.40 3.91 2.48 23.0 102
## 176 13.27 4.28 2.26 20.0 120
## 177 13.17 2.59 2.37 20.0 120
## 178 14.13 4.10 2.74 24.5 96
In this matrix scatterplot, the diagonal cells show histograms of each of the variables, in this case the concentrations of the first five chemicals (variables V2, V3, V4, V5, V6).
Each of the off-diagonal cells is a scatterplot of two of the five chemicals, for example, the second cell in the first row is a scatterplot of V2 (y-axis) against V3 (x-axis).
If you see an interesting scatterplot for two variables in the matrix scatterplot, you may want to plot that scatterplot in more detail, with the data points labelled by their group (their cultivar in this case).
For example, in the matrix scatterplot above, the cell in the third column of the fourth row down is a scatterplot of V5 (x-axis) against V4 (y-axis). If you look at this scatterplot, it appears that there may be a positive relationship between V5 and V4.
If we want to label the data points by their group (the cultivar of wine here), we can use the “text” function in R to plot some text beside every data point. In this case, the cultivar of wine is stored in the column V1 of the variable “wine”, so we type:
If you look at the help page for the “text” function, you will see that “pos=4” will plot the text just to the right of the symbol for a data point. The “cex=0.5” option will plot the text at half the default size, and the “col=red” option will plot the text in red. This gives us the following plot:
## [1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
## [38] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
## [75] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
## [112] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
## [149] 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
It is often of interest to investigate whether any of the variables in a multivariate data set are significantly correlated.
To calculate the linear (Pearson) correlation coefficient for a pair of variables, you can use the “cor.test()” function in R. For example, to calculate the correlation coefficient for the first two chemicals’ concentrations, V2 and V3, we type:
## [1] 0.09439694
##
## Pearson's product-moment correlation
##
## data: wine$V2 and wine$V3
## t = 1.2579, df = 176, p-value = 0.2101
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
## -0.05342959 0.23817474
## sample estimates:
## cor
## 0.09439694
This tells us that the correlation coefficient is about 0.094, which is a very weak correlation. Furthermore, the P-value for the statistical test of whether the correlation coefficient is significantly different from zero is 0.21. This is much greater than 0.05 (which we can use here as a cutoff for statistical significance), so there is very weak evidence that that the correlation is non-zero.
If you have a lot of variables, you can use “cor.test()” to calculate the correlation coefficient for each pair of variables, but you might be just interested in finding out what are the most highly correlated pairs of variables. For this you can use the function “mosthighlycorrelated()” below.
mosthighlycorrelated <- function(mydataframe,numtoreport){
# find the correlations
cormatrix <- cor(mydataframe)
# set the correlations on the diagonal or lower triangle to zero,
# so they will not be reported as the highest ones:
diag(cormatrix) <- 0
cormatrix[lower.tri(cormatrix)] <- 0
# flatten the matrix into a dataframe for easy sorting
fm <- as.data.frame(as.table(cormatrix))
# assign human-friendly names
names(fm) <- c("First.Variable", "Second.Variable","Correlation")
# sort and print the top n correlations
head(fm[order(abs(fm$Correlation),decreasing=T),],n=numtoreport)}
mosthighlycorrelated(wine[2:14], 10)## First.Variable Second.Variable Correlation
## 84 V7 V8 0.8645635
## 150 V8 V13 0.7871939
## 149 V7 V13 0.6999494
## 111 V8 V10 0.6526918
## 157 V2 V14 0.6437200
## 110 V7 V10 0.6124131
## 154 V12 V13 0.5654683
## 132 V3 V12 -0.5612957
## 118 V2 V11 0.5463642
## 137 V8 V12 0.5434786
What if we want a way to look at all our variables at once?
And we have more than a handful of variables?
Dimension reduction: A transformation from a high-dimensional space into a low-dimensional space so that the low-dimensional space retains some meaningful properties of the original data.
The purpose of principal component analysis is to find the best low-dimensional representation of the variation in a multivariate data set.
For example, in the case of the wine data set, we have 13 chemical concentrations describing wine samples from three different cultivars.
We can carry out a principal component analysis to investigate whether we can capture most of the variation between samples using a smaller number of new variables (principal components), where each of these new variables is a linear combination of all or some of the 13 chemical concentrations.
To carry out a principal component analysis (PCA) on a multivariate data set, the first step is often to standardise the variables under study using the “scale()” function.
This is necessary if the input variables have very different variances, which is true in this case as the concentrations of the 13 chemicals have very different variances.
Once you have standardised your variables, you can carry out a principal component analysis using the “prcomp()” function in R.
## V2 V3 V4 V5
## Min. :11.03 Min. :0.740 Min. :1.360 Min. :10.60
## 1st Qu.:12.36 1st Qu.:1.603 1st Qu.:2.210 1st Qu.:17.20
## Median :13.05 Median :1.865 Median :2.360 Median :19.50
## Mean :13.00 Mean :2.336 Mean :2.367 Mean :19.49
## 3rd Qu.:13.68 3rd Qu.:3.083 3rd Qu.:2.558 3rd Qu.:21.50
## Max. :14.83 Max. :5.800 Max. :3.230 Max. :30.00
## V6
## Min. : 70.00
## 1st Qu.: 88.00
## Median : 98.00
## Mean : 99.74
## 3rd Qu.:107.00
## Max. :162.00
## standardise the variables
standardisedconcentrations <- as.data.frame(scale(wine[2:14]))
# do a PCA
wine.pca <- prcomp(standardisedconcentrations)
wine.pca## Standard deviations (1, .., p=13):
## [1] 2.1692972 1.5801816 1.2025273 0.9586313 0.9237035 0.8010350 0.7423128
## [8] 0.5903367 0.5374755 0.5009017 0.4751722 0.4108165 0.3215244
##
## Rotation (n x k) = (13 x 13):
## PC1 PC2 PC3 PC4 PC5 PC6
## V2 -0.144329395 0.483651548 -0.20738262 0.01785630 -0.26566365 0.21353865
## V3 0.245187580 0.224930935 0.08901289 -0.53689028 0.03521363 0.53681385
## V4 0.002051061 0.316068814 0.62622390 0.21417556 -0.14302547 0.15447466
## V5 0.239320405 -0.010590502 0.61208035 -0.06085941 0.06610294 -0.10082451
## V6 -0.141992042 0.299634003 0.13075693 0.35179658 0.72704851 0.03814394
## V7 -0.394660845 0.065039512 0.14617896 -0.19806835 -0.14931841 -0.08412230
## V8 -0.422934297 -0.003359812 0.15068190 -0.15229479 -0.10902584 -0.01892002
## V9 0.298533103 0.028779488 0.17036816 0.20330102 -0.50070298 -0.25859401
## V10 -0.313429488 0.039301722 0.14945431 -0.39905653 0.13685982 -0.53379539
## V11 0.088616705 0.529995672 -0.13730621 -0.06592568 -0.07643678 -0.41864414
## V12 -0.296714564 -0.279235148 0.08522192 0.42777141 -0.17361452 0.10598274
## V13 -0.376167411 -0.164496193 0.16600459 -0.18412074 -0.10116099 0.26585107
## V14 -0.286752227 0.364902832 -0.12674592 0.23207086 -0.15786880 0.11972557
## PC7 PC8 PC9 PC10 PC11 PC12
## V2 -0.05639636 0.39613926 -0.50861912 0.21160473 0.22591696 -0.26628645
## V3 0.42052391 0.06582674 0.07528304 -0.30907994 -0.07648554 0.12169604
## V4 -0.14917061 -0.17026002 0.30769445 -0.02712539 0.49869142 -0.04962237
## V5 -0.28696914 0.42797018 -0.20044931 0.05279942 -0.47931378 -0.05574287
## V6 0.32288330 -0.15636143 -0.27140257 0.06787022 -0.07128891 0.06222011
## V7 -0.02792498 -0.40593409 -0.28603452 -0.32013135 -0.30434119 -0.30388245
## V8 -0.06068521 -0.18724536 -0.04957849 -0.16315051 0.02569409 -0.04289883
## V9 0.59544729 -0.23328465 -0.19550132 0.21553507 -0.11689586 0.04235219
## V10 0.37213935 0.36822675 0.20914487 0.13418390 0.23736257 -0.09555303
## V11 -0.22771214 -0.03379692 -0.05621752 -0.29077518 -0.03183880 0.60422163
## V12 0.23207564 0.43662362 -0.08582839 -0.52239889 0.04821201 0.25921400
## V13 -0.04476370 -0.07810789 -0.13722690 0.52370587 -0.04642330 0.60095872
## V14 0.07680450 0.12002267 0.57578611 0.16211600 -0.53926983 -0.07940162
## PC13
## V2 0.01496997
## V3 0.02596375
## V4 -0.14121803
## V5 0.09168285
## V6 0.05677422
## V7 -0.46390791
## V8 0.83225706
## V9 0.11403985
## V10 -0.11691707
## V11 -0.01199280
## V12 -0.08988884
## V13 -0.15671813
## V14 0.01444734
## Importance of components:
## PC1 PC2 PC3 PC4 PC5 PC6 PC7
## Standard deviation 2.169 1.5802 1.2025 0.95863 0.92370 0.80103 0.74231
## Proportion of Variance 0.362 0.1921 0.1112 0.07069 0.06563 0.04936 0.04239
## Cumulative Proportion 0.362 0.5541 0.6653 0.73599 0.80162 0.85098 0.89337
## PC8 PC9 PC10 PC11 PC12 PC13
## Standard deviation 0.59034 0.53748 0.5009 0.47517 0.41082 0.32152
## Proportion of Variance 0.02681 0.02222 0.0193 0.01737 0.01298 0.00795
## Cumulative Proportion 0.92018 0.94240 0.9617 0.97907 0.99205 1.00000
## [1] 2.1692972 1.5801816 1.2025273 0.9586313 0.9237035 0.8010350 0.7423128
## [8] 0.5903367 0.5374755 0.5009017 0.4751722 0.4108165 0.3215244
## [1] 13
In this case, we see that the total variance is 13, which is equal to the number of standardized variables (13 variables).
This is because for standardized data, the variance of each standardized variable is 1.
The total variance is equal to the sum of the variances of the individual variables, and since the variance of each standardized variable is 1, the total variance should be equal to the number of variables (13 here).
The most obvious change in slope in the scree plot occurs at component 4, which is the “elbow” of the scree plot.
Therefore, it cound be argued based on the basis of the scree plot that the first three components should be retained.
## [1] 4.7058503 2.4969737 1.4460720 0.9189739 0.8532282 0.6416570 0.5510283
## [8] 0.3484974 0.2888799 0.2509025 0.2257886 0.1687702 0.1033779
The loadings for the principal components are stored in a named element “rotation” of the variable returned by “prcomp()”. This contains a matrix with the loadings of each principal component, where the first column in the matrix contains the loadings for the first principal component, the second column contains the loadings for the second principal component, and so on.
Therefore, to obtain the loadings for the first principal component in our analysis of the 13 chemical concentrations in wine samples, we type:
## V2 V3 V4 V5 V6 V7
## -0.144329395 0.245187580 0.002051061 0.239320405 -0.141992042 -0.394660845
## V8 V9 V10 V11 V12 V13
## -0.422934297 0.298533103 -0.313429488 0.088616705 -0.296714564 -0.376167411
## V14
## -0.286752227
In this example, let’s use the iris dataset in R.
Recall, PCA is done on only the independent variables in the dataset.
## [1] "Sepal.Length" "Sepal.Width" "Petal.Length" "Petal.Width" "Species"
## Sepal.Length Sepal.Width Petal.Length Petal.Width
## Min. :4.300 Min. :2.000 Min. :1.000 Min. :0.100
## 1st Qu.:5.100 1st Qu.:2.800 1st Qu.:1.600 1st Qu.:0.300
## Median :5.800 Median :3.000 Median :4.350 Median :1.300
## Mean :5.843 Mean :3.057 Mean :3.758 Mean :1.199
## 3rd Qu.:6.400 3rd Qu.:3.300 3rd Qu.:5.100 3rd Qu.:1.800
## Max. :7.900 Max. :4.400 Max. :6.900 Max. :2.500
## Species
## setosa :50
## versicolor:50
## virginica :50
##
##
##
## Sepal.Length Sepal.Width Petal.Length Petal.Width
## Sepal.Length 1.0000000 -0.1175698 0.8717538 0.8179411
## Sepal.Width -0.1175698 1.0000000 -0.4284401 -0.3661259
## Petal.Length 0.8717538 -0.4284401 1.0000000 0.9628654
## Petal.Width 0.8179411 -0.3661259 0.9628654 1.0000000
## standardize the variables
standardisediris <- as.data.frame(scale(iris[1:4]))
# do a PCA
iris.pca <- prcomp(standardisediris)
summary(iris.pca)## Importance of components:
## PC1 PC2 PC3 PC4
## Standard deviation 1.7084 0.9560 0.38309 0.14393
## Proportion of Variance 0.7296 0.2285 0.03669 0.00518
## Cumulative Proportion 0.7296 0.9581 0.99482 1.00000
The first principal components explain the variability around 73% and its captures the majority of the variability.
The second principal components explain the variability around 22% and its captures the majority of the variability.
Stick with PC1 and PC2.
## Standard deviations (1, .., p=4):
## [1] 1.7083611 0.9560494 0.3830886 0.1439265
##
## Rotation (n x k) = (4 x 4):
## PC1 PC2 PC3 PC4
## Sepal.Length 0.5210659 -0.37741762 0.7195664 0.2612863
## Sepal.Width -0.2693474 -0.92329566 -0.2443818 -0.1235096
## Petal.Length 0.5804131 -0.02449161 -0.1421264 -0.8014492
## Petal.Width 0.5648565 -0.06694199 -0.6342727 0.5235971
PCA lies within -1 and +1 (these are correlations).
For example, PC1 increases when Sepal.Length, Petal.Length, and Petal.Width are increased and it is positively correlated whereas PC1 increases Sepal.Width decreases because this value are negatively correlated.
## PC1 PC2 PC3 PC4
## PC1 1.000000e+00 -9.760996e-17 -1.685228e-15 1.996979e-15
## PC2 -9.760996e-17 1.000000e+00 2.381560e-16 2.409194e-15
## PC3 -1.685228e-15 2.381560e-16 1.000000e+00 -6.157152e-16
## PC4 1.996979e-15 2.409194e-15 -6.157152e-16 1.000000e+00
## [1] "mpg" "cyl" "disp" "hp" "drat" "wt" "qsec" "vs" "am" "gear"
## [11] "carb"
## mpg disp drat wt qsec
## Mazda RX4 21.0 160 3.90 2.620 16.46
## Mazda RX4 Wag 21.0 160 3.90 2.875 17.02
## Datsun 710 22.8 108 3.85 2.320 18.61
## Hornet 4 Drive 21.4 258 3.08 3.215 19.44
## Hornet Sportabout 18.7 360 3.15 3.440 17.02
## Valiant 18.1 225 2.76 3.460 20.22
## standardize the variables
standardisedconcentrations <- as.data.frame(scale(cars_pca_df))
# do a PCA
cars.pca <- prcomp(standardisedconcentrations)
summary(cars.pca)## Importance of components:
## PC1 PC2 PC3 PC4 PC5
## Standard deviation 1.8685 0.9880 0.5714 0.39202 0.22852
## Proportion of Variance 0.6983 0.1952 0.0653 0.03074 0.01044
## Cumulative Proportion 0.6983 0.8935 0.9588 0.98956 1.00000
The first principal components explain the variability around 70% and its captures the majority of the variability.
In this case, the first two components capture the majority of the variability.
## [1] 3.49139561 0.97621539 0.32648787 0.15367899 0.05222214
We see that the variance is above 1 for principal components 1.
Therefore, using Kaiser’s criterion, we would retain the only first principal component.
## Standard deviations (1, .., p=5):
## [1] 1.8685277 0.9880361 0.5713912 0.3920191 0.2285216
##
## Rotation (n x k) = (5 x 5):
## PC1 PC2 PC3 PC4 PC5
## mpg -0.5032625 0.05743315 0.2415625 -0.75397195 0.3414705
## disp 0.5110687 -0.05613132 -0.1353800 -0.64001016 -0.5547227
## drat -0.4336644 -0.35497557 -0.8197828 -0.04879738 -0.1072501
## wt 0.4971610 0.21503428 -0.4398510 -0.13804916 0.7028974
## qsec -0.2242805 0.90625786 -0.2404311 0.02178386 -0.2647891
## mpg cyl disp hp drat wt qsec vs am gear carb
## Mazda RX4 21.0 6 160 110 3.90 2.620 16.46 0 1 4 4
## Mazda RX4 Wag 21.0 6 160 110 3.90 2.875 17.02 0 1 4 4
## Datsun 710 22.8 4 108 93 3.85 2.320 18.61 1 1 4 1
## Hornet 4 Drive 21.4 6 258 110 3.08 3.215 19.44 1 0 3 1
## Hornet Sportabout 18.7 8 360 175 3.15 3.440 17.02 0 0 3 2
## Valiant 18.1 6 225 105 2.76 3.460 20.22 1 0 3 1
##
## Call:
## lm(formula = hp ~ mpg + disp + drat + wt + qsec, data = mtcars)
##
## Residuals:
## Min 1Q Median 3Q Max
## -42.959 -18.390 -4.042 14.879 102.802
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 392.8117 119.1055 3.298 0.00282 **
## mpg -3.1703 2.3523 -1.348 0.18936
## disp 0.1864 0.1356 1.375 0.18099
## drat 12.9668 16.7779 0.773 0.44658
## wt 7.9787 18.6915 0.427 0.67299
## qsec -16.6811 4.9192 -3.391 0.00223 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 31.74 on 26 degrees of freedom
## Multiple R-squared: 0.8203, Adjusted R-squared: 0.7858
## F-statistic: 23.74 on 5 and 26 DF, p-value: 6.262e-09
## mpg cyl disp hp drat wt qsec vs am gear carb PC1
## Mazda RX4 21.0 6 160 110 3.90 2.620 16.46 0 1 4 4 -0.7428349
## Mazda RX4 Wag 21.0 6 160 110 3.90 2.875 17.02 0 1 4 4 -0.6835537
## Datsun 710 22.8 4 108 93 3.85 2.320 18.61 1 1 4 1 -1.4892902
## Hornet 4 Drive 21.4 6 258 110 3.08 3.215 19.44 1 0 3 1 0.2212561
## Hornet Sportabout 18.7 8 360 175 3.15 3.440 17.02 0 0 3 2 1.2285996
## Valiant 18.1 6 225 105 2.76 3.460 20.22 1 0 3 1 0.6468671
## PC2 PC3 PC4 PC5
## Mazda RX4 -0.9963269 0.10380021 0.2910819 0.08392926
## Mazda RX4 Wag -0.6562789 -0.08617859 0.2619310 0.18413360
## Datsun 710 0.1020252 0.15498713 0.4075253 -0.10541625
## Hornet 4 Drive 1.1495880 0.60160117 -0.2378055 -0.18169682
## Hornet Sportabout -0.1466781 0.49910499 -0.4943902 -0.28501318
## Valiant 1.7949605 0.78092999 0.3495812 -0.09635587
##
## Call:
## lm(formula = hp ~ PC1, data = combine)
##
## Residuals:
## Min 1Q Median 3Q Max
## -60.435 -17.249 -7.495 4.353 149.253
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 146.688 7.557 19.411 < 2e-16 ***
## PC1 28.981 4.109 7.053 7.68e-08 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 42.75 on 30 degrees of freedom
## Multiple R-squared: 0.6238, Adjusted R-squared: 0.6113
## F-statistic: 49.75 on 1 and 30 DF, p-value: 7.68e-08
##
## Call:
## lm(formula = hp ~ PC1 + PC2, data = combine)
##
## Residuals:
## Min 1Q Median 3Q Max
## -45.239 -15.952 -4.245 13.318 100.560
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 146.688 5.418 27.076 < 2e-16 ***
## PC1 28.981 2.946 9.838 9.54e-11 ***
## PC2 -30.190 5.571 -5.419 7.92e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 30.65 on 29 degrees of freedom
## Multiple R-squared: 0.8131, Adjusted R-squared: 0.8002
## F-statistic: 63.08 on 2 and 29 DF, p-value: 2.745e-11