Fitting Models to Data

Linear Models

Terminology/Definitions

Pearson’s Correlation Coefficient

Pearson’s Correlation Coefficient

Correlation and Simple Linear Regression (SLR)

Initial Assumptions

Residual Assumptions

Example 1 - Correlation and SLR

# Entering Raw Data into R 

x <- c(10, 18, 25, 40, 50, 63, 42, 30, 5, 55) # Independent variable (ingredients)
y <- c(25, 55, 50, 75, 110, 138, 90, 60, 10, 100) # Dependent variable (sales) 

Scatterplot

plot(x,y, xlab="Ingredients", ylab="Sales", main="Scatterplot of Sales vs Ingredients")  

Correlation

cor(x,y)
## [1] 0.9762662

Correlation with Testing

cor.test(x, y) 
## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = 12.75, df = 8, p-value = 1.349e-06
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.8996224 0.9945560
## sample estimates:
##       cor 
## 0.9762662

Correlation with Testing

cor.test(x, y, method="pearson", conf.level = 0.90) 
## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = 12.75, df = 8, p-value = 1.349e-06
## alternative hypothesis: true correlation is not equal to 0
## 90 percent confidence interval:
##  0.9200475 0.9930968
## sample estimates:
##       cor 
## 0.9762662

Coefficient of Determination

rho <- cor(x,y)
rho^2
## [1] 0.9530957

Check for Normality (Dependent Variable)

# p-value > 0.05, the data is normal 

shapiro.test(y)   
## 
##  Shapiro-Wilk normality test
## 
## data:  y
## W = 0.98633, p-value = 0.99

Simple Linear Regression

#  Linear Model 

model <- lm(y~x) 
summary(model) 
## 
## Call:
## lm(formula = y ~ x)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -13.074  -4.403  -1.607   5.719  14.834 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)   4.6979     5.9520   0.789    0.453    
## x             1.9705     0.1545  12.750 1.35e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 9.022 on 8 degrees of freedom
## Multiple R-squared:  0.9531, Adjusted R-squared:  0.9472 
## F-statistic: 162.6 on 1 and 8 DF,  p-value: 1.349e-06

Regression Coefficients

coefficients(model) 
## (Intercept)           x 
##    4.697852    1.970478

Confidence Intervals

confint(model, level = 0.95)
##                 2.5 %    97.5 %
## (Intercept) -9.027533 18.423236
## x            1.614088  2.326867

Anova Table

anova(model) 
## Analysis of Variance Table
## 
## Response: y
##           Df  Sum Sq Mean Sq F value    Pr(>F)    
## x          1 13231.0 13231.0  162.56 1.349e-06 ***
## Residuals  8   651.1    81.4                      
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Prediction - main goal of Regression

#Predict y when x=15
newdata = data.frame(x=15) 

predict(model, newdata)
##        1 
## 34.25502
#Predict with CI
predict(model, newdata, interval="conf")
##        fit      lwr      upr
## 1 34.25502 24.86499 43.64505

Model Diagnostics (Examining your Residuals)

Normality using Q-Q (Quantile Quantile) Plot

pas<-summary(model)
qqnorm(pas$residuals) 
qqline(pas$residuals,  lwd = 2)

Normality using Boxplot

boxplot(pas$residuals, main = "Boxplot of residuals")   

Example 2 - Extensions of SLR

Scatterplot

plot(wn$ScoreSub1, wn$FamilyMonthlyInc, main="Scatterplot of Income vs Score")

Pearson Correlation

cor.test(wn$ScoreSub1, wn$FamilyMonthlyInc, method="pearson")
## 
##  Pearson's product-moment correlation
## 
## data:  wn$ScoreSub1 and wn$FamilyMonthlyInc
## t = 2.2703, df = 448, p-value = 0.02366
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.01435481 0.19714661
## sample estimates:
##       cor 
## 0.1066518

Correlation Matrix

cor(wn[,c(2,6,7, 8, 9)])
##                          Age FamilyMonthlyInc   ScoreSub1   ScoreSub2
## Age               1.00000000     -0.043594688  0.02500791 -0.03567968
## FamilyMonthlyInc -0.04359469      1.000000000  0.10665176 -0.02403096
## ScoreSub1         0.02500791      0.106651759  1.00000000 -0.04958413
## ScoreSub2        -0.03567968     -0.024030962 -0.04958413  1.00000000
## ScoreSub3         0.05129655     -0.003516893  0.01919580  0.05149042
##                     ScoreSub3
## Age               0.051296546
## FamilyMonthlyInc -0.003516893
## ScoreSub1         0.019195800
## ScoreSub2         0.051490416
## ScoreSub3         1.000000000

Q-Q to test for Normality

qqnorm(wn$FamilyMonthlyInc) 
qqline(wn$FamilyMonthlyInc, lwd = 2) 

Testing for Normality

shapiro.test(wn$ScoreSub1)  
## 
##  Shapiro-Wilk normality test
## 
## data:  wn$ScoreSub1
## W = 0.99544, p-value = 0.213

Simple Linear Regression

model2<- lm(wn$FamilyMonthlyInc ~ wn$ScoreSub1) 
summary(model2) 
## 
## Call:
## lm(formula = wn$FamilyMonthlyInc ~ wn$ScoreSub1)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4024.0 -1830.8  -109.9  1549.4  6533.9 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)   
## (Intercept)   3058.07     942.49   3.245  0.00126 **
## wn$ScoreSub1    25.39      11.18   2.270  0.02366 * 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 2227 on 448 degrees of freedom
## Multiple R-squared:  0.01137,    Adjusted R-squared:  0.009168 
## F-statistic: 5.154 on 1 and 448 DF,  p-value: 0.02366

Regression Coefficients

# Gives you intercept and slope
coefficients(model2) 
##  (Intercept) wn$ScoreSub1 
##   3058.06803     25.38689
# 95% confidence intervals for intercept and slope
confint(model2, level = 0.95) 
##                    2.5 %     97.5 %
## (Intercept)  1205.813118 4910.32295
## wn$ScoreSub1    3.411261   47.36252

Residual Analysis

Check for Normality

resid <- residuals(model2)
boxplot(resid)

Check for Normality (another way)

resid <- residuals(model2) 
qqnorm(resid)
qqline(resid,  lwd = 2)  

Check all Regression Residual Assumptions

Residual Assumptions (Recall)

All Assumptions

par(mfrow = c(2, 2))
plot(model2)

Checking Assumptions using Statistical Tests

Checking for Linearity

raintest(model2)
## 
##  Rainbow test
## 
## data:  model2
## Rain = 1.0124, df1 = 225, df2 = 223, p-value = 0.4635

Checking for Normality

resid <- residuals(model2)
shapiro.test(resid)
## 
##  Shapiro-Wilk normality test
## 
## data:  resid
## W = 0.9716, p-value = 1.153e-07

Checking for Homoscedasticity

ols_test_breusch_pagan(model2)
## 
##  Breusch Pagan Test for Heteroskedasticity
##  -----------------------------------------
##  Ho: the variance is constant            
##  Ha: the variance is not constant        
## 
##                      Data                       
##  -----------------------------------------------
##  Response : wn$FamilyMonthlyInc 
##  Variables: fitted values of wn$FamilyMonthlyInc 
## 
##         Test Summary          
##  -----------------------------
##  DF            =    1 
##  Chi2          =    2.865727 
##  Prob > Chi2   =    0.09048481

Checking for Outliers

ols_plot_resid_stud(model2) 

How to delete or remove a row in excel

# Actual sample size is 450
length(wn$FamilyMonthlyInc)    
## [1] 450
# how to delete row 57 from excel
wn57 <- wn[-c(57),]  
length(wn57$FamilyMonthlyInc)   # you see sample size is 449 
## [1] 449
# deleting three rows at a time
wn5723 <- wn[-c(57, 200, 300),]  
length(wn5723$FamilyMonthlyInc)   # you see sample size is 447  
## [1] 447

Some take aways:

Example 3 - Review this example with interpretations

Multiple Linear Regression

Multiple Linear Regression (MLR)

MLR Assumptions

Data Transformations

More on Assumptions

Model Comparisons

Example 1 - MLR

Normality of outcome variable (GPA)

qqnorm(Focus$GPA) 
qqline(Focus$GPA,  lwd = 2)  

Normality

shapiro.test(Focus$GPA) 
## 
##  Shapiro-Wilk normality test
## 
## data:  Focus$GPA
## W = 0.98968, p-value = 0.1601

Regression Equation for Model 1

mlr1<- lm(Focus$GPA ~  Focus$AGE + Focus$CREDITS)
summary(mlr1)
## 
## Call:
## lm(formula = Focus$GPA ~ Focus$AGE + Focus$CREDITS)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.29806 -0.39124  0.01604  0.33502  1.09542 
## 
## Coefficients:
##                Estimate Std. Error t value Pr(>|t|)    
## (Intercept)    4.317654   0.387577  11.140  < 2e-16 ***
## Focus$AGE     -0.084106   0.020922  -4.020 8.28e-05 ***
## Focus$CREDITS  0.005602   0.001262   4.438 1.51e-05 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4917 on 197 degrees of freedom
## Multiple R-squared:  0.1008, Adjusted R-squared:  0.09165 
## F-statistic: 11.04 on 2 and 197 DF,  p-value: 2.858e-05

Regression Equation for Model 2

mlr2<- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS) 
summary(mlr2)
## 
## Call:
## lm(formula = Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.21681 -0.37086  0.00697  0.32877  1.17585 
## 
## Coefficients:
##                     Estimate Std. Error t value Pr(>|t|)    
## (Intercept)         4.356240   0.386381  11.274  < 2e-16 ***
## factor(Focus$SEX)M -0.120611   0.070774  -1.704   0.0899 .  
## Focus$AGE          -0.084202   0.020821  -4.044 7.55e-05 ***
## Focus$CREDITS       0.005786   0.001261   4.589 7.95e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4893 on 196 degrees of freedom
## Multiple R-squared:  0.1139, Adjusted R-squared:  0.1003 
## F-statistic: 8.398 on 3 and 196 DF,  p-value: 2.801e-05

Comments from Model 2

Regression Equation for Model 2

# Changing reference category (from Females to Males) 
Focus$SEX <- relevel(factor(Focus$SEX), ref = "M") 
mlr1<- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS) 
summary(mlr1)
## 
## Call:
## lm(formula = Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.21681 -0.37086  0.00697  0.32877  1.17585 
## 
## Coefficients:
##                     Estimate Std. Error t value Pr(>|t|)    
## (Intercept)         4.235630   0.388708  10.897  < 2e-16 ***
## factor(Focus$SEX)F  0.120611   0.070774   1.704   0.0899 .  
## Focus$AGE          -0.084202   0.020821  -4.044 7.55e-05 ***
## Focus$CREDITS       0.005786   0.001261   4.589 7.95e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4893 on 196 degrees of freedom
## Multiple R-squared:  0.1139, Adjusted R-squared:  0.1003 
## F-statistic: 8.398 on 3 and 196 DF,  p-value: 2.801e-05

Comments from Model 2 by changing the reference category

Regression Equation to use going forward

mlr<- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS) 
summary(mlr) 
## 
## Call:
## lm(formula = Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -1.21681 -0.37086  0.00697  0.32877  1.17585 
## 
## Coefficients:
##                     Estimate Std. Error t value Pr(>|t|)    
## (Intercept)         4.235630   0.388708  10.897  < 2e-16 ***
## factor(Focus$SEX)F  0.120611   0.070774   1.704   0.0899 .  
## Focus$AGE          -0.084202   0.020821  -4.044 7.55e-05 ***
## Focus$CREDITS       0.005786   0.001261   4.589 7.95e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.4893 on 196 degrees of freedom
## Multiple R-squared:  0.1139, Adjusted R-squared:  0.1003 
## F-statistic: 8.398 on 3 and 196 DF,  p-value: 2.801e-05

Confidence Intervals for Regression Coefficients

confint(mlr) 
##                           2.5 %       97.5 %
## (Intercept)         3.469041839  5.002217423
## factor(Focus$SEX)F -0.018965819  0.260186897
## Focus$AGE          -0.125264445 -0.043139145
## Focus$CREDITS       0.003299627  0.008273339
# by default is 95% 
confint(mlr, level = 0.95)
##                           2.5 %       97.5 %
## (Intercept)         3.469041839  5.002217423
## factor(Focus$SEX)F -0.018965819  0.260186897
## Focus$AGE          -0.125264445 -0.043139145
## Focus$CREDITS       0.003299627  0.008273339

Confidence Intervals for Regression Coefficients

confint(mlr) 
##                           2.5 %       97.5 %
## (Intercept)         3.469041839  5.002217423
## factor(Focus$SEX)F -0.018965819  0.260186897
## Focus$AGE          -0.125264445 -0.043139145
## Focus$CREDITS       0.003299627  0.008273339
# by default is 90% 
confint(mlr, level = 0.90)
##                             5 %         95 %
## (Intercept)         3.593224896  4.878034366
## factor(Focus$SEX)F  0.003644793  0.237576285
## Focus$AGE          -0.118612519 -0.049791071
## Focus$CREDITS       0.003702484  0.007870482

Residual Analysis

Normality

resid <- residuals(mlr) 
qqnorm(resid)
qqline(resid,  lwd = 2)  

Normality (another approach)

resid <- residuals(mlr) 
# good models should have no pattern (alternative)
plot(resid, pch = 16, col = "red", main="Scatterplot of Residuals") 

Normality (another approach)

resid <- residuals(mlr) 
shapiro.test(resid)
## 
##  Shapiro-Wilk normality test
## 
## data:  resid
## W = 0.99119, p-value = 0.2649

Checking for Multicollinearity

# library(car)
vif(mlr)  
## factor(Focus$SEX)         Focus$AGE     Focus$CREDITS 
##          1.011989          1.686409          1.698870

Checking for Autocorrelation

# library(lmtest)
dwtest(mlr) 
## 
##  Durbin-Watson test
## 
## data:  mlr
## DW = 2.0225, p-value = 0.5668
## alternative hypothesis: true autocorrelation is greater than 0

Checking for Linearity

raintest(mlr)
## 
##  Rainbow test
## 
## data:  mlr
## Rain = 1.0055, df1 = 100, df2 = 96, p-value = 0.4898

Checking for Homoscedasticity

ols_test_breusch_pagan(mlr)
## 
##  Breusch Pagan Test for Heteroskedasticity
##  -----------------------------------------
##  Ho: the variance is constant            
##  Ha: the variance is not constant        
## 
##                 Data                  
##  -------------------------------------
##  Response : Focus$GPA 
##  Variables: fitted values of Focus$GPA 
## 
##         Test Summary         
##  ----------------------------
##  DF            =    1 
##  Chi2          =    1.972315 
##  Prob > Chi2   =    0.1602023

Model Selection (select only one of these approaches to use)

# For Nested Models 
model1 <- lm(Focus$GPA ~ factor(Focus$SEX))
model2 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE)
model3 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)

# decide on best model using p-value obtained
anova(model1, model2) 
## Analysis of Variance Table
## 
## Model 1: Focus$GPA ~ factor(Focus$SEX)
## Model 2: Focus$GPA ~ factor(Focus$SEX) + Focus$AGE
##   Res.Df    RSS Df Sum of Sq      F Pr(>F)
## 1    198 52.487                           
## 2    197 51.977  1   0.50982 1.9323 0.1661

Model Selection (select only one of these approaches to use)

# For Nested Models 
model1 <- lm(Focus$GPA ~ factor(Focus$SEX))
model2 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE)
model3 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)

# decide on best model using p-value obtained
anova(model1, model3)
## Analysis of Variance Table
## 
## Model 1: Focus$GPA ~ factor(Focus$SEX)
## Model 2: Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS
##   Res.Df    RSS Df Sum of Sq      F    Pr(>F)    
## 1    198 52.487                                  
## 2    196 46.935  2    5.5523 11.593 1.743e-05 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Model Selection (select only one of these approaches to use)

# For Nested Models 
model1 <- lm(Focus$GPA ~ factor(Focus$SEX))
model2 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE)
model3 <- lm(Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS)

# decide on best model using p-value obtained
anova(model2, model3)
## Analysis of Variance Table
## 
## Model 1: Focus$GPA ~ factor(Focus$SEX) + Focus$AGE
## Model 2: Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS
##   Res.Df    RSS Df Sum of Sq      F    Pr(>F)    
## 1    197 51.977                                  
## 2    196 46.935  1    5.0425 21.057 7.947e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Using AIC and BIC (Other Models)

# library(rcompanion)

compareLM(model1, model2, model3)
## $Models
##   Formula                                                    
## 1 "Focus$GPA ~ factor(Focus$SEX)"                            
## 2 "Focus$GPA ~ factor(Focus$SEX) + Focus$AGE"                
## 3 "Focus$GPA ~ factor(Focus$SEX) + Focus$AGE + Focus$CREDITS"
## 
## $Fit.criteria
##   Rank Df.res   AIC  AICc   BIC R.squared Adj.R.sq   p.value Shapiro.W
## 1    2    198 306.0 306.1 315.9  0.009081 0.004077 1.795e-01    0.9913
## 2    3    197 306.1 306.3 319.3  0.018710 0.008744 1.557e-01    0.9912
## 3    4    196 287.7 288.0 304.2  0.113900 0.100300 2.801e-05    0.9912
##   Shapiro.p
## 1    0.2752
## 2    0.2672
## 3    0.2649

Example 2 - Focusing on Multicolinearity Problem in MLR

# Calling Data from a Package 
data("Boston", package = "MASS")
# library(olsrr)

#View variables in Boston Dataset
names(Boston)  
##  [1] "crim"    "zn"      "indus"   "chas"    "nox"     "rm"      "age"    
##  [8] "dis"     "rad"     "tax"     "ptratio" "black"   "lstat"   "medv"

MLR Regression Model

model <- lm(medv ~ ., data = Boston)
summary(model) 
## 
## Call:
## lm(formula = medv ~ ., data = Boston)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -15.595  -2.730  -0.518   1.777  26.199 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  3.646e+01  5.103e+00   7.144 3.28e-12 ***
## crim        -1.080e-01  3.286e-02  -3.287 0.001087 ** 
## zn           4.642e-02  1.373e-02   3.382 0.000778 ***
## indus        2.056e-02  6.150e-02   0.334 0.738288    
## chas         2.687e+00  8.616e-01   3.118 0.001925 ** 
## nox         -1.777e+01  3.820e+00  -4.651 4.25e-06 ***
## rm           3.810e+00  4.179e-01   9.116  < 2e-16 ***
## age          6.922e-04  1.321e-02   0.052 0.958229    
## dis         -1.476e+00  1.995e-01  -7.398 6.01e-13 ***
## rad          3.060e-01  6.635e-02   4.613 5.07e-06 ***
## tax         -1.233e-02  3.760e-03  -3.280 0.001112 ** 
## ptratio     -9.527e-01  1.308e-01  -7.283 1.31e-12 ***
## black        9.312e-03  2.686e-03   3.467 0.000573 ***
## lstat       -5.248e-01  5.072e-02 -10.347  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 4.745 on 492 degrees of freedom
## Multiple R-squared:  0.7406, Adjusted R-squared:  0.7338 
## F-statistic: 108.1 on 13 and 492 DF,  p-value: < 2.2e-16

Checking for Multicolinearity

car::vif(model) 
##     crim       zn    indus     chas      nox       rm      age      dis 
## 1.792192 2.298758 3.991596 1.073995 4.393720 1.933744 3.100826 3.955945 
##      rad      tax  ptratio    black    lstat 
## 7.484496 9.008554 1.799084 1.348521 2.941491

Refit a model excluding the tax variable to solve the issue

model2 <- lm(medv ~ . - tax, data = Boston)
car::vif(model2) 
##     crim       zn    indus     chas      nox       rm      age      dis 
## 1.791940 2.184240 3.226015 1.058220 4.369271 1.923075 3.098044 3.954446 
##      rad  ptratio    black    lstat 
## 2.837494 1.788839 1.347564 2.940800

Residual Analysis

Testing all assumptions

par(mfrow = c(2, 2))
plot(model2) 

Checking for Linearity

raintest(model2)
## 
##  Rainbow test
## 
## data:  model2
## Rain = 0.59738, df1 = 253, df2 = 240, p-value = 1

Normality

resid <- residuals(model2) 
shapiro.test(resid)
## 
##  Shapiro-Wilk normality test
## 
## data:  resid
## W = 0.9075, p-value < 2.2e-16

Checking for Homoscedasticity

ols_test_breusch_pagan(model2)
## 
##  Breusch Pagan Test for Heteroskedasticity
##  -----------------------------------------
##  Ho: the variance is constant            
##  Ha: the variance is not constant        
## 
##               Data               
##  --------------------------------
##  Response : medv 
##  Variables: fitted values of medv 
## 
##          Test Summary           
##  -------------------------------
##  DF            =    1 
##  Chi2          =    15.63522 
##  Prob > Chi2   =    7.681069e-05

Checking for Outliers

ols_plot_resid_stud(model2) 

Example 3 - Work through this example

Non-linear Models

Logistic Regression

Logistic Regression Assumptions

Example 1 - Binary Dataset

head(Binary)  
##   admit gre  gpa rank id
## 1     0 380 3.61    3  1
## 2     1 660 3.67    3  2
## 3     1 800 4.00    1  3
## 4     1 640 3.19    4  4
## 5     0 520 2.93    4  5
## 6     1 760 3.00    2  6

Frequency Tables

table(Binary$admit)
## 
##   0   1 
## 273 127
table(Binary$rank)
## 
##   1   2   3   4 
##  61 151 121  67

Contingency Tables

table(Binary$admit, Binary$rank)
##    
##      1  2  3  4
##   0 28 97 93 55
##   1 33 54 28 12
xtabs(~Binary$admit + Binary$rank) 
##             Binary$rank
## Binary$admit  1  2  3  4
##            0 28 97 93 55
##            1 33 54 28 12

Chi-square test

chisq.test(Binary$admit, Binary$rank)
## 
##  Pearson's Chi-squared test
## 
## data:  Binary$admit and Binary$rank
## X-squared = 25.242, df = 3, p-value = 1.374e-05

Using the Logit Model

model <- glm(Binary$admit ~ Binary$gre + Binary$gpa + factor(Binary$rank),  family = "binomial")

Fitting Model

model <- glm(Binary$admit ~ Binary$gre + Binary$gpa + factor(Binary$rank),  family = "binomial")
summary(model)
## 
## Call:
## glm(formula = Binary$admit ~ Binary$gre + Binary$gpa + factor(Binary$rank), 
##     family = "binomial")
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -1.6268  -0.8662  -0.6388   1.1490   2.0790  
## 
## Coefficients:
##                       Estimate Std. Error z value Pr(>|z|)    
## (Intercept)          -3.989979   1.139951  -3.500 0.000465 ***
## Binary$gre            0.002264   0.001094   2.070 0.038465 *  
## Binary$gpa            0.804038   0.331819   2.423 0.015388 *  
## factor(Binary$rank)2 -0.675443   0.316490  -2.134 0.032829 *  
## factor(Binary$rank)3 -1.340204   0.345306  -3.881 0.000104 ***
## factor(Binary$rank)4 -1.551464   0.417832  -3.713 0.000205 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 499.98  on 399  degrees of freedom
## Residual deviance: 458.52  on 394  degrees of freedom
## AIC: 470.52
## 
## Number of Fisher Scoring iterations: 4

Interpretations

Confidence Intervals (CI)

CIs using Profiled Log-likelihood

confint(model)
## Waiting for profiling to be done...
##                              2.5 %       97.5 %
## (Intercept)          -6.2716202334 -1.792547080
## Binary$gre            0.0001375921  0.004435874
## Binary$gpa            0.1602959439  1.464142727
## factor(Binary$rank)2 -1.3008888002 -0.056745722
## factor(Binary$rank)3 -2.0276713127 -0.670372346
## factor(Binary$rank)4 -2.4000265384 -0.753542605

CIs using Standard Errors

confint.default(model)
##                              2.5 %       97.5 %
## (Intercept)          -6.2242418514 -1.755716295
## Binary$gre            0.0001202298  0.004408622
## Binary$gpa            0.1536836760  1.454391423
## factor(Binary$rank)2 -1.2957512650 -0.055134591
## factor(Binary$rank)3 -2.0169920597 -0.663415773
## factor(Binary$rank)4 -2.3703986294 -0.732528724

Odds Ratio

Odds Ratio (OR) and Confidence Intervals (CI)

exp(cbind(OR = coef(model), confint(model)))
## Waiting for profiling to be done...
##                             OR       2.5 %    97.5 %
## (Intercept)          0.0185001 0.001889165 0.1665354
## Binary$gre           1.0022670 1.000137602 1.0044457
## Binary$gpa           2.2345448 1.173858216 4.3238349
## factor(Binary$rank)2 0.5089310 0.272289674 0.9448343
## factor(Binary$rank)3 0.2617923 0.131641717 0.5115181
## factor(Binary$rank)4 0.2119375 0.090715546 0.4706961

Example 2 - Logistic Regression using Consumption Dataset

# Reading data into R and make the response variable is categorical 
head(FC_Edited)  
##   Experience Income Location Switch Perception Days  IQ Stress College FC
## 1         16     no       no    yes  decreased    5  85     17      No No
## 2         13     no       no    yes  decreased   13  95     25      No No
## 3         40    yes       no     no  decreased    7 110     12      No No
## 4         31     no       no     no  decreased   20 120     18      No No
## 5         32     no      yes     no  decreased   11 105     13     Yes No
## 6         46     no       no     no  decreased   16 100     27      No No
##   Marital.Status  Sex Job.Status Obese Age   LogAge
## 1        Married Male       Full    No  27 1.431364
## 2        Married Male       Full    No  47 1.672098
## 3        Married Male       Full    No  48 1.681241
## 4        Married Male       Full    No  64 1.806180
## 5        Married Male       Part    No  48 1.681241
## 6        Married Male       Full    No  58 1.763428

Re-coding a categorical variable

# library(plyr)
FC_Edited$Obese_cat <- revalue(FC_Edited$Obese, c("No"="0", "Yes"="1"))
head(FC_Edited)
##   Experience Income Location Switch Perception Days  IQ Stress College FC
## 1         16     no       no    yes  decreased    5  85     17      No No
## 2         13     no       no    yes  decreased   13  95     25      No No
## 3         40    yes       no     no  decreased    7 110     12      No No
## 4         31     no       no     no  decreased   20 120     18      No No
## 5         32     no      yes     no  decreased   11 105     13     Yes No
## 6         46     no       no     no  decreased   16 100     27      No No
##   Marital.Status  Sex Job.Status Obese Age   LogAge Obese_cat
## 1        Married Male       Full    No  27 1.431364         0
## 2        Married Male       Full    No  47 1.672098         0
## 3        Married Male       Full    No  48 1.681241         0
## 4        Married Male       Full    No  64 1.806180         0
## 5        Married Male       Part    No  48 1.681241         0
## 6        Married Male       Full    No  58 1.763428         0

Contingency Tables

table(FC_Edited$Obese)
## 
##  No Yes 
## 131  68
table(FC_Edited$Obese_cat)
## 
##   0   1 
## 131  68

Chi-square Tests

chisq.test(table(FC_Edited$Obese_cat, FC_Edited$Sex))
## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  table(FC_Edited$Obese_cat, FC_Edited$Sex)
## X-squared = 67.772, df = 1, p-value < 2.2e-16
chisq.test(table(FC_Edited$Obese_cat, FC_Edited$Marital.Status))
## Warning in chisq.test(table(FC_Edited$Obese_cat, FC_Edited$Marital.Status)):
## Chi-squared approximation may be incorrect
## 
##  Pearson's Chi-squared test
## 
## data:  table(FC_Edited$Obese_cat, FC_Edited$Marital.Status)
## X-squared = 1.927, df = 2, p-value = 0.3816

Chi-square Tests with continuity correction

xtabs(~FC_Edited$Obese + FC_Edited$Sex)
##                FC_Edited$Sex
## FC_Edited$Obese Female Male
##             No      83   48
##             Yes      1   67
chisq.test(table(FC_Edited$Obese, FC_Edited$Sex), correct = T)
## 
##  Pearson's Chi-squared test with Yates' continuity correction
## 
## data:  table(FC_Edited$Obese, FC_Edited$Sex)
## X-squared = 67.772, df = 1, p-value < 2.2e-16
chisq.test(table(FC_Edited$Obese, FC_Edited$Sex), correct = F)
## 
##  Pearson's Chi-squared test
## 
## data:  table(FC_Edited$Obese, FC_Edited$Sex)
## X-squared = 70.286, df = 1, p-value < 2.2e-16

Fitting Logistic Regression Models

Model 1

model1 <- glm(factor(Obese_cat) ~ factor(Sex), data = FC_Edited, family = "binomial")
summary(model1)
## 
## Call:
## glm(formula = factor(Obese_cat) ~ factor(Sex), family = "binomial", 
##     data = FC_Edited)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -1.3219  -0.1548  -0.1548   1.0395   2.9769  
## 
## Coefficients:
##                 Estimate Std. Error z value Pr(>|z|)    
## (Intercept)       -4.419      1.006  -4.392 1.12e-05 ***
## factor(Sex)Male    4.752      1.024   4.643 3.44e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 255.58  on 198  degrees of freedom
## Residual deviance: 167.12  on 197  degrees of freedom
## AIC: 171.12
## 
## Number of Fisher Scoring iterations: 7

Model 2

model2 <- glm(factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status), 
               data = FC_Edited, family = "binomial")
summary(model2)
## 
## Call:
## glm(formula = factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status), 
##     family = "binomial", data = FC_Edited)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -1.7427  -0.2349  -0.1390   1.0885   3.0477  
## 
## Coefficients:
##                                Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                     -4.6346     1.0243  -4.524 6.06e-06 ***
## factor(Sex)Male                  4.8473     1.0332   4.691 2.71e-06 ***
## factor(Marital.Status)Separate   1.0443     1.0396   1.005    0.315    
## factor(Marital.Status)Single     1.0585     0.7563   1.399    0.162    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 255.58  on 198  degrees of freedom
## Residual deviance: 163.96  on 195  degrees of freedom
## AIC: 171.96
## 
## Number of Fisher Scoring iterations: 7

Model 3

model3 <- glm(factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status) + Stress, 
              data = FC_Edited, family = "binomial")
summary(model3)
## 
## Call:
## glm(formula = factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status) + 
##     Stress, family = "binomial", data = FC_Edited)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -2.0136  -0.2481  -0.1268   0.8401   2.8666  
## 
## Coefficients:
##                                Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                    -3.70064    1.09149  -3.390 0.000698 ***
## factor(Sex)Male                 4.72849    1.03543   4.567 4.95e-06 ***
## factor(Marital.Status)Separate  0.87648    1.05368   0.832 0.405503    
## factor(Marital.Status)Single    1.30580    0.78198   1.670 0.094945 .  
## Stress                         -0.05595    0.02517  -2.223 0.026219 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 255.58  on 198  degrees of freedom
## Residual deviance: 158.75  on 194  degrees of freedom
## AIC: 168.75
## 
## Number of Fisher Scoring iterations: 7

Using ANOVA to determine best competing model

anova(model1, model2, test="Chisq")
## Analysis of Deviance Table
## 
## Model 1: factor(Obese_cat) ~ factor(Sex)
## Model 2: factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status)
##   Resid. Df Resid. Dev Df Deviance Pr(>Chi)
## 1       197     167.12                     
## 2       195     163.96  2    3.157   0.2063

Using ANOVA to determine best competing model

anova(model1, model3, test="Chisq")
## Analysis of Deviance Table
## 
## Model 1: factor(Obese_cat) ~ factor(Sex)
## Model 2: factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status) + Stress
##   Resid. Df Resid. Dev Df Deviance Pr(>Chi)  
## 1       197     167.12                       
## 2       194     158.75  3   8.3719  0.03892 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Using ANOVA to determine best competing model

anova(model2, model3, test="Chisq")
## Analysis of Deviance Table
## 
## Model 1: factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status)
## Model 2: factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status) + Stress
##   Resid. Df Resid. Dev Df Deviance Pr(>Chi)  
## 1       195     163.96                       
## 2       194     158.75  1   5.2149  0.02239 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Look at AIC Values to determine best competing model

Best Model (Model 3)

summary(model3)
## 
## Call:
## glm(formula = factor(Obese_cat) ~ factor(Sex) + factor(Marital.Status) + 
##     Stress, family = "binomial", data = FC_Edited)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -2.0136  -0.2481  -0.1268   0.8401   2.8666  
## 
## Coefficients:
##                                Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                    -3.70064    1.09149  -3.390 0.000698 ***
## factor(Sex)Male                 4.72849    1.03543   4.567 4.95e-06 ***
## factor(Marital.Status)Separate  0.87648    1.05368   0.832 0.405503    
## factor(Marital.Status)Single    1.30580    0.78198   1.670 0.094945 .  
## Stress                         -0.05595    0.02517  -2.223 0.026219 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 255.58  on 198  degrees of freedom
## Residual deviance: 158.75  on 194  degrees of freedom
## AIC: 168.75
## 
## Number of Fisher Scoring iterations: 7

Interpretations from Previous Slide

Confidence Intervals for Model 3

confint.default(model3)
##                                     2.5 %       97.5 %
## (Intercept)                    -5.8399196 -1.561355809
## factor(Sex)Male                 2.6990929  6.757890337
## factor(Marital.Status)Separate -1.1886879  2.941657414
## factor(Marital.Status)Single   -0.2268448  2.838451758
## Stress                         -0.1052740 -0.006618739

Odds Ratio Results for Model 3

# Using Profile Log-Likelihood
exp(cbind(OR = coef(model3), confint(model3))) 
## Waiting for profiling to be done...
##                                          OR        2.5 %       97.5 %
## (Intercept)                      0.02470777  0.001283817    0.1412557
## factor(Sex)Male                113.12479826 23.103969985 2054.8394463
## factor(Marital.Status)Separate   2.40243969  0.359460256   26.5982718
## factor(Marital.Status)Single     3.69065325  0.885282035   20.5110786
## Stress                           0.94558983  0.898370076    0.9922267

Odds Ratio Results for Model 3

# Using Standard Errors
exp(cbind(OR = coef(model3), confint.default(model3))) 
##                                          OR        2.5 %      97.5 %
## (Intercept)                      0.02470777  0.002909077   0.2098514
## factor(Sex)Male                113.12479826 14.866240441 860.8242300
## factor(Marital.Status)Separate   2.40243969  0.304620694  18.9472237
## factor(Marital.Status)Single     3.69065325  0.797044469  17.0892867
## Stress                           0.94558983  0.900077832   0.9934031

Model Accuracy

# Checking Accuracy of the Model 

table(factor(FC_Edited$Obese), predict(model3)>0)
##      
##       FALSE TRUE
##   No     98   33
##   Yes    13   55

T-tests

Two-sample Independent

Assumptions

Example 1: Two-sample Independent t-tests

# Reading data into R 
fresh <- c(10.2,10.5,10.3,10.8,9.8,10.6,10.7,10.2,10.0,10.6)
stored <- c(9.8,9.6,10.1,10.2,10.1,9.7,9.5,9.6,9.8,9.9)


# Re-format Data  

Potency <- c(10.2,10.5,10.3,10.8,9.8,10.6,10.7,10.2,10.0,10.6,
             9.8,9.6,10.1,10.2,10.1,9.7,9.5,9.6,9.8,9.9)
Group <- c(1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2)

Is data is Normal?

shapiro.test(Potency)
## 
##  Shapiro-Wilk normality test
## 
## data:  Potency
## W = 0.95248, p-value = 0.4064

Is data is Normal?

boxplot(Potency ~ Group)  

Levene’s test (test for equal variance)

var.test(fresh, stored) 
## 
##  F test to compare two variances
## 
## data:  fresh and stored
## F = 1.8061, num df = 9, denom df = 9, p-value = 0.3917
## alternative hypothesis: true ratio of variances is not equal to 1
## 95 percent confidence interval:
##  0.4486201 7.2715173
## sample estimates:
## ratio of variances 
##           1.806142

Levene’s test (test for equal variance)

# using summary statistics 
sd(fresh)
## [1] 0.3233505
sd(stored)
## [1] 0.2406011

Parametric Test

t.test(Potency ~ Group, var.equal = TRUE)  
## 
##  Two Sample t-test
## 
## data:  Potency by Group
## t = 4.2368, df = 18, p-value = 0.0004959
## alternative hypothesis: true difference in means between group 1 and group 2 is not equal to 0
## 95 percent confidence interval:
##  0.2722297 0.8077703
## sample estimates:
## mean in group 1 mean in group 2 
##           10.37            9.83

Phrasing Results

Parametric Test

t.test(Potency ~ Group, var.equal = FALSE, conf.level = 0.90)  
## 
##  Welch Two Sample t-test
## 
## data:  Potency by Group
## t = 4.2368, df = 16.628, p-value = 0.000581
## alternative hypothesis: true difference in means between group 1 and group 2 is not equal to 0
## 90 percent confidence interval:
##  0.3179948 0.7620052
## sample estimates:
## mean in group 1 mean in group 2 
##           10.37            9.83

Non-parametric Test

wilcox.test(Potency ~ Group, var.equal = TRUE)
## Warning in wilcox.test.default(x = DATA[[1L]], y = DATA[[2L]], ...): cannot
## compute exact p-value with ties
## 
##  Wilcoxon rank sum test with continuity correction
## 
## data:  Potency by Group
## W = 91, p-value = 0.00211
## alternative hypothesis: true location shift is not equal to 0

Phrasing Results from Wilcoxon-Mann-Whitney Test

Example 2: Two-sample Independent t-tests (data from Excel)

head(Handout2) 
##   COMMIT AGE SEX SCHTYPE SCHLEVEL SALARY CLASSSIZE RESOURCES AUTONOMY CLIMATE
## 1     38  30   1       1        1  42.28        26         5       12      12
## 2     10  30   1       1        3  40.15        21         5       11       9
## 3     30  26   1       1        1  41.83        28         7        9      10
## 4     10  27   1       1        3  37.00        22         7        5      10
## 5     16  24   1       1        1  39.03        25         7       10      11
## 6     38  30   1       1        1  40.79        23         5       10       9
##   SUPPORT
## 1      12
## 2      11
## 3      12
## 4       8
## 5       9
## 6      13

Normality

shapiro.test(Handout2$COMMIT)
## 
##  Shapiro-Wilk normality test
## 
## data:  Handout2$COMMIT
## W = 0.99047, p-value = 0.4027

Normality

ggplot(data = Handout2,
aes(x = factor(SEX), y = COMMIT)) +
geom_boxplot() +
labs(title = "Commitment Scores versus Sex",
x = "Sex", y = "Commitment Scores")

Levene’s test

var.test(Handout2$COMMIT ~ Handout2$SEX)
## 
##  F test to compare two variances
## 
## data:  Handout2$COMMIT by Handout2$SEX
## F = 0.86064, num df = 78, denom df = 71, p-value = 0.5164
## alternative hypothesis: true ratio of variances is not equal to 1
## 95 percent confidence interval:
##  0.5429501 1.3574243
## sample estimates:
## ratio of variances 
##          0.8606385

Parametric Test

t.test(Handout2$COMMIT ~ Handout2$SEX, var.equal = TRUE) 
## 
##  Two Sample t-test
## 
## data:  Handout2$COMMIT by Handout2$SEX
## t = -2.0004, df = 149, p-value = 0.04728
## alternative hypothesis: true difference in means between group 1 and group 2 is not equal to 0
## 95 percent confidence interval:
##  -9.87413819 -0.06046097
## sample estimates:
## mean in group 1 mean in group 2 
##        37.94937        42.91667

Non-parametric Test

wilcox.test(Handout2$COMMIT ~ Handout2$SEX, var.equal = TRUE) 
## 
##  Wilcoxon rank sum test with continuity correction
## 
## data:  Handout2$COMMIT by Handout2$SEX
## W = 2347, p-value = 0.06429
## alternative hypothesis: true location shift is not equal to 0

Multivariate Data

Outline

  1. Multivariate data and dimension reduction
  2. Principal Components Analysis (PCA): Application

Main Goal: Practice using R to implement a statistical method to analyze multivariate data. This slides shows you how to use the R statistical software to carry out some simple multivariate analyses, with a focus on principal components analysis (PCA). PCA is used in exploratory data analysis and for making decisions in predictive models.

Part 1: Multivariate Data

Example 1: Multivariate Data

data(gapminder)
head(gapminder)
## # A tibble: 6 × 6
##   country     continent  year lifeExp      pop gdpPercap
##   <fct>       <fct>     <int>   <dbl>    <int>     <dbl>
## 1 Afghanistan Asia       1952    28.8  8425333      779.
## 2 Afghanistan Asia       1957    30.3  9240934      821.
## 3 Afghanistan Asia       1962    32.0 10267083      853.
## 4 Afghanistan Asia       1967    34.0 11537966      836.
## 5 Afghanistan Asia       1972    36.1 13079460      740.
## 6 Afghanistan Asia       1977    38.4 14880372      786.
names(gapminder)
## [1] "country"   "continent" "year"      "lifeExp"   "pop"       "gdpPercap"

Example 1: Multivariate Data

cor(gapminder$lifeExp, gapminder$gdpPercap)
## [1] 0.5837062

Example 1: Multivariate Data

Example 1: Multivariate Data

## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## `geom_smooth()` using formula = 'y ~ x'

Example 2: Multivariate Data (Another Example)

# library(car)
# wine <- read.table("http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine #  .data", sep=",")

head(wine) 
##   V1    V2   V3   V4   V5  V6   V7   V8   V9  V10  V11  V12  V13  V14
## 1  1 14.23 1.71 2.43 15.6 127 2.80 3.06 0.28 2.29 5.64 1.04 3.92 1065
## 2  1 13.20 1.78 2.14 11.2 100 2.65 2.76 0.26 1.28 4.38 1.05 3.40 1050
## 3  1 13.16 2.36 2.67 18.6 101 2.80 3.24 0.30 2.81 5.68 1.03 3.17 1185
## 4  1 14.37 1.95 2.50 16.8 113 3.85 3.49 0.24 2.18 7.80 0.86 3.45 1480
## 5  1 13.24 2.59 2.87 21.0 118 2.80 2.69 0.39 1.82 4.32 1.04 2.93  735
## 6  1 14.20 1.76 2.45 15.2 112 3.27 3.39 0.34 1.97 6.75 1.05 2.85 1450
table(wine$V1)
## 
##  1  2  3 
## 59 71 48

Example 2: Multivariate Data

head(wine[2:6])
##      V2   V3   V4   V5  V6
## 1 14.23 1.71 2.43 15.6 127
## 2 13.20 1.78 2.14 11.2 100
## 3 13.16 2.36 2.67 18.6 101
## 4 14.37 1.95 2.50 16.8 113
## 5 13.24 2.59 2.87 21.0 118
## 6 14.20 1.76 2.45 15.2 112

Example 2: Plotting Multivariate Data

scatterplotMatrix(wine[2:6])

Example 2: Scatterplot

plot(wine$V4, wine$V5)

Example 2: Scatterplot

wine$V1
##   [1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
##  [38] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
##  [75] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
## [112] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
## [149] 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3

Example 2: Scatterplot

plot(wine$V4, wine$V5)
text(wine$V4, wine$V5, wine$V1, cex=0.7, pos=4, col="red")

Example 2: Calculating Correlations for Multivariate Data

cor(wine$V2, wine$V3)
## [1] 0.09439694
cor.test(wine$V2, wine$V3)
## 
##  Pearson's product-moment correlation
## 
## data:  wine$V2 and wine$V3
## t = 1.2579, df = 176, p-value = 0.2101
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.05342959  0.23817474
## sample estimates:
##        cor 
## 0.09439694

Example 2: Calculating Correlations for Multivariate Data

Example 2: Calculating Correlations for Multivariate Data

mosthighlycorrelated <- function(mydataframe,numtoreport){
     # find the correlations
     cormatrix <- cor(mydataframe)
     # set the correlations on the diagonal or lower triangle to zero,
     # so they will not be reported as the highest ones:
     diag(cormatrix) <- 0
     cormatrix[lower.tri(cormatrix)] <- 0
     # flatten the matrix into a dataframe for easy sorting
     fm <- as.data.frame(as.table(cormatrix))
     # assign human-friendly names
     names(fm) <- c("First.Variable", "Second.Variable","Correlation")
     # sort and print the top n correlations
     head(fm[order(abs(fm$Correlation),decreasing=T),],n=numtoreport)}

mosthighlycorrelated(wine[2:14], 10)
##     First.Variable Second.Variable Correlation
## 84              V7              V8   0.8645635
## 150             V8             V13   0.7871939
## 149             V7             V13   0.6999494
## 111             V8             V10   0.6526918
## 157             V2             V14   0.6437200
## 110             V7             V10   0.6124131
## 154            V12             V13   0.5654683
## 132             V3             V12  -0.5612957
## 118             V2             V11   0.5463642
## 137             V8             V12   0.5434786

Example 2: Calculating Correlations for Multivariate Data

mosthighlycorrelated(wine[2:14], 15)
##     First.Variable Second.Variable Correlation
## 84              V7              V8   0.8645635
## 150             V8             V13   0.7871939
## 149             V7             V13   0.6999494
## 111             V8             V10   0.6526918
## 157             V2             V14   0.6437200
## 110             V7             V10   0.6124131
## 154            V12             V13   0.5654683
## 132             V3             V12  -0.5612957
## 118             V2             V11   0.5463642
## 137             V8             V12   0.5434786
## 98              V8              V9  -0.5378996
## 140            V11             V12  -0.5218132
## 152            V10             V13   0.5190671
## 151             V9             V13  -0.5032696
## 162             V7             V14   0.4981149

Multivariate Data

Part 2: Principal Components Analysis (PCA)

Part 2: Principal Components Analysis (PCA)

Part 2: Principal Components Analysis (PCA)

summary(wine[2:6])
##        V2              V3              V4              V5       
##  Min.   :11.03   Min.   :0.740   Min.   :1.360   Min.   :10.60  
##  1st Qu.:12.36   1st Qu.:1.603   1st Qu.:2.210   1st Qu.:17.20  
##  Median :13.05   Median :1.865   Median :2.360   Median :19.50  
##  Mean   :13.00   Mean   :2.336   Mean   :2.367   Mean   :19.49  
##  3rd Qu.:13.68   3rd Qu.:3.083   3rd Qu.:2.558   3rd Qu.:21.50  
##  Max.   :14.83   Max.   :5.800   Max.   :3.230   Max.   :30.00  
##        V6        
##  Min.   : 70.00  
##  1st Qu.: 88.00  
##  Median : 98.00  
##  Mean   : 99.74  
##  3rd Qu.:107.00  
##  Max.   :162.00

Part 2: Principal Components Analysis (PCA)

## standardise the variables
standardisedconcentrations <- as.data.frame(scale(wine[2:14]))

# do a PCA
wine.pca <- prcomp(standardisedconcentrations) 
wine.pca
## Standard deviations (1, .., p=13):
##  [1] 2.1692972 1.5801816 1.2025273 0.9586313 0.9237035 0.8010350 0.7423128
##  [8] 0.5903367 0.5374755 0.5009017 0.4751722 0.4108165 0.3215244
## 
## Rotation (n x k) = (13 x 13):
##              PC1          PC2         PC3         PC4         PC5         PC6
## V2  -0.144329395  0.483651548 -0.20738262  0.01785630 -0.26566365  0.21353865
## V3   0.245187580  0.224930935  0.08901289 -0.53689028  0.03521363  0.53681385
## V4   0.002051061  0.316068814  0.62622390  0.21417556 -0.14302547  0.15447466
## V5   0.239320405 -0.010590502  0.61208035 -0.06085941  0.06610294 -0.10082451
## V6  -0.141992042  0.299634003  0.13075693  0.35179658  0.72704851  0.03814394
## V7  -0.394660845  0.065039512  0.14617896 -0.19806835 -0.14931841 -0.08412230
## V8  -0.422934297 -0.003359812  0.15068190 -0.15229479 -0.10902584 -0.01892002
## V9   0.298533103  0.028779488  0.17036816  0.20330102 -0.50070298 -0.25859401
## V10 -0.313429488  0.039301722  0.14945431 -0.39905653  0.13685982 -0.53379539
## V11  0.088616705  0.529995672 -0.13730621 -0.06592568 -0.07643678 -0.41864414
## V12 -0.296714564 -0.279235148  0.08522192  0.42777141 -0.17361452  0.10598274
## V13 -0.376167411 -0.164496193  0.16600459 -0.18412074 -0.10116099  0.26585107
## V14 -0.286752227  0.364902832 -0.12674592  0.23207086 -0.15786880  0.11972557
##             PC7         PC8         PC9        PC10        PC11        PC12
## V2  -0.05639636  0.39613926 -0.50861912  0.21160473  0.22591696 -0.26628645
## V3   0.42052391  0.06582674  0.07528304 -0.30907994 -0.07648554  0.12169604
## V4  -0.14917061 -0.17026002  0.30769445 -0.02712539  0.49869142 -0.04962237
## V5  -0.28696914  0.42797018 -0.20044931  0.05279942 -0.47931378 -0.05574287
## V6   0.32288330 -0.15636143 -0.27140257  0.06787022 -0.07128891  0.06222011
## V7  -0.02792498 -0.40593409 -0.28603452 -0.32013135 -0.30434119 -0.30388245
## V8  -0.06068521 -0.18724536 -0.04957849 -0.16315051  0.02569409 -0.04289883
## V9   0.59544729 -0.23328465 -0.19550132  0.21553507 -0.11689586  0.04235219
## V10  0.37213935  0.36822675  0.20914487  0.13418390  0.23736257 -0.09555303
## V11 -0.22771214 -0.03379692 -0.05621752 -0.29077518 -0.03183880  0.60422163
## V12  0.23207564  0.43662362 -0.08582839 -0.52239889  0.04821201  0.25921400
## V13 -0.04476370 -0.07810789 -0.13722690  0.52370587 -0.04642330  0.60095872
## V14  0.07680450  0.12002267  0.57578611  0.16211600 -0.53926983 -0.07940162
##            PC13
## V2   0.01496997
## V3   0.02596375
## V4  -0.14121803
## V5   0.09168285
## V6   0.05677422
## V7  -0.46390791
## V8   0.83225706
## V9   0.11403985
## V10 -0.11691707
## V11 -0.01199280
## V12 -0.08988884
## V13 -0.15671813
## V14  0.01444734

Part 2: Principal Components Analysis (PCA)

summary(wine.pca)
## Importance of components:
##                          PC1    PC2    PC3     PC4     PC5     PC6     PC7
## Standard deviation     2.169 1.5802 1.2025 0.95863 0.92370 0.80103 0.74231
## Proportion of Variance 0.362 0.1921 0.1112 0.07069 0.06563 0.04936 0.04239
## Cumulative Proportion  0.362 0.5541 0.6653 0.73599 0.80162 0.85098 0.89337
##                            PC8     PC9   PC10    PC11    PC12    PC13
## Standard deviation     0.59034 0.53748 0.5009 0.47517 0.41082 0.32152
## Proportion of Variance 0.02681 0.02222 0.0193 0.01737 0.01298 0.00795
## Cumulative Proportion  0.92018 0.94240 0.9617 0.97907 0.99205 1.00000

Part 2: Principal Components Analysis (PCA)

wine.pca$sdev
##  [1] 2.1692972 1.5801816 1.2025273 0.9586313 0.9237035 0.8010350 0.7423128
##  [8] 0.5903367 0.5374755 0.5009017 0.4751722 0.4108165 0.3215244
sum((wine.pca$sdev)^2)
## [1] 13

Part 2: Deciding How Many Principal Components to Retain

screeplot(wine.pca, type="lines")

Part 2: Deciding How Many Principal Components to Retain (Another Approach)

(wine.pca$sdev)^2
##  [1] 4.7058503 2.4969737 1.4460720 0.9189739 0.8532282 0.6416570 0.5510283
##  [8] 0.3484974 0.2888799 0.2509025 0.2257886 0.1687702 0.1033779

Part 2: Loadings for the Principal Components

wine.pca$rotation[,1]
##           V2           V3           V4           V5           V6           V7 
## -0.144329395  0.245187580  0.002051061  0.239320405 -0.141992042 -0.394660845 
##           V8           V9          V10          V11          V12          V13 
## -0.422934297  0.298533103 -0.313429488  0.088616705 -0.296714564 -0.376167411 
##          V14 
## -0.286752227

Example 3: Principal Components Analysis (PCA)

data(iris)
head(iris)
##   Sepal.Length Sepal.Width Petal.Length Petal.Width Species
## 1          5.1         3.5          1.4         0.2  setosa
## 2          4.9         3.0          1.4         0.2  setosa
## 3          4.7         3.2          1.3         0.2  setosa
## 4          4.6         3.1          1.5         0.2  setosa
## 5          5.0         3.6          1.4         0.2  setosa
## 6          5.4         3.9          1.7         0.4  setosa
names(iris)
## [1] "Sepal.Length" "Sepal.Width"  "Petal.Length" "Petal.Width"  "Species"

Example 3: Principal Components Analysis (PCA)

summary(iris)
##   Sepal.Length    Sepal.Width     Petal.Length    Petal.Width   
##  Min.   :4.300   Min.   :2.000   Min.   :1.000   Min.   :0.100  
##  1st Qu.:5.100   1st Qu.:2.800   1st Qu.:1.600   1st Qu.:0.300  
##  Median :5.800   Median :3.000   Median :4.350   Median :1.300  
##  Mean   :5.843   Mean   :3.057   Mean   :3.758   Mean   :1.199  
##  3rd Qu.:6.400   3rd Qu.:3.300   3rd Qu.:5.100   3rd Qu.:1.800  
##  Max.   :7.900   Max.   :4.400   Max.   :6.900   Max.   :2.500  
##        Species  
##  setosa    :50  
##  versicolor:50  
##  virginica :50  
##                 
##                 
## 

Example 3: Principal Components Analysis (PCA)

cor(iris[1:4])
##              Sepal.Length Sepal.Width Petal.Length Petal.Width
## Sepal.Length    1.0000000  -0.1175698    0.8717538   0.8179411
## Sepal.Width    -0.1175698   1.0000000   -0.4284401  -0.3661259
## Petal.Length    0.8717538  -0.4284401    1.0000000   0.9628654
## Petal.Width     0.8179411  -0.3661259    0.9628654   1.0000000
pairs(iris[1:4])

Example 3: Principal Components Analysis (PCA)

## standardize the variables
standardisediris <- as.data.frame(scale(iris[1:4]))

# do a PCA
iris.pca <- prcomp(standardisediris) 

summary(iris.pca)
## Importance of components:
##                           PC1    PC2     PC3     PC4
## Standard deviation     1.7084 0.9560 0.38309 0.14393
## Proportion of Variance 0.7296 0.2285 0.03669 0.00518
## Cumulative Proportion  0.7296 0.9581 0.99482 1.00000

Example 3: Principal Components Analysis (PCA)

## Deciding How Many Principal Components to Retain

screeplot(iris.pca, type="lines")

Example 3: Principal Components Analysis (PCA)

# do a PCA
iris.pca <- prcomp(standardisediris) 
iris.pca
## Standard deviations (1, .., p=4):
## [1] 1.7083611 0.9560494 0.3830886 0.1439265
## 
## Rotation (n x k) = (4 x 4):
##                     PC1         PC2        PC3        PC4
## Sepal.Length  0.5210659 -0.37741762  0.7195664  0.2612863
## Sepal.Width  -0.2693474 -0.92329566 -0.2443818 -0.1235096
## Petal.Length  0.5804131 -0.02449161 -0.1421264 -0.8014492
## Petal.Width   0.5648565 -0.06694199 -0.6342727  0.5235971

Example 3: Principal Components Analysis (PCA)

# do a PCA
iris.pca <- prcomp(standardisediris) 

cor(data.frame(iris.pca$x))
##               PC1           PC2           PC3           PC4
## PC1  1.000000e+00 -9.760996e-17 -1.685228e-15  1.996979e-15
## PC2 -9.760996e-17  1.000000e+00  2.381560e-16  2.409194e-15
## PC3 -1.685228e-15  2.381560e-16  1.000000e+00 -6.157152e-16
## PC4  1.996979e-15  2.409194e-15 -6.157152e-16  1.000000e+00

Principal Components Regression Analysis

Another Example of Principal Components Analysis (PCA)

data(mtcars)
names(mtcars)
##  [1] "mpg"  "cyl"  "disp" "hp"   "drat" "wt"   "qsec" "vs"   "am"   "gear"
## [11] "carb"
cars_pca_df <- mtcars %>% 
  select(mpg, disp, drat, wt, qsec)
head(cars_pca_df)
##                    mpg disp drat    wt  qsec
## Mazda RX4         21.0  160 3.90 2.620 16.46
## Mazda RX4 Wag     21.0  160 3.90 2.875 17.02
## Datsun 710        22.8  108 3.85 2.320 18.61
## Hornet 4 Drive    21.4  258 3.08 3.215 19.44
## Hornet Sportabout 18.7  360 3.15 3.440 17.02
## Valiant           18.1  225 2.76 3.460 20.22

Another Example of Principal Components Analysis (PCA)

## standardize the variables
standardisedconcentrations <- as.data.frame(scale(cars_pca_df))

# do a PCA
cars.pca <- prcomp(standardisedconcentrations) 


summary(cars.pca)
## Importance of components:
##                           PC1    PC2    PC3     PC4     PC5
## Standard deviation     1.8685 0.9880 0.5714 0.39202 0.22852
## Proportion of Variance 0.6983 0.1952 0.0653 0.03074 0.01044
## Cumulative Proportion  0.6983 0.8935 0.9588 0.98956 1.00000

Another Example of Principal Components Analysis (PCA)

## Deciding How Many Principal Components to Retain

screeplot(cars.pca, type="lines")

Another Example of Principal Components Analysis (PCA)

## Deciding How Many Principal Components to Retain (Another way)
(cars.pca$sdev)^2
## [1] 3.49139561 0.97621539 0.32648787 0.15367899 0.05222214

Another Example of Principal Components Analysis (PCA)

# do a PCA
cars.pca <- prcomp(standardisedconcentrations) 
cars.pca
## Standard deviations (1, .., p=5):
## [1] 1.8685277 0.9880361 0.5713912 0.3920191 0.2285216
## 
## Rotation (n x k) = (5 x 5):
##             PC1         PC2        PC3         PC4        PC5
## mpg  -0.5032625  0.05743315  0.2415625 -0.75397195  0.3414705
## disp  0.5110687 -0.05613132 -0.1353800 -0.64001016 -0.5547227
## drat -0.4336644 -0.35497557 -0.8197828 -0.04879738 -0.1072501
## wt    0.4971610  0.21503428 -0.4398510 -0.13804916  0.7028974
## qsec -0.2242805  0.90625786 -0.2404311  0.02178386 -0.2647891

Another Example of Principal Components Analysis (PCA)

# Multiple Linear Regression
data(mtcars)
head(mtcars)
##                    mpg cyl disp  hp drat    wt  qsec vs am gear carb
## Mazda RX4         21.0   6  160 110 3.90 2.620 16.46  0  1    4    4
## Mazda RX4 Wag     21.0   6  160 110 3.90 2.875 17.02  0  1    4    4
## Datsun 710        22.8   4  108  93 3.85 2.320 18.61  1  1    4    1
## Hornet 4 Drive    21.4   6  258 110 3.08 3.215 19.44  1  0    3    1
## Hornet Sportabout 18.7   8  360 175 3.15 3.440 17.02  0  0    3    2
## Valiant           18.1   6  225 105 2.76 3.460 20.22  1  0    3    1

Regular Multiple Linear Regression

model <- lm(hp~ mpg +  disp + drat + wt +  qsec, data=mtcars)
summary(model)
## 
## Call:
## lm(formula = hp ~ mpg + disp + drat + wt + qsec, data = mtcars)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -42.959 -18.390  -4.042  14.879 102.802 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)   
## (Intercept) 392.8117   119.1055   3.298  0.00282 **
## mpg          -3.1703     2.3523  -1.348  0.18936   
## disp          0.1864     0.1356   1.375  0.18099   
## drat         12.9668    16.7779   0.773  0.44658   
## wt            7.9787    18.6915   0.427  0.67299   
## qsec        -16.6811     4.9192  -3.391  0.00223 **
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 31.74 on 26 degrees of freedom
## Multiple R-squared:  0.8203, Adjusted R-squared:  0.7858 
## F-statistic: 23.74 on 5 and 26 DF,  p-value: 6.262e-09

Another Example of Principal Components Analysis (PCA)

combine <- cbind(mtcars, data.frame(cars.pca$x))
head(combine)
##                    mpg cyl disp  hp drat    wt  qsec vs am gear carb        PC1
## Mazda RX4         21.0   6  160 110 3.90 2.620 16.46  0  1    4    4 -0.7428349
## Mazda RX4 Wag     21.0   6  160 110 3.90 2.875 17.02  0  1    4    4 -0.6835537
## Datsun 710        22.8   4  108  93 3.85 2.320 18.61  1  1    4    1 -1.4892902
## Hornet 4 Drive    21.4   6  258 110 3.08 3.215 19.44  1  0    3    1  0.2212561
## Hornet Sportabout 18.7   8  360 175 3.15 3.440 17.02  0  0    3    2  1.2285996
## Valiant           18.1   6  225 105 2.76 3.460 20.22  1  0    3    1  0.6468671
##                          PC2         PC3        PC4         PC5
## Mazda RX4         -0.9963269  0.10380021  0.2910819  0.08392926
## Mazda RX4 Wag     -0.6562789 -0.08617859  0.2619310  0.18413360
## Datsun 710         0.1020252  0.15498713  0.4075253 -0.10541625
## Hornet 4 Drive     1.1495880  0.60160117 -0.2378055 -0.18169682
## Hornet Sportabout -0.1466781  0.49910499 -0.4943902 -0.28501318
## Valiant            1.7949605  0.78092999  0.3495812 -0.09635587

Another Example of Principal Components Analysis (PCA)

model2 <- lm(hp~ PC1, data=combine)
summary(model2)
## 
## Call:
## lm(formula = hp ~ PC1, data = combine)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -60.435 -17.249  -7.495   4.353 149.253 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  146.688      7.557  19.411  < 2e-16 ***
## PC1           28.981      4.109   7.053 7.68e-08 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 42.75 on 30 degrees of freedom
## Multiple R-squared:  0.6238, Adjusted R-squared:  0.6113 
## F-statistic: 49.75 on 1 and 30 DF,  p-value: 7.68e-08

Another Example of Principal Components Analysis (PCA)

model2 <- lm(hp ~ PC1 + PC2, data=combine)
summary(model2)
## 
## Call:
## lm(formula = hp ~ PC1 + PC2, data = combine)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -45.239 -15.952  -4.245  13.318 100.560 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  146.688      5.418  27.076  < 2e-16 ***
## PC1           28.981      2.946   9.838 9.54e-11 ***
## PC2          -30.190      5.571  -5.419 7.92e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 30.65 on 29 degrees of freedom
## Multiple R-squared:  0.8131, Adjusted R-squared:  0.8002 
## F-statistic: 63.08 on 2 and 29 DF,  p-value: 2.745e-11