Machine Learning Quiz4
Xiaoping Li
February 8, 2017
Question 1:
Load the vowel.train and vowel.test data sets.
Set the variable y to be a factor variable in both the training and test set. Then set the seed to 33833. Fit (1) a random forest predictor relating the factor variable y to the remaining variables and (2) a boosted predictor using the “gbm” method. Fit these both with the train() command in the caret package.
What are the accuracies for the two approaches on the test data set? What is the accuracy among the test set samples where the two methods agree?
library(ElemStatLearn)
library(caret)
data(vowel.train)
data(vowel.test)
vowel.train$y = as.factor(vowel.train$y)
vowel.test$y = as.factor(vowel.test$y)
set.seed(33833)
modFit_rf <- train(y ~., method = "rf", data = vowel.train)
modFit_gbm <- train(y ~., method = "gbm", data = vowel.train, verbose = FALSE)
pred_rf <- predict(modFit_rf, vowel.test)
pred_gbm <- predict(modFit_gbm, vowel.test)
predDF <- data.frame(pred_rf, pred_gbm, y = vowel.test$y)
combModFit0 <- train( y ~., data = predDF)
pred3 <- predict(combModFit0, predDF)
rbind(postResample(pred_rf, obs = vowel.test$y),
postResample(pred_gbm, obs = vowel.test$y),
postResample(pred3, obs = vowel.test$y))## Accuracy Kappa
## [1,] 0.6147186 0.5761905
## [2,] 0.5367965 0.4904762
## [3,] 0.6948052 0.6642857Question 2
Load the Alzheimer’s data using the following commands
Set the seed to 62433 and predict diagnosis with all the other variables using a random forest (“rf”), boosted trees (“gbm”) and linear discriminant analysis (“lda”) model. Stack the predictions together using random forests (“rf”). What is the resulting accuracy on the test set? Is it better or worse than each of the individual predictions?
library(caret)
library(gbm)
set.seed(3433)
library(AppliedPredictiveModeling)
data(AlzheimerDisease)
adData = data.frame(diagnosis,predictors)
inTrain = createDataPartition(adData$diagnosis, p = 3/4)[[1]]
training = adData[ inTrain,]
testing = adData[-inTrain,]
set.seed(62433)
mod1 = train(diagnosis ~ ., method = "rf", data = training)
mod2 = train(diagnosis ~ ., method = "gbm", data = training, verbose = FALSE)
mod3 = train(diagnosis ~ ., method = "lda", data = training)
pred1 = predict(mod1, testing)
pred2 = predict(mod2, testing)
pred3 = predict(mod3, testing)
confusionMatrix(testing$diagnosis, pred1)## Confusion Matrix and Statistics
##
## Reference
## Prediction Impaired Control
## Impaired 10 12
## Control 7 53
##
## Accuracy : 0.7683
## 95% CI : (0.662, 0.8544)
## No Information Rate : 0.7927
## P-Value [Acc > NIR] : 0.7565
##
## Kappa : 0.3641
## Mcnemar's Test P-Value : 0.3588
##
## Sensitivity : 0.5882
## Specificity : 0.8154
## Pos Pred Value : 0.4545
## Neg Pred Value : 0.8833
## Prevalence : 0.2073
## Detection Rate : 0.1220
## Detection Prevalence : 0.2683
## Balanced Accuracy : 0.7018
##
## 'Positive' Class : Impaired
## confusionMatrix(testing$diagnosis, pred2)## Confusion Matrix and Statistics
##
## Reference
## Prediction Impaired Control
## Impaired 14 8
## Control 9 51
##
## Accuracy : 0.7927
## 95% CI : (0.6889, 0.8743)
## No Information Rate : 0.7195
## P-Value [Acc > NIR] : 0.08536
##
## Kappa : 0.4795
## Mcnemar's Test P-Value : 1.00000
##
## Sensitivity : 0.6087
## Specificity : 0.8644
## Pos Pred Value : 0.6364
## Neg Pred Value : 0.8500
## Prevalence : 0.2805
## Detection Rate : 0.1707
## Detection Prevalence : 0.2683
## Balanced Accuracy : 0.7366
##
## 'Positive' Class : Impaired
## confusionMatrix(testing$diagnosis, pred3)## Confusion Matrix and Statistics
##
## Reference
## Prediction Impaired Control
## Impaired 16 6
## Control 13 47
##
## Accuracy : 0.7683
## 95% CI : (0.662, 0.8544)
## No Information Rate : 0.6463
## P-Value [Acc > NIR] : 0.01219
##
## Kappa : 0.4639
## Mcnemar's Test P-Value : 0.16867
##
## Sensitivity : 0.5517
## Specificity : 0.8868
## Pos Pred Value : 0.7273
## Neg Pred Value : 0.7833
## Prevalence : 0.3537
## Detection Rate : 0.1951
## Detection Prevalence : 0.2683
## Balanced Accuracy : 0.7193
##
## 'Positive' Class : Impaired
## postResample(pred1, testing$diagnosis)## Accuracy Kappa
## 0.7682927 0.3640816postResample(pred2, testing$diagnosis)## Accuracy Kappa
## 0.7926829 0.4794623postResample(pred3, testing$diagnosis)## Accuracy Kappa
## 0.7682927 0.4638679predDF = data.frame(pred1, pred2, pred3, diagnosis= testing$diagnosis)
CModFit = train(diagnosis ~., method = "rf", data = predDF)## note: only 2 unique complexity parameters in default grid. Truncating the grid to 2 .Cpred = predict(CModFit, predDF)
postResample(Cpred, testing$diagnosis)## Accuracy Kappa
## 0.8048780 0.5030303Question 3
Load the concrete data with the commands:
Set the seed to 233 and fit a lasso model to predict Compressive Strength. Which variable is the last coefficient to be set to zero as the penalty increases? (Hint: it may be useful to look up ?plot.enet).
set.seed(3523)
library(AppliedPredictiveModeling)
data(concrete)
inTrain = createDataPartition(concrete$CompressiveStrength, p = 3/4)[[1]]
training = concrete[ inTrain,]
testing = concrete[-inTrain,]
set.seed(233)
fit <- train(CompressiveStrength ~., method = "lasso", data = training)
predict.enet(fit$finalModel, type = "coefficients")## $s
## [1] 1 2 3 4 5 6 7 8 9 10 11
##
## $fraction
## [1] 0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0
##
## $mode
## [1] "step"
##
## $coefficients
## Cement BlastFurnaceSlag FlyAsh Water Superplasticizer
## 0 0.00000000 0.00000000 0.00000000 0.0000000 0.0000000
## 1 0.01802441 0.00000000 0.00000000 0.0000000 0.0000000
## 2 0.02729231 0.00000000 0.00000000 0.0000000 0.1559023
## 3 0.03847584 0.00000000 0.00000000 0.0000000 0.4343197
## 4 0.04599070 0.01006844 0.00000000 0.0000000 0.5510774
## 5 0.06499050 0.03893552 0.00000000 -0.1174308 0.5737881
## 6 0.06678431 0.04187677 0.00000000 -0.1401379 0.5739760
## 7 0.08502706 0.06390658 0.03312720 -0.1768114 0.4070855
## 8 0.10150634 0.08388323 0.06377012 -0.2190396 0.2504283
## 9 0.10301008 0.08567182 0.06624407 -0.2188092 0.2387808
## 10 0.11351343 0.09842089 0.08097549 -0.1794448 0.2567532
## CoarseAggregate FineAggregate Age
## 0 0.00000000 0.000000000 0.00000000
## 1 0.00000000 0.000000000 0.00000000
## 2 0.00000000 0.000000000 0.00000000
## 3 0.00000000 0.000000000 0.02765634
## 4 0.00000000 0.000000000 0.04042040
## 5 0.00000000 0.000000000 0.07968720
## 6 0.00000000 -0.003483276 0.08499933
## 7 0.00000000 0.000000000 0.09810130
## 8 0.00000000 0.000000000 0.11088807
## 9 0.00000000 0.001412076 0.11162794
## 10 0.01130084 0.014480343 0.11346049plot(fit$finalModel, xvar = "step")
Question 4
Load the data on the number of visitors to the instructors blog from here:
Using the commands:
url <- "https://d396qusza40orc.cloudfront.net/predmachlearn/gaData.csv"
filename <- "gaData.csv"
tmp <- tempfile()
if(!file.exists(filename))download.file(url, destfile = filename)
gaData <- read.csv(filename)
library(lubridate)
library(forecast)
training <- gaData[year(gaData$date) < 2012,]
testing <- gaData[year(gaData$date) > 2011,]
tstrain <- ts(training$visitsTumblr)
mod <- bats(tstrain)
## h is the length of testing set or ts testing
fcast <- forecast(mod, h = 235)
tstest <- ts(testing$visitsTumblr)
mean(as.numeric(tstest) < fcast$upper[,2] & as.numeric(tstest) > fcast$lower[,2])## [1] 0.9617021Question 5
Load the concrete data with the commands:
Set the seed to 325 and fit a support vector machine using the e1071 package to predict Compressive Strength using the default settings. Predict on the testing set. What is the RMSE?
library(e1071)
set.seed(3523)
data(concrete)
inTrain = createDataPartition(concrete$CompressiveStrength, p = 3/4)[[1]]
training = concrete[ inTrain,]
testing = concrete[-inTrain,]
set.seed(325)
svm.mod <- svm(CompressiveStrength ~., data = training)
pred <- predict(svm.mod, testing)
RMSE <- sqrt(sum((testing$CompressiveStrength - pred)^2)/nrow(testing))