Data Mining: Homework 8
Alejandro Aguirre
May 2, 2019
Question 7
In this problem, you will use support vector approaches in order to predict whether a given car gets high or low gas mileage based on the Auto data set.
A.) Create a binary variable that takes on a 1 for cars with gas mileage above the median, and a 0 for cars with gas mileage below the median.
library(ISLR)
gas.med = median(Auto$mpg)
new.var = ifelse(Auto$mpg > gas.med, 1, 0)
Auto$mpglevel = as.factor(new.var)B.) Fit a support vector classifier to the data with various values of cost, in order to predict whether a car gets high or low gas mileage. Report the cross-validation errors associated with different values of this parameter. Comment on your results.
library(e1071)## Warning: package 'e1071' was built under R version 3.5.3set.seed(3255)
tune.out = tune(svm, mpglevel ~ ., data = Auto, kernel = "linear", ranges = list(cost = c(0.01, 0.1, 1, 5, 10, 100)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 1
##
## - best performance: 0.01282051
##
## - Detailed performance results:
## cost error dispersion
## 1 1e-02 0.07384615 0.04219942
## 2 1e-01 0.04083333 0.03008810
## 3 1e+00 0.01282051 0.02179068
## 4 5e+00 0.01538462 0.02477158
## 5 1e+01 0.02044872 0.02354784
## 6 1e+02 0.03070513 0.02357884C.) Now repeat (b), this time using SVMs with radial and polynomial basis kernels, with different values of gamma and degree and cost. Comment on your results.
set.seed(21)
tune.out = tune(svm, mpglevel ~ ., data = Auto, kernel = "polynomial", ranges = list(cost = c(0.1, 1, 5, 10), degree = c(2, 3, 4)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost degree
## 10 2
##
## - best performance: 0.5508974
##
## - Detailed performance results:
## cost degree error dispersion
## 1 0.1 2 0.5867308 0.07310319
## 2 1.0 2 0.5867308 0.07310319
## 3 5.0 2 0.5867308 0.07310319
## 4 10.0 2 0.5508974 0.11697667
## 5 0.1 3 0.5867308 0.07310319
## 6 1.0 3 0.5867308 0.07310319
## 7 5.0 3 0.5867308 0.07310319
## 8 10.0 3 0.5867308 0.07310319
## 9 0.1 4 0.5867308 0.07310319
## 10 1.0 4 0.5867308 0.07310319
## 11 5.0 4 0.5867308 0.07310319
## 12 10.0 4 0.5867308 0.07310319set.seed(463)
tune.out = tune(svm, mpglevel ~ ., data = Auto, kernel = "radial", ranges = list(cost = c(0.1, 1, 5, 10), gamma = c(0.01, 0.1, 1, 5, 10, 100)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost gamma
## 10 0.01
##
## - best performance: 0.02288462
##
## - Detailed performance results:
## cost gamma error dispersion
## 1 0.1 1e-02 0.08916667 0.04526330
## 2 1.0 1e-02 0.07397436 0.03896185
## 3 5.0 1e-02 0.05102564 0.03813274
## 4 10.0 1e-02 0.02288462 0.03286718
## 5 0.1 1e-01 0.07903846 0.04724112
## 6 1.0 1e-01 0.05602564 0.03950993
## 7 5.0 1e-01 0.02801282 0.02231663
## 8 10.0 1e-01 0.02551282 0.02093755
## 9 0.1 1e+00 0.55102564 0.03813274
## 10 1.0 1e+00 0.06365385 0.04199145
## 11 5.0 1e+00 0.06108974 0.04358351
## 12 10.0 1e+00 0.06108974 0.04358351
## 13 0.1 5e+00 0.55102564 0.03813274
## 14 1.0 5e+00 0.48717949 0.03963085
## 15 5.0 5e+00 0.49224359 0.04525523
## 16 10.0 5e+00 0.49224359 0.04525523
## 17 0.1 1e+01 0.55102564 0.03813274
## 18 1.0 1e+01 0.50506410 0.04235779
## 19 5.0 1e+01 0.49993590 0.04269277
## 20 10.0 1e+01 0.49993590 0.04269277
## 21 0.1 1e+02 0.55102564 0.03813274
## 22 1.0 1e+02 0.55102564 0.03813274
## 23 5.0 1e+02 0.55102564 0.03813274
## 24 10.0 1e+02 0.55102564 0.03813274D.) Make some plots to justify (C) and (D)
svm.linear = svm(mpglevel ~ ., data = Auto, kernel = "linear", cost = 1)
svm.poly = svm(mpglevel ~ ., data = Auto, kernel = "polynomial", cost = 10,
degree = 2)
svm.radial = svm(mpglevel ~ ., data = Auto, kernel = "radial", cost = 10, gamma = 0.01)
plotpairs = function(fit) {
for (name in names(Auto)[!(names(Auto) %in% c("mpg", "mpglevel", "name"))]) {
plot(fit, Auto, as.formula(paste("mpg~", name, sep = "")))
}
}
plotpairs(svm.linear)
plotpairs(svm.poly)
plotpairs(svm.radial)
Question 8
This problem involves the OJ data set which is part of the ISLR package.
A.) Create a training set containing a random sample of 800 observations, and a test set containing the remaining observations.
library(ISLR)
set.seed(9004)
train = sample(dim(OJ)[1], 800)
OJ.train = OJ[train, ]
OJ.test = OJ[-train, ]B.) Fit a support vector classifier to the training data using cost=0.01, with Purchase as the response and the other variables as predictors. Use the summary() function to produce summary statistics, and describe the results obtained.
library(e1071)
svm.linear = svm(Purchase ~ ., kernel = "linear", data = OJ.train, cost = 0.01)
summary(svm.linear)##
## Call:
## svm(formula = Purchase ~ ., data = OJ.train, kernel = "linear",
## cost = 0.01)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: linear
## cost: 0.01
## gamma: 0.05555556
##
## Number of Support Vectors: 432
##
## ( 217 215 )
##
##
## Number of Classes: 2
##
## Levels:
## CH MMC.) What are the training and test error rates?
train.pred = predict(svm.linear, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 439 53
## MM 82 226(82 + 53)/(439 + 53 + 82 + 226)## [1] 0.16875test.pred = predict(svm.linear, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 142 19
## MM 29 80(19 + 29)/(142 + 19 + 29 + 80)## [1] 0.1777778The training and testing errors are listed above.
D.) Use the tune() function to select an optimal cost. Consider values in the range 0.01 to 10.
set.seed(1554)
tune.out = tune(svm, Purchase ~ ., data = OJ.train, kernel = "linear", ranges = list(cost = 10^seq(-2, 1, by = 0.25)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 0.3162278
##
## - best performance: 0.16875
##
## - Detailed performance results:
## cost error dispersion
## 1 0.01000000 0.16875 0.03691676
## 2 0.01778279 0.16875 0.03397814
## 3 0.03162278 0.17125 0.03230175
## 4 0.05623413 0.17250 0.03162278
## 5 0.10000000 0.17000 0.03291403
## 6 0.17782794 0.17125 0.03335936
## 7 0.31622777 0.16875 0.03498512
## 8 0.56234133 0.17000 0.03129164
## 9 1.00000000 0.16875 0.03397814
## 10 1.77827941 0.16875 0.03240906
## 11 3.16227766 0.16875 0.03294039
## 12 5.62341325 0.17125 0.03120831
## 13 10.00000000 0.17125 0.03283481E.) Compute the training and test error rates using this new value for cost.
svm.linear = svm(Purchase ~ ., kernel = "linear", data = OJ.train, cost = tune.out$best.parameters$cost)
train.pred = predict(svm.linear, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 435 57
## MM 71 237(57 + 71)/(435 + 57 + 71 + 237)## [1] 0.16test.pred = predict(svm.linear, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 141 20
## MM 29 80(29 + 20)/(141 + 20 + 29 + 80)## [1] 0.1814815The testing and training error are listed above.
F.) Repeat parts (b) through (e) using a support vector machine with a radial kernel. Use the default value for gamma.
set.seed(410)
svm.radial = svm(Purchase ~ ., data = OJ.train, kernel = "radial")
summary(svm.radial)##
## Call:
## svm(formula = Purchase ~ ., data = OJ.train, kernel = "radial")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 1
## gamma: 0.05555556
##
## Number of Support Vectors: 367
##
## ( 184 183 )
##
##
## Number of Classes: 2
##
## Levels:
## CH MMtrain.pred = predict(svm.radial, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 452 40
## MM 78 230(40 + 78)/(452 + 40 + 78 + 230)## [1] 0.1475test.pred = predict(svm.radial, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 146 15
## MM 27 82(27 + 15)/(146 + 15 + 27 + 82)## [1] 0.1555556Training and test error for radial listed above. We can see that it is a slight improvement from the linear model before.
set.seed(755)
tune.out = tune(svm, Purchase ~ ., data = OJ.train, kernel = "radial", ranges = list(cost = 10^seq(-2, 1, by = 0.25)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 0.5623413
##
## - best performance: 0.165
##
## - Detailed performance results:
## cost error dispersion
## 1 0.01000000 0.38500 0.06258328
## 2 0.01778279 0.38500 0.06258328
## 3 0.03162278 0.37625 0.06908379
## 4 0.05623413 0.21000 0.03855011
## 5 0.10000000 0.18625 0.03143004
## 6 0.17782794 0.18375 0.03230175
## 7 0.31622777 0.17125 0.03438447
## 8 0.56234133 0.16500 0.03763863
## 9 1.00000000 0.17500 0.03584302
## 10 1.77827941 0.17375 0.04059026
## 11 3.16227766 0.17625 0.03747684
## 12 5.62341325 0.17625 0.03839216
## 13 10.00000000 0.17375 0.03458584svm.radial = svm(Purchase ~ ., data = OJ.train, kernel = "radial", cost = tune.out$best.parameters$cost)
train.pred = predict(svm.radial, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 452 40
## MM 77 231(77 + 40)/(452 + 40 + 77 + 231)## [1] 0.14625test.pred = predict(svm.radial, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 146 15
## MM 28 81(28 + 15)/(146 + 15 + 28 + 81)## [1] 0.1592593Tuning slightly decreases training error to 14.6% and slightly increases test error to 16%
G.) Repeat parts (b) through (e) using a support vector machine with a polynomial kernel. Set degree=2.
set.seed(8000)
svm.poly = svm(Purchase ~ ., data = OJ.train, kernel = "poly", degree = 2)
summary(svm.poly)##
## Call:
## svm(formula = Purchase ~ ., data = OJ.train, kernel = "poly",
## degree = 2)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: polynomial
## cost: 1
## degree: 2
## gamma: 0.05555556
## coef.0: 0
##
## Number of Support Vectors: 452
##
## ( 232 220 )
##
##
## Number of Classes: 2
##
## Levels:
## CH MMtrain.pred = predict(svm.poly, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 460 32
## MM 105 203(32 + 105)/(460 + 32 + 105 + 203)## [1] 0.17125test.pred = predict(svm.poly, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 149 12
## MM 37 72(12 + 37)/(149 + 12 + 37 + 72)## [1] 0.1814815This produces a train error of 17.1% and a test error of 18.1% which are slightly higher than the errors produces by radial but lower than the errors produced by linear.
set.seed(300)
tune.out = tune(svm, Purchase ~ ., data = OJ.train, kernel = "poly", degree = 2,
ranges = list(cost = 10^seq(-2, 1, by = 0.25)))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 5.623413
##
## - best performance: 0.17875
##
## - Detailed performance results:
## cost error dispersion
## 1 0.01000000 0.38500 0.04281744
## 2 0.01778279 0.36750 0.03496029
## 3 0.03162278 0.35750 0.03641962
## 4 0.05623413 0.34500 0.03238227
## 5 0.10000000 0.31250 0.03864008
## 6 0.17782794 0.25125 0.04226652
## 7 0.31622777 0.21500 0.04993051
## 8 0.56234133 0.20750 0.05210833
## 9 1.00000000 0.19875 0.04267529
## 10 1.77827941 0.18875 0.04730589
## 11 3.16227766 0.18375 0.04678927
## 12 5.62341325 0.17875 0.04788949
## 13 10.00000000 0.18750 0.04823265svm.poly = svm(Purchase ~ ., data = OJ.train, kernel = "poly", degree = 2, cost = tune.out$best.parameters$cost)
train.pred = predict(svm.poly, OJ.train)
table(OJ.train$Purchase, train.pred)## train.pred
## CH MM
## CH 455 37
## MM 84 224(37 + 84)/(455 + 37 + 84 + 224)## [1] 0.15125test.pred = predict(svm.poly, OJ.test)
table(OJ.test$Purchase, test.pred)## test.pred
## CH MM
## CH 148 13
## MM 34 75(13 + 34)/(148 + 13 + 34 + 75)## [1] 0.1740741Tuning reduces the training error to 15.12% and test error to 17.4% which is worse than radial but better than linear .
H.) Overall, which approach seems to give the best results on this data?
Overall, radial seems to be the best option with lowest errors.