#Chapter 9 question 5
library(e1071)
## Warning: package 'e1071' was built under R version 4.5.3
set.seed(1)
x1 <- runif(500) - 0.5
x2 <- runif(500) - 0.5
y <- 1 * (x1^2 - x2^2 > 0)
dat <- data.frame(
x1 = x1,
x2 = x2,
y = factor(y)
)
table(dat$y)
##
## 0 1
## 261 239
#b
plot(
dat$x1,
dat$x2,
col = ifelse(dat$y == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "True Class Labels"
)
legend(
"topright",
legend = c("Class 0", "Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
#c
logistic_linear <- glm(
y ~ x1 + x2,
data = dat,
family = binomial
)
summary(logistic_linear)
##
## Call:
## glm(formula = y ~ x1 + x2, family = binomial, data = dat)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -0.087260 0.089579 -0.974 0.330
## x1 0.196199 0.316864 0.619 0.536
## x2 -0.002854 0.305712 -0.009 0.993
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 692.18 on 499 degrees of freedom
## Residual deviance: 691.79 on 497 degrees of freedom
## AIC: 697.79
##
## Number of Fisher Scoring iterations: 3
#d
linear_prob <- predict(
logistic_linear,
newdata = dat,
type = "response"
)
linear_pred <- ifelse(linear_prob > 0.50, 1, 0)
linear_pred <- factor(
linear_pred,
levels = c(0, 1)
)
# Confusion matrix
table(
Predicted = linear_pred,
Actual = dat$y
)
## Actual
## Predicted 0 1
## 0 258 212
## 1 3 27
# Accuracy
linear_accuracy <- mean(linear_pred == dat$y)
# Error rate
linear_error <- mean(linear_pred != dat$y)
linear_accuracy
## [1] 0.57
linear_error
## [1] 0.43
plot(
dat$x1,
dat$x2,
col = ifelse(linear_pred == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "Linear Logistic Regression Predictions"
)
legend(
"topright",
legend = c("Predicted Class 0", "Predicted Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
#e
logistic_nonlinear <- glm(
y ~ x1 + x2 + I(x1^2) + I(x2^2) + I(x1 * x2),
data = dat,
family = binomial
)
## Warning: glm.fit: algorithm did not converge
## Warning: glm.fit: fitted probabilities numerically 0 or 1 occurred
summary(logistic_nonlinear)
##
## Call:
## glm(formula = y ~ x1 + x2 + I(x1^2) + I(x2^2) + I(x1 * x2), family = binomial,
## data = dat)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -10.16 713.54 -0.014 0.989
## x1 42.10 15492.58 0.003 0.998
## x2 -66.81 14788.95 -0.005 0.996
## I(x1^2) 16757.98 519013.02 0.032 0.974
## I(x2^2) -16671.65 508668.89 -0.033 0.974
## I(x1 * x2) -206.38 41802.81 -0.005 0.996
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 6.9218e+02 on 499 degrees of freedom
## Residual deviance: 3.5810e-06 on 494 degrees of freedom
## AIC: 12
##
## Number of Fisher Scoring iterations: 25
#f
nonlinear_prob <- predict(
logistic_nonlinear,
newdata = dat,
type = "response"
)
nonlinear_pred <- ifelse(nonlinear_prob > 0.50, 1, 0)
nonlinear_pred <- factor(
nonlinear_pred,
levels = c(0, 1)
)
# Confusion matrix
table(
Predicted = nonlinear_pred,
Actual = dat$y
)
## Actual
## Predicted 0 1
## 0 261 0
## 1 0 239
# Accuracy
nonlinear_accuracy <- mean(nonlinear_pred == dat$y)
# Error rate
nonlinear_error <- mean(nonlinear_pred != dat$y)
nonlinear_accuracy
## [1] 1
nonlinear_error
## [1] 0
# Plot predicted classes
plot(
dat$x1,
dat$x2,
col = ifelse(nonlinear_pred == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "Nonlinear Logistic Regression Predictions"
)
legend(
"topright",
legend = c("Predicted Class 0", "Predicted Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
#g
svc_fit <- svm(
y ~ x1 + x2,
data = dat,
kernel = "linear",
cost = 1,
scale = TRUE
)
summary(svc_fit)
##
## Call:
## svm(formula = y ~ x1 + x2, data = dat, kernel = "linear", cost = 1,
## scale = TRUE)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: linear
## cost: 1
##
## Number of Support Vectors: 480
##
## ( 239 241 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
# Predictions
svc_pred <- predict(
svc_fit,
newdata = dat
)
svc_pred <- factor(
svc_pred,
levels = levels(dat$y)
)
# Confusion matrix
table(
Predicted = svc_pred,
Actual = dat$y
)
## Actual
## Predicted 0 1
## 0 261 239
## 1 0 0
# Accuracy
svc_accuracy <- mean(svc_pred == dat$y)
# Error rate
svc_error <- mean(svc_pred != dat$y)
svc_accuracy
## [1] 0.522
svc_error
## [1] 0.478
# Number of support vectors
length(svc_fit$index)
## [1] 480
# Support vector observation numbers
svc_fit$index
## [1] 1 4 5 7 10 12 15 17 18 20 21 22 23 24 25 27 34 37
## [19] 38 41 43 49 52 55 56 61 66 68 69 72 76 77 78 80 85 86
## [37] 88 90 91 92 94 96 97 98 99 100 104 107 111 112 113 114 115 116
## [55] 118 121 125 128 129 132 133 135 139 149 153 158 161 162 164 165 167 169
## [73] 172 173 174 175 176 177 180 182 183 185 186 187 188 192 193 194 195 196
## [91] 197 198 200 204 205 208 210 211 212 214 215 216 218 219 222 225 227 228
## [109] 229 232 241 242 243 244 246 247 248 250 251 252 256 257 260 267 268 270
## [127] 271 272 275 276 278 281 283 284 285 286 287 288 289 293 294 296 297 298
## [145] 299 300 306 307 309 311 312 313 315 319 321 322 324 327 328 329 330 331
## [163] 334 335 336 340 341 344 345 346 348 350 352 355 357 358 359 363 366 367
## [181] 368 373 375 378 379 384 385 388 389 390 393 395 397 400 402 403 404 405
## [199] 409 411 413 427 428 429 430 431 432 435 436 438 440 441 444 445 448 450
## [217] 451 455 458 459 461 463 465 466 467 472 473 475 482 484 487 488 490 491
## [235] 492 493 494 499 500 2 3 6 8 9 11 13 14 16 19 26 28 29
## [253] 31 32 33 35 36 39 40 42 44 45 46 48 50 51 53 54 57 58
## [271] 59 60 62 63 64 65 67 70 71 73 74 75 79 81 82 83 84 87
## [289] 93 95 101 102 103 105 108 109 110 117 119 120 122 123 124 126 127 130
## [307] 131 134 136 137 138 140 141 142 143 144 145 146 148 150 151 152 154 155
## [325] 156 157 159 160 163 166 168 170 171 178 179 181 184 189 190 191 201 202
## [343] 203 206 207 209 213 217 220 223 224 226 230 231 233 234 235 236 237 238
## [361] 239 240 245 249 253 254 255 258 259 261 262 263 264 265 266 269 273 279
## [379] 280 282 290 291 292 295 301 302 303 304 305 308 310 314 316 317 318 320
## [397] 323 325 326 332 333 337 338 339 342 343 347 349 351 353 354 360 362 365
## [415] 369 370 371 372 374 377 381 382 383 386 387 392 394 396 398 401 406 407
## [433] 408 410 412 414 415 416 417 418 419 420 421 422 423 425 426 433 434 437
## [451] 439 442 443 446 447 449 452 453 454 456 457 460 462 464 468 470 474 476
## [469] 477 478 479 480 481 483 485 486 495 496 497 498
# Plot predicted classes
plot(
dat$x1,
dat$x2,
col = ifelse(svc_pred == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "Linear Support Vector Classifier Predictions"
)
legend(
"topright",
legend = c("Predicted Class 0", "Predicted Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
#h
svm_radial <- svm(
y ~ x1 + x2,
data = dat,
kernel = "radial",
cost = 10,
gamma = 1,
scale = TRUE
)
summary(svm_radial)
##
## Call:
## svm(formula = y ~ x1 + x2, data = dat, kernel = "radial", cost = 10,
## gamma = 1, scale = TRUE)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 10
##
## Number of Support Vectors: 72
##
## ( 36 36 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
# Predictions
radial_pred <- predict(
svm_radial,
newdata = dat
)
table(
Predicted = radial_pred,
Actual = dat$y
)
## Actual
## Predicted 0 1
## 0 257 3
## 1 4 236
radial_accuracy <- mean(radial_pred == dat$y)
radial_error <- mean(radial_pred != dat$y)
radial_accuracy
## [1] 0.986
radial_error
## [1] 0.014
plot(
dat$x1,
dat$x2,
col = ifelse(radial_pred == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "Radial Kernel SVM Predictions"
)
legend(
"topright",
legend = c("Predicted Class 0", "Predicted Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
set.seed(1)
tune_output <- tune(
svm,
y ~ x1 + x2,
data = dat,
kernel = "radial",
ranges = list(
cost = c(0.1, 1, 10, 100),
gamma = c(0.5, 1, 2, 4)
)
)
summary(tune_output)
##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost gamma
## 100 1
##
## - best performance: 0.016
##
## - Detailed performance results:
## cost gamma error dispersion
## 1 0.1 0.5 0.072 0.04341019
## 2 1.0 0.5 0.054 0.04115013
## 3 10.0 0.5 0.038 0.03190263
## 4 100.0 0.5 0.028 0.01686548
## 5 0.1 1.0 0.062 0.04565572
## 6 1.0 1.0 0.044 0.04195235
## 7 10.0 1.0 0.036 0.03238655
## 8 100.0 1.0 0.016 0.01577621
## 9 0.1 2.0 0.056 0.04087923
## 10 1.0 2.0 0.034 0.03405877
## 11 10.0 2.0 0.022 0.02394438
## 12 100.0 2.0 0.018 0.01475730
## 13 0.1 4.0 0.050 0.04642796
## 14 1.0 4.0 0.028 0.02699794
## 15 10.0 4.0 0.020 0.01885618
## 16 100.0 4.0 0.020 0.01632993
# Extract best model
best_radial <- tune_output$best.model
summary(best_radial)
##
## Call:
## best.tune(METHOD = svm, train.x = y ~ x1 + x2, data = dat, ranges = list(cost = c(0.1,
## 1, 10, 100), gamma = c(0.5, 1, 2, 4)), kernel = "radial")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 100
##
## Number of Support Vectors: 36
##
## ( 18 18 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
# Best tuning parameters
tune_output$best.parameters
## cost gamma
## 8 100 1
# Predictions from tuned model
best_radial_pred <- predict(
best_radial,
newdata = dat
)
# Confusion matrix
table(
Predicted = best_radial_pred,
Actual = dat$y
)
## Actual
## Predicted 0 1
## 0 260 1
## 1 1 238
# Accuracy
best_radial_accuracy <- mean(best_radial_pred == dat$y)
# Error rate
best_radial_error <- mean(best_radial_pred != dat$y)
best_radial_accuracy
## [1] 0.996
best_radial_error
## [1] 0.004
plot(
dat$x1,
dat$x2,
col = ifelse(best_radial_pred == "1", "red", "blue"),
pch = 19,
xlab = "X1",
ylab = "X2",
main = "Tuned Radial Kernel SVM Predictions"
)
legend(
"topright",
legend = c("Predicted Class 0", "Predicted Class 1"),
col = c("blue", "red"),
pch = 19,
bty = "n"
)
model_results <- data.frame(
Model = c(
"Linear Logistic Regression",
"Nonlinear Logistic Regression",
"Linear Support Vector Classifier",
"Radial Kernel SVM",
"Tuned Radial Kernel SVM"
),
Accuracy = c(
linear_accuracy,
nonlinear_accuracy,
svc_accuracy,
radial_accuracy,
best_radial_accuracy
),
Error_Rate = c(
linear_error,
nonlinear_error,
svc_error,
radial_error,
best_radial_error
)
)
model_results[
order(model_results$Accuracy, decreasing = TRUE),
]
## Model Accuracy Error_Rate
## 2 Nonlinear Logistic Regression 1.000 0.000
## 5 Tuned Radial Kernel SVM 0.996 0.004
## 4 Radial Kernel SVM 0.986 0.014
## 1 Linear Logistic Regression 0.570 0.430
## 3 Linear Support Vector Classifier 0.522 0.478
# Question 7 chapter 9
library(ISLR2)
## Warning: package 'ISLR2' was built under R version 4.5.3
library(e1071)
data(Auto)
Auto <- na.omit(Auto)
median_mpg <- median(Auto$mpg)
Auto$mpg01 <- ifelse(Auto$mpg > median_mpg, 1, 0)
Auto$mpg01 <- as.factor(Auto$mpg01)
table(Auto$mpg01)
##
## 0 1
## 196 196
Auto_svm <- subset(Auto, select = -c(mpg, name))
set.seed(1)
tune_linear <- tune(
svm,
mpg01 ~ .,
data = Auto_svm,
kernel = "linear",
ranges = list(
cost = c(0.01,0.1,1,5,10,100)
)
)
summary(tune_linear)
##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 1
##
## - best performance: 0.08435897
##
## - Detailed performance results:
## cost error dispersion
## 1 1e-02 0.08923077 0.04698309
## 2 1e-01 0.09185897 0.04393409
## 3 1e+00 0.08435897 0.03662670
## 4 5e+00 0.08948718 0.03898410
## 5 1e+01 0.08948718 0.03898410
## 6 1e+02 0.08692308 0.03887151
###########
best_linear <- tune_linear$best.model
summary(best_linear)
##
## Call:
## best.tune(METHOD = svm, train.x = mpg01 ~ ., data = Auto_svm, ranges = list(cost = c(0.01,
## 0.1, 1, 5, 10, 100)), kernel = "linear")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: linear
## cost: 1
##
## Number of Support Vectors: 88
##
## ( 43 45 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
##############
linear_pred <- predict(best_linear, Auto_svm)
table(
Predicted = linear_pred,
Actual = Auto_svm$mpg01
)
## Actual
## Predicted 0 1
## 0 172 9
## 1 24 187
linear_accuracy <- mean(linear_pred == Auto_svm$mpg01)
linear_accuracy
## [1] 0.9158163
set.seed(1)
tune_radial <- tune(
svm,
mpg01 ~ .,
data = Auto_svm,
kernel = "radial",
ranges = list(
cost = c(0.1,1,10,100),
gamma = c(0.01,0.1,1,2)
)
)
summary(tune_radial)
##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost gamma
## 1 1
##
## - best performance: 0.06634615
##
## - Detailed performance results:
## cost gamma error dispersion
## 1 0.1 0.01 0.11224359 0.03836937
## 2 1.0 0.01 0.08673077 0.04551036
## 3 10.0 0.01 0.08673077 0.04040897
## 4 100.0 0.01 0.08685897 0.03483004
## 5 0.1 0.10 0.08923077 0.04698309
## 6 1.0 0.10 0.08923077 0.04376306
## 7 10.0 0.10 0.08166667 0.04149504
## 8 100.0 0.10 0.08410256 0.03390616
## 9 0.1 1.00 0.08673077 0.04535158
## 10 1.0 1.00 0.06634615 0.03244101
## 11 10.0 1.00 0.08923077 0.02732003
## 12 100.0 1.00 0.10448718 0.04560852
## 13 0.1 2.00 0.14282051 0.07578262
## 14 1.0 2.00 0.08673077 0.04371113
## 15 10.0 2.00 0.09429487 0.05387705
## 16 100.0 2.00 0.09435897 0.05261602
best_radial <- tune_radial$best.model
summary(best_radial)
##
## Call:
## best.tune(METHOD = svm, train.x = mpg01 ~ ., data = Auto_svm, ranges = list(cost = c(0.1,
## 1, 10, 100), gamma = c(0.01, 0.1, 1, 2)), kernel = "radial")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 1
##
## Number of Support Vectors: 184
##
## ( 92 92 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
radial_pred <- predict(best_radial, Auto_svm)
table(
Predicted = radial_pred,
Actual = Auto_svm$mpg01
)
## Actual
## Predicted 0 1
## 0 188 6
## 1 8 190
radial_accuracy <- mean(radial_pred == Auto_svm$mpg01)
radial_accuracy
## [1] 0.9642857
set.seed(1)
tune_poly <- tune(
svm,
mpg01 ~ .,
data = Auto_svm,
kernel = "polynomial",
ranges = list(
cost = c(0.1,1,10),
degree = c(2,3,4)
)
)
summary(tune_poly)
##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost degree
## 10 3
##
## - best performance: 0.08435897
##
## - Detailed performance results:
## cost degree error dispersion
## 1 0.1 2 0.27846154 0.09486227
## 2 1.0 2 0.25307692 0.13751948
## 3 10.0 2 0.18647436 0.05598001
## 4 0.1 3 0.20192308 0.11347783
## 5 1.0 3 0.09448718 0.04180527
## 6 10.0 3 0.08435897 0.04544023
## 7 0.1 4 0.26564103 0.09977887
## 8 1.0 4 0.21205128 0.09560470
## 9 10.0 4 0.16589744 0.06962914
best_poly <- tune_poly$best.model
summary(best_poly)
##
## Call:
## best.tune(METHOD = svm, train.x = mpg01 ~ ., data = Auto_svm, ranges = list(cost = c(0.1,
## 1, 10), degree = c(2, 3, 4)), kernel = "polynomial")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: polynomial
## cost: 10
## degree: 3
## coef.0: 0
##
## Number of Support Vectors: 99
##
## ( 49 50 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1
poly_pred <- predict(best_poly, Auto_svm)
table(
Predicted = poly_pred,
Actual = Auto_svm$mpg01
)
## Actual
## Predicted 0 1
## 0 187 10
## 1 9 186
poly_accuracy <- mean(poly_pred == Auto_svm$mpg01)
poly_accuracy
## [1] 0.9515306
results <- data.frame(
Model = c(
"Linear",
"Radial",
"Polynomial"
),
Accuracy = c(
linear_accuracy,
radial_accuracy,
poly_accuracy
)
)
results
## Model Accuracy
## 1 Linear 0.9158163
## 2 Radial 0.9642857
## 3 Polynomial 0.9515306
results[order(results$Accuracy, decreasing = TRUE), ]
## Model Accuracy
## 2 Radial 0.9642857
## 3 Polynomial 0.9515306
## 1 Linear 0.9158163
plot(
best_linear,
Auto_svm,
horsepower ~ weight
)
plot(
best_radial,
Auto_svm,
horsepower ~ weight
)
plot(
best_poly,
Auto_svm,
horsepower ~ weight
)
summary(tune_linear)
##
## Parameter tuning of 'svm':
##
## - sampling method: 10-fold cross validation
##
## - best parameters:
## cost
## 1
##
## - best performance: 0.08435897
##
## - Detailed performance results:
## cost error dispersion
## 1 1e-02 0.08923077 0.04698309
## 2 1e-01 0.09185897 0.04393409
## 3 1e+00 0.08435897 0.03662670
## 4 5e+00 0.08948718 0.03898410
## 5 1e+01 0.08948718 0.03898410
## 6 1e+02 0.08692308 0.03887151