Lab 10

Question 4

# Load necessary libraries
library(e1071)    # for svm
## Warning: package 'e1071' was built under R version 4.4.3
library(ggplot2)  # for plotting
## Warning: package 'ggplot2' was built under R version 4.4.3
library(caret)    # for confusionMatrix
## Warning: package 'caret' was built under R version 4.4.3
## Loading required package: lattice
set.seed(123) # for reproducibility

# Generate data
n <- 100
X1 <- rnorm(n)
X2 <- rnorm(n)
Y <- ifelse(X1^2 + X2^2 > 1.5, 1, 0) # Non-linear boundary: circle
data <- data.frame(X1, X2, Y = as.factor(Y))

# Split into training and test sets
train_idx <- sample(1:n, n/2)
train_data <- data[train_idx, ]
test_data <- data[-train_idx, ]

# SVM with Linear Kernel
svm_linear <- svm(Y ~ ., data = train_data, kernel = "linear", cost = 1)

# SVM with Polynomial Kernel (degree = 3)
svm_poly <- svm(Y ~ ., data = train_data, kernel = "polynomial", degree = 3, cost = 1)

# SVM with Radial Kernel
svm_radial <- svm(Y ~ ., data = train_data, kernel = "radial", cost = 1)

# Predict and compute errors
predict_linear_train <- predict(svm_linear, train_data)
predict_linear_test <- predict(svm_linear, test_data)

predict_poly_train <- predict(svm_poly, train_data)
predict_poly_test <- predict(svm_poly, test_data)

predict_radial_train <- predict(svm_radial, train_data)
predict_radial_test <- predict(svm_radial, test_data)

# Training errors
train_error_linear <- mean(predict_linear_train != train_data$Y)
train_error_poly <- mean(predict_poly_train != train_data$Y)
train_error_radial <- mean(predict_radial_train != train_data$Y)

# Test errors
test_error_linear <- mean(predict_linear_test != test_data$Y)
test_error_poly <- mean(predict_poly_test != test_data$Y)
test_error_radial <- mean(predict_radial_test != test_data$Y)

# Print error rates
cat("Training Errors:\n")
## Training Errors:
cat("Linear Kernel:", round(train_error_linear, 3), "\n")
## Linear Kernel: 0.32
cat("Polynomial Kernel:", round(train_error_poly, 3), "\n")
## Polynomial Kernel: 0.3
cat("Radial Kernel:", round(train_error_radial, 3), "\n\n")
## Radial Kernel: 0.02
cat("Test Errors:\n")
## Test Errors:
cat("Linear Kernel:", round(test_error_linear, 3), "\n")
## Linear Kernel: 0.3
cat("Polynomial Kernel:", round(test_error_poly, 3), "\n")
## Polynomial Kernel: 0.26
cat("Radial Kernel:", round(test_error_radial, 3), "\n")
## Radial Kernel: 0.08
# Plotting decision boundaries
plot_svm <- function(model, data, title) {
  grid <- expand.grid(X1 = seq(min(data$X1) - 0.5, max(data$X1) + 0.5, length = 100),
                      X2 = seq(min(data$X2) - 0.5, max(data$X2) + 0.5, length = 100))
  grid$Y <- predict(model, grid)
  
  ggplot() +
    geom_point(data = data, aes(x = X1, y = X2, color = Y), size = 2) +
    geom_contour(data = grid, aes(x = X1, y = X2, z = as.numeric(Y)), breaks = 1.5, color = "black") +
    ggtitle(title) +
    theme_minimal()
}

# Plot decision boundaries
plot_svm(svm_linear, train_data, "SVM with Linear Kernel (Training Set)")

plot_svm(svm_poly, train_data, "SVM with Polynomial Kernel (Training Set)")

plot_svm(svm_radial, train_data, "SVM with Radial Kernel (Training Set)")

Data Generation

Generated a simulated two-class dataset with 100 observations and two features.
The class boundary was designed to be non-linear, roughly circular in nature.

Models Trained

Firstly trained three different SVM models: - SVM with Linear Kernel - SVM with Polynomial Kernel (degree = 3) - SVM with Radial Kernel (RBF)

Training Errors

  • Linear Kernel: 0.32
  • Polynomial Kernel: 0.30
  • Radial Kernel: 0.02

Test Errors

  • Linear Kernel: 0.30
  • Polynomial Kernel: 0.26
  • Radial Kernel: 0.08

Observations

  • The linear kernel performed poorly, as it could not capture the non-linear boundary between the two classes.
  • The polynomial kernel (degree 3) improved performance by capturing some non-linear structure, but still had some errors.
  • The radial kernel achieved the lowest training and test error rates, demonstrating its ability to model complex non-linear relationships.

Plots

  • The decision boundary for the linear SVM is roughly a straight line, failing to adapt to the circular separation.
  • The polynomial SVM produced a more flexible boundary but not perfectly adapted to the circular shape.
  • The radial SVM captured the circular pattern of the data, resulting in much better separation between classes.

The radial kernel SVM outperformed both the linear and polynomial kernel SVMs, achieving the best fit for both the training and testing data.
This highlights the importance of using non-linear kernels when the class boundaries are non-linear.

Question 7

# Safe install & load block
if (!require(ISLR)) {
  install.packages("ISLR")
  library(ISLR)
} else {
  library(ISLR)
}
## Loading required package: ISLR
## Warning: package 'ISLR' was built under R version 4.4.3
if (!require(e1071)) {
  install.packages("e1071")
  library(e1071)
} else {
  library(e1071)
}

(a)

data("Auto")
AutoMod <- Auto
AutoMod$mpg_binary <- as.factor(ifelse(AutoMod$mpg > median(AutoMod$mpg), 1, 0))

(b)

set.seed(100)
linear_costs <- c(0.01, 0.1, 1, 10, 100)
linear_results <- lapply(linear_costs, function(c_val) {
  svm(mpg_binary ~ displacement + weight, data = AutoMod, kernel = "linear", cost = c_val, scale = TRUE)
})

# Use tune() to find best linear model
linear_tune <- tune(svm, mpg_binary ~ displacement + weight, data = AutoMod, kernel = "linear",
                    ranges = list(cost = 10^seq(-2, 2)))
summary(linear_tune)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost
##   0.1
## 
## - best performance: 0.09692308 
## 
## - Detailed performance results:
##    cost      error dispersion
## 1 1e-02 0.10461538 0.04261286
## 2 1e-01 0.09692308 0.04476973
## 3 1e+00 0.09948718 0.04741645
## 4 1e+01 0.09948718 0.04741645
## 5 1e+02 0.09948718 0.04741645
best_linear_model <- linear_tune$best.model
table(Predicted = predict(best_linear_model, AutoMod), Actual = AutoMod$mpg_binary)
##          Actual
## Predicted   0   1
##         0 167   8
##         1  29 188
cv_error_linear <- linear_tune$performances[linear_tune$performances$cost == best_linear_model$cost, ]$error

(C)

Radial:

set.seed(100)
poly_tune <- tune(svm, mpg_binary ~ displacement + weight, data = AutoMod, kernel = "polynomial",
                  ranges = list(cost = 10^seq(-1, 2), degree = 2:4))
summary(poly_tune)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost degree
##   0.1      3
## 
## - best performance: 0.148141 
## 
## - Detailed performance results:
##     cost degree     error dispersion
## 1    0.1      2 0.3496795 0.06147407
## 2    1.0      2 0.3444872 0.05072363
## 3   10.0      2 0.3369231 0.06227318
## 4  100.0      2 0.3139744 0.06136743
## 5    0.1      3 0.1481410 0.05240094
## 6    1.0      3 0.1734615 0.03566290
## 7   10.0      3 0.1734615 0.04450522
## 8  100.0      3 0.1862821 0.05536720
## 9    0.1      4 0.3623718 0.04735888
## 10   1.0      4 0.3573077 0.04937349
## 11  10.0      4 0.3062179 0.05963495
## 12 100.0      4 0.3062821 0.06235276
best_poly_model <- poly_tune$best.model
cv_error_poly <- poly_tune$performances[
  poly_tune$performances$cost == best_poly_model$cost &
  poly_tune$performances$degree == best_poly_model$degree, ]$error

Polynomial:

set.seed(100)
radial_tune <- tune(svm, mpg_binary ~ displacement + weight, data = AutoMod, kernel = "radial",
                    ranges = list(cost = 10^seq(-1, 2), gamma = 10^seq(-2, 1)))
summary(radial_tune)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost gamma
##     1     1
## 
## - best performance: 0.09435897 
## 
## - Detailed performance results:
##     cost gamma      error dispersion
## 1    0.1  0.01 0.13775641 0.05549179
## 2    1.0  0.01 0.10461538 0.04899252
## 3   10.0  0.01 0.09692308 0.04476973
## 4  100.0  0.01 0.09948718 0.04741645
## 5    0.1  0.10 0.09948718 0.04422798
## 6    1.0  0.10 0.09948718 0.04893284
## 7   10.0  0.10 0.09948718 0.04741645
## 8  100.0  0.10 0.09692308 0.04637274
## 9    0.1  1.00 0.10205128 0.04828663
## 10   1.0  1.00 0.09435897 0.04673366
## 11  10.0  1.00 0.10717949 0.03783657
## 12 100.0  1.00 0.09942308 0.04047716
## 13   0.1 10.00 0.09955128 0.02812740
## 14   1.0 10.00 0.10198718 0.04488039
## 15  10.0 10.00 0.09679487 0.04092424
## 16 100.0 10.00 0.09942308 0.03688662
best_radial_model <- radial_tune$best.model
cv_error_radial <- radial_tune$performances[
  radial_tune$performances$cost == best_radial_model$cost &
  radial_tune$performances$gamma == best_radial_model$gamma, ]$error

(d)

plot(best_radial_model, AutoMod, displacement ~ weight)

# Set up custom colors
custom_colors <- c("#a6cee3", "#fb9a99")  # light blue and pink for regions
custom_symbols <- c(1, 4)                # open circle and X for classes

# Plot with custom palette
plot(best_radial_model, AutoMod, displacement ~ weight,
     palette = custom_colors,
     symbolPalette = custom_symbols)

Question_8

library(ISLR2)
## Warning: package 'ISLR2' was built under R version 4.4.3
## 
## Attaching package: 'ISLR2'
## The following object is masked _by_ '.GlobalEnv':
## 
##     Auto
## The following objects are masked from 'package:ISLR':
## 
##     Auto, Credit
library(e1071)

(a).

set.seed(1)
train_index <- sample(1:nrow(OJ), 800)
train_data <- OJ[train_index, ]
test_data <- OJ[-train_index, ]

(b).

# Fit the model
svc_model <- svm(Purchase ~ ., data = train_data, kernel = "linear", cost = 0.01, scale = TRUE)

# View summary
summary(svc_model)
## 
## Call:
## svm(formula = Purchase ~ ., data = train_data, kernel = "linear", 
##     cost = 0.01, scale = TRUE)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  linear 
##        cost:  0.01 
## 
## Number of Support Vectors:  435
## 
##  ( 219 216 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  CH MM

(c).

train_preds <- predict(svc_model, train_data)
test_preds <- predict(svc_model, test_data)

train_error <- mean(train_preds != train_data$Purchase)
test_error <- mean(test_preds != test_data$Purchase)
train_error
## [1] 0.175
test_error
## [1] 0.1777778

(d).

set.seed(1)
tune_result <- tune(svm, Purchase ~ ., data = train_data, kernel = "linear",
                    ranges = list(cost = 10^seq(-2, 1)))
summary(tune_result)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost
##   0.1
## 
## - best performance: 0.1725 
## 
## - Detailed performance results:
##    cost   error dispersion
## 1  0.01 0.17625 0.02853482
## 2  0.10 0.17250 0.03162278
## 3  1.00 0.17500 0.02946278
## 4 10.00 0.17375 0.03197764
best_linear <- tune_result$best.model

(e).

best_train_preds <- predict(best_linear, train_data)
best_test_preds <- predict(best_linear, test_data)

best_train_error <- mean(best_train_preds != train_data$Purchase)
best_test_error <- mean(best_test_preds != test_data$Purchase)
best_train_error
## [1] 0.165
best_test_error
## [1] 0.162963

(f).

set.seed(1)
tune_radial <- tune(svm, Purchase ~ ., data = train_data, kernel = "radial",
                    ranges = list(cost = 10^seq(-2, 1)))
summary(tune_radial)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost
##     1
## 
## - best performance: 0.17125 
## 
## - Detailed performance results:
##    cost   error dispersion
## 1  0.01 0.39375 0.04007372
## 2  0.10 0.18625 0.02853482
## 3  1.00 0.17125 0.02128673
## 4 10.00 0.18625 0.02853482
best_radial <- tune_radial$best.model

radial_train_preds <- predict(best_radial, train_data)
radial_test_preds <- predict(best_radial, test_data)

radial_train_error <- mean(radial_train_preds != train_data$Purchase)
radial_test_error <- mean(radial_test_preds != test_data$Purchase)
radial_train_error
## [1] 0.15125
radial_test_error
## [1] 0.1851852

(g).

set.seed(1)
tune_poly <- tune(svm, Purchase ~ ., data = train_data, kernel = "polynomial",
                  ranges = list(cost = 10^seq(-2, 1)), degree = 2)
summary(tune_poly)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost
##    10
## 
## - best performance: 0.18125 
## 
## - Detailed performance results:
##    cost   error dispersion
## 1  0.01 0.39125 0.04210189
## 2  0.10 0.32125 0.05001736
## 3  1.00 0.20250 0.04116363
## 4 10.00 0.18125 0.02779513
best_poly <- tune_poly$best.model

poly_train_preds <- predict(best_poly, train_data)
poly_test_preds <- predict(best_poly, test_data)

poly_train_error <- mean(poly_train_preds != train_data$Purchase)
poly_test_error <- mean(poly_test_preds != test_data$Purchase)
poly_train_error
## [1] 0.15
poly_test_error
## [1] 0.1888889

(h).

cat("Linear Test Error:", best_test_error, "\n")
## Linear Test Error: 0.162963
cat("Radial Test Error:", radial_test_error, "\n")
## Radial Test Error: 0.1851852
cat("Polynomial Test Error:", poly_test_error, "\n")
## Polynomial Test Error: 0.1888889