if (!require(kernlab)) {
  install.packages("kernlab")
  library(kernlab)
}
## Loading required package: kernlab
if (!require(ggplot2)) {
  install.packages("ggplot2")
  library(ggplot2)
}
## Loading required package: ggplot2
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:kernlab':
## 
##     alpha
if (!require(pROC)) {
  install.packages("pROC")
  library(pROC)
}
## Loading required package: pROC
## Warning: package 'pROC' was built under R version 4.4.3
## Type 'citation("pROC")' for a citation.
## 
## Attaching package: 'pROC'
## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var
df1 <- read.csv("D:/Semester 5/Statistical Machine Learning/c96ddc62dc4bdb665eb499d56e6bed25-4829995d34fc945c2bdfbdcd04bcb7f0a0a1d6bc/data1.csv")
df2 <- read.csv("D:/Semester 5/Statistical Machine Learning/7fb9645c4361d0e957535776afb7a0de-0b901c2121b126313d84b7f54c2d518dc8db398f/data2.csv")
head(df1)
##           x1         x2 y
## 1 -0.4248450 -0.2927878 1
## 2  0.5766103 -0.2671171 1
## 3 -0.1820462 -0.4257997 0
## 4  0.7660348 -0.8400542 1
## 5  0.8809346 -0.2690915 1
## 6 -0.9088870 -0.6439724 1
head(df2)
##           x1        x2 y
## 1  0.5437821 1.0931537 0
## 2 -0.8992556 0.5951635 0
## 3  0.2132927 1.0616862 0
## 4 -0.9151249 0.4020860 0
## 5 -0.6628595 0.1487150 0
## 6  0.9720628 0.1185531 0
ggplot(df1, aes(x = x1, y = x2)) +
  geom_point(size = 2, alpha = 0.7, color = "blue") +
  labs(title = "Plot Dataset 1")

ggplot(df2, aes(x = x1, y = x2)) +
  geom_point(size = 2, alpha = 0.7, color = "blue") +
  labs(title = "Plot Dataset 2")

df1_train <- df1[1:400, ]
df1_test <- df1[401:500, ]
df1_train$y <- as.factor(df1_train$y)
df1_test$y  <- as.factor(df1_test$y)
df2_train <- df2[1:400, ]
df2_test <- df2[401:500, ]
df2_train$y <- as.factor(df2_train$y)
df2_test$y  <- as.factor(df2_test$y)

SVM with Kernel Methods

Vanilladot kernel

Kernel: \[ K(x_i, x) = x_{i1} x_1 + x_{i2} x_2 \] Inner product untuk 2 covariate:
\[ x_i^\top x = x_{i1} x_1 + x_{i2} x_2 \] Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i (x_{i1} x_1 + x_{i2} x_2) + b \] DATASET 1

# Linear SVM dengan kernel vanilladot
vanilladotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "vanilladot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters  
## maximum number of iterations reached 9.749045e-05 -9.748027e-05
vanilladotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Linear (vanilla) kernel function. 
## 
## Number of Support Vectors : 149 
## 
## Objective Function Value : -700 
## Training error : 0.175 
## Cross validation error : 0.175046 
## Probability model included.
predict(vanilladotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(vanilladotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 0.79

DATA SET 2

# Linear SVM dengan kernel vanilladot
vanilladotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "vanilladot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
vanilladotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Linear (vanilla) kernel function. 
## 
## Number of Support Vectors : 12 
## 
## Objective Function Value : -39.3607 
## Training error : 0.005 
## Cross validation error : 0.005013 
## Probability model included.
predict(vanilladotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(vanilladotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 0.98

Interpretasi Kernel Vanilla dot : Model Kernel Vanilla dot berhasil memprediksi datatest df1 dengan akurasi sebesar 0.79 dan berhasil memprediksi datatest df2 dengan akurasi hampir sempurna yakni 0.99.

Radial Basis Kernel

Kernel: \[ K(x_i, x) = \exp(-\gamma \| x_i - x \|^2) \] Jarak kuadrat L2 untuk 2 covariate:
\[ \| x_i - x \|^2 = (x_{i1}-x_1)^2 + (x_{i2}-x_2)^2 \]

Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i \exp(-\gamma ((x_{i1}-x_1)^2 + (x_{i2}-x_2)^2)) + b \] DATASET 1

# Model SVM dengan Gaussian kernel radial basis
rbfdotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "rbfdot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
rbfdotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Gaussian Radial Basis kernel function. 
##  Hyperparameter : sigma =  1.2119472103582 
## 
## Number of Support Vectors : 42 
## 
## Objective Function Value : -138.96 
## Training error : 0.0075 
## Cross validation error : 0.0225 
## Probability model included.
 predict(rbfdotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(rbfdotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 0.99

DATASET 2

# Model SVM dengan Gaussian kernel radial basis
rbfdotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "rbfdot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
rbfdotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Gaussian Radial Basis kernel function. 
##  Hyperparameter : sigma =  2.63195415728032 
## 
## Number of Support Vectors : 35 
## 
## Objective Function Value : -8.6281 
## Training error : 0 
## Cross validation error : 0 
## Probability model included.
 predict(rbfdotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(rbfdotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 1

Interpretasi Kernel Radial Basis : Secara keseluruhan model SVM dengan kernel radial basis memberikan hasil pemisahan yang sangat dominan pada kedua dataset. Pada datatest df1, model berhasil memprediksi semua data dengan tepat sehingga mendapatkan akurasi sebesar 1. Begitu pun dengan datatest df2, model berhasil memprediksi semua data dengan tepat sehingga mendapatkan akurasi sebesar 1.

Tanh Kernel

Kernel:
\[ K(x_i, x) = \tanh(\kappa x_i^\top x + c) \] Inner product 2 covariate:
\[ x_i^\top x = x_{i1} x_1 + x_{i2} x_2 \] Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i \tanh(\kappa (x_{i1} x_1 + x_{i2} x_2) + c) + b \] DATASET 1

# Model hyperbolic tangent SVM dengan tanh dot
tanhdotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "tanhdot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
tanhdotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Hyperbolic Tangent kernel function. 
##  Hyperparameters : scale =  1  offset =  1 
## 
## Number of Support Vectors : 141 
## 
## Objective Function Value : -29508.75 
## Training error : 0.3475 
## Cross validation error : 0.329948 
## Probability model included.
 predict(tanhdotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(tanhdotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 0.79

DATASET 2

# Model hyperbolic tangent SVM dengan tanh dot
tanhdotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "tanhdot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
tanhdotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Hyperbolic Tangent kernel function. 
##  Hyperparameters : scale =  1  offset =  1 
## 
## Number of Support Vectors : 82 
## 
## Objective Function Value : -4080.346 
## Training error : 0.2 
## Cross validation error : 0.15234 
## Probability model included.
 predict(tanhdotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(tanhdotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 0.77

Interpretasi Kernel Tanh dot : Secara keseluruhan model SVM dengan kernel Tanh dot memberikan hasil prediksi kelas yang kurang cukup baik dengan hanya mendapatkan akurasi pada datatest df1 sebesar 0.8 dan mendapatkan akurasi pada datatest df2 sebesar 0.73.

Bessel Kernel

Definisi kernel:
\[ K(x_i, x) = \frac{J_\nu(\sigma \|x_i - x\|)}{(\sigma \|x_i - x\|)^\nu} \]

Jarak L2 untuk 2 covariate:
\[ \| x_i - x \| = \sqrt{(x_{i1}-x_1)^2 + (x_{i2}-x_2)^2} \]

Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i \frac{J_\nu(\sigma \sqrt{(x_{i1}-x_1)^2 + (x_{i2}-x_2)^2})}{(\sigma \sqrt{(x_{i1}-x_1)^2 + (x_{i2}-x_2)^2})^\nu} + b \] DATASET 1

# Model bessel SVM dengan bessel dot
besseldotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "besseldot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
besseldotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Bessel kernel function. 
##  Hyperparameter : sigma =  1 order =  1 degree =  1 
## 
## Number of Support Vectors : 96 
## 
## Objective Function Value : -349.3807 
## Training error : 0.015 
## Cross validation error : 0.012531 
## Probability model included.
predict(besseldotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(besseldotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 1

DATASET 2

# Model bessel SVM dengan bessel dot
besseldotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "besseldot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
besseldotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Bessel kernel function. 
##  Hyperparameter : sigma =  1 order =  1 degree =  1 
## 
## Number of Support Vectors : 20 
## 
## Objective Function Value : -68.2073 
## Training error : 0.005 
## Cross validation error : 0.005013 
## Probability model included.
predict(besseldotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(besseldotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 0.99

Interpretasi kernel Bessel dot : Menggunakan kernel bessel dot, datatest df 1 berhasil diprediksi dengan akurasi sebesar 1, sedangkan datatest df 2 berhasil diprediksi dengan akurasi 0.99.

Laplace Kernel

Kernel:
\[ K(x_i, x) = \exp(-\gamma \| x_i - x \|_1) \] Jarak L1 untuk 2 covariate:
\[ \| x_i - x \|_1 = |x_{i1}-x_1| + |x_{i2}-x_2| \] Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i \exp(-\gamma (|x_{i1}-x_1| + |x_{i2}-x_2|)) + b \] DATASET 1

# Model SVM dengan laplace dot
laplacedotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "laplacedot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
laplacedotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Laplace kernel function. 
##  Hyperparameter : sigma =  1.22994908575151 
## 
## Number of Support Vectors : 51 
## 
## Objective Function Value : -91.0379 
## Training error : 0 
## Cross validation error : 0.027475 
## Probability model included.
 predict(laplacedotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(laplacedotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 0.98

DATASET 2

# Model SVM dengan laplace dot
laplacedotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "laplacedot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
laplacedotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Laplace kernel function. 
##  Hyperparameter : sigma =  2.01436814625131 
## 
## Number of Support Vectors : 139 
## 
## Objective Function Value : -9.0285 
## Training error : 0 
## Cross validation error : 0 
## Probability model included.
 predict(laplacedotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(laplacedotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 1

Interpretasi kernel laplace dot : Dengan kernel laplace dot, datatest df1 berhasil diprediksi dengan akurasi sebesar 0.98 dan datatest df2 berhasil diprediksi dengan akurasi sempurna yakni 1.

Anova Radial Basis Kernel

Kernel:
\[ K(x_i, x) = \left( \sum_{j=1}^{d} \exp(-\sigma (x_{ij} - x_j)^2) \right)^p \]

Untuk 2 covariate (d=2):
\[ K(x_i, x) = (\exp(-\sigma (x_{i1}-x_1)^2) + \exp(-\sigma (x_{i2}-x_2)^2))^p \]

Substitusi ke SVM:
\[ f(x_1, x_2) = \sum_{i=1}^{n} \alpha_i (\exp(-\sigma (x_{i1}-x_1)^2) + \exp(-\sigma (x_{i2}-x_2)^2))^p + b \] DATASET 1

# Model SVM dengan anova dot
anovadotdf1 <- ksvm(y ~ ., data = df1_train, kernel = "anovadot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
anovadotdf1
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Anova RBF kernel function. 
##  Hyperparameter : sigma =  1 degree =  1 
## 
## Number of Support Vectors : 41 
## 
## Objective Function Value : -146.7261 
## Training error : 0.01 
## Cross validation error : 0.0225 
## Probability model included.
 predict(anovadotdf1, df1_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(anovadotdf1, df1_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df1_test$y)
accuracy
## [1] 0.99

DATASET 2

# Model SVM dengan anova dot
anovadotdf2 <- ksvm(y ~ ., data = df2_train, kernel = "anovadot", kpar = "automatic", C = 5, cross = 3, prob.model = TRUE)
##  Setting default kernel parameters
anovadotdf2
## Support Vector Machine object of class "ksvm" 
## 
## SV type: C-svc  (classification) 
##  parameter : cost C = 5 
## 
## Anova RBF kernel function. 
##  Hyperparameter : sigma =  1 degree =  1 
## 
## Number of Support Vectors : 10 
## 
## Objective Function Value : -9.2791 
## Training error : 0 
## Cross validation error : 0.002488 
## Probability model included.
 predict(anovadotdf2, df2_test[1, ])
## [1] 1
## Levels: 0 1
pred_probs <- predict(anovadotdf2, df2_test, type = "probabilities")
pred_class <- apply(pred_probs, 1, function(x) colnames(pred_probs)[which.max(x)])
accuracy <- mean(pred_class == df2_test$y)
accuracy
## [1] 1

Interpretasi kernel anova dot : Dengan kernel Anova dot, datatest df1 berhasil diprediksi ke kelas masing-masing dengan akurasi 0.99, sedangkan datatest df2 berhasil diprediksi dengan akurasi sempurna yakni 1.

KESIMPULAN :

Menggunakan nilai accuracy pada data test dan cross validation error sebagai model evaluasi, didapatkan model yang paling cocok untuk dataset 1 dan dataset 2 sebagai berikut :

  1. DATASET 1 : Model SVM yang paling cocok untuk mengklasifikasikan kelas pada dataset 1 adalah model Bessel kernel dengan nilai Cross validation error : 0.0225 dan dengan accuracy pada datatest sebesar 1.

  2. DATASET 2 : Model SVM yang paling cocok untuk mengklasifikasikan kelas pada dataset 2 adalah model Laplace dot kernel dengan Cross validation error : 0 dan accuracy pada datatest sebesar 1.

Akan tetapi, jika ingin mengevaluasi tanpa melihat cross-validation error dan juga prediksi dilakukan bebarengan pada dataset 1 dan dataset 2, model terbaik yang didapatkan adalah Radial Basis karena pada kedua datatest mendapatkan tingkat akurasi sempurna yakni 1.