libraries

library(pander)
library(FRESA.CAD)
library("mlbench")
library(dplyr)
library(beepr)

Sonar dataset

data(Sonar, package = "mlbench")
Sonar$Class <- 1*(Sonar$Class == "M")
cvlist <- list()
modelsnames <- c("BSWiMS.model","filteredFit","NAIVE_BAYES","LASSO_1SE","LASSO_MIN","GLMNET_RIDGE_MIN","GLMNET_ELASTICNET_MIN")

BSWIMScv <- randomCV(Sonar,
      "Class",
      BSWiMS.model,
      trainFraction = 0.7,
      repetitions = 20)
## .[++++++-+-].[+++--+-+-].[++++++++-].[+++++-].[++++-+-].[+++++--].[++++++-].[++++++++++-]..[+++++++++--].[+++-]10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.370997 
## .[+++++++-].[++++-].[++++++--].[++++-+-].[+++++++-].[++++++-+++-].[++++--].[++++++--].[+++++-+-].[++-++--]20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.3727587
cvlist[[modelsnames[1]]] <- BSWIMScv




filteredFitcv <- randomCV(Sonar,
    "Class",
    filteredFit,
    trainSampleSets  = BSWIMScv$trainSamplesSets,
    fitmethod=glm,
    filtermethod=univariate_BinEnsemble,
    filtermethod.control = list(pvalue=0.05),
    family = "binomial")
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.2579247 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.2518095
cvlist[[modelsnames[2]]] <- filteredFitcv

  
NAIVE_BAYEScv <- randomCV(Sonar,
      "Class",
      NAIVE_BAYES,
      trainSampleSets  = BSWIMScv$trainSamplesSets)
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.2931819 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.2996297
cvlist[[modelsnames[3]]] <- NAIVE_BAYEScv



LASSO_1SEcv <- randomCV(Sonar,
      "Class",
      LASSO_1SE,
      trainSampleSets  = BSWIMScv$trainSamplesSets)
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.4358892 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.4267017
cvlist[[modelsnames[4]]] <- LASSO_1SEcv


LASSO_MINcv <- randomCV(Sonar,
      "Class",
      LASSO_MIN,
      trainSampleSets  = BSWIMScv$trainSamplesSets)
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.4534764 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.4562231
cvlist[[modelsnames[5]]] <- LASSO_MINcv


GLMNET_RIDGE_MINcv <- randomCV(Sonar,
      "Class",
      GLMNET_RIDGE_MIN,
      trainSampleSets  = BSWIMScv$trainSamplesSets)
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.4513263 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.4547085
cvlist[[modelsnames[6]]] <- GLMNET_RIDGE_MINcv


GLMNET_ELASTICNET_MINcv <- randomCV(Sonar,
      "Class",
      GLMNET_ELASTICNET_MIN,
      trainSampleSets  = BSWIMScv$trainSamplesSets)
## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.4518452 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.4548971
cvlist[[modelsnames[7]]] <- GLMNET_ELASTICNET_MINcv
beep()
cp <- BinaryBenchmark(referenceCV = cvlist)
## BSWiMS.model

## filteredFit

## NAIVE_BAYES

## LASSO_1SE

## LASSO_MIN

## GLMNET_RIDGE_MIN

## GLMNET_ELASTICNET_MIN

## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.3268283 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.324835 
## Random Forest

## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.3104744 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.2934764 
## RPART

## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.346791 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.3394423 
## LASSO

## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.3042093 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.3051135 
## SVM

## ..........10  Tested: 198 Min Tests: 1 Max Tests: 8 Mean Tests: 3.232323 . MAD: 0.2931218 
## ..........20  Tested: 206 Min Tests: 2 Max Tests: 13 Mean Tests: 6.213592 . MAD: 0.2922332 
## KNN

## Ensemble

sonar_algo_auctable <- get.auctable(cp,is.lc = FALSE)
par(mfrow = c(1,1),
    cex = 0.7,
    xpd = T, #
    pty = 'm', #maximal plotting region
    mar = c(3,3,3,10)) # Making space for the legend
cp <- trim.cp(cp)
prBenchmark <- plot(cp)

pander::pander(prBenchmark$metrics,caption = "Classifier Performance",round = 3)
Classifier Performance (continued below)
  BSWiMS.model filteredFit LASSO_1SE NAIVE_BAYES LASSO_MIN
BER 0.231 0.239 0.26 0.268 0.485
ACC 0.767 0.762 0.743 0.718 0.553
AUC 0.837 0.84 0.81 0.829 0.846
SEN 0.757 0.784 0.775 0.55 1
SPE 0.779 0.737 0.705 0.916 0.032
CIDX 0.845 0.827 0.824 0.719 0.84
  GLMNET_ELASTICNET_MIN GLMNET_RIDGE_MIN
BER 0.485 0.49
ACC 0.553 0.549
AUC 0.863 0.864
SEN 1 1
SPE 0.032 0.021
CIDX 0.865 0.865 
pander::pander(sonar_algo_auctable,caption = "ROC AUC table",round = 3)
ROC AUC table
  AUC ci
BSWiMS.model 0.837 [0.781,0.892]
filteredFit 0.84 [0.787,0.893]
NAIVE_BAYES 0.829 [0.774,0.884]
LASSO_1SE 0.81 [0.751,0.87]
LASSO_MIN 0.846 [0.793,0.899]
GLMNET_RIDGE_MIN 0.864 [0.814,0.913]
GLMNET_ELASTICNET_MIN 0.863 [0.813,0.913]
RF 0.934 [0.904,0.965]
RPART 0.811 [0.75,0.871]
LASSO 0.844 [0.79,0.898]
SVM 0.875 [0.827,0.922]
KNN 0.865 [0.816,0.914]
ENS 0.895 [0.853,0.937]