1 FRESA.CAD Sonar Benchmark

1.1 Sonar, Mines vs. Rocks Data Set

op <- par(no.readonly = TRUE)

data(Sonar)
Sonar$Class <- 1*(Sonar$Class == "M")
Sonar.mat <- as.data.frame(model.matrix(Class~.*.,Sonar))
#Sonar.mat <- as.data.frame(model.matrix(Class~.,Sonar))
Sonar.mat$`(Intercept)` <- NULL
Sonar.mat$Class <- as.numeric(Sonar$Class)

fnames <- colnames(Sonar.mat)
fnames <- str_replace_all(fnames," ","_")
fnames <- str_replace_all(fnames,"/","_")
fnames <- str_replace_all(fnames,":",".")
colnames(Sonar.mat) <- fnames

ExperimentName <- "Sonar"
bswimsReps <- 20;
theData <- Sonar.mat;
theOutcome <- "Class";
reps <- 120;
fraction <- 0.75;

BSIWIMSFileName <- paste(ExperimentName,"FRESAMethod_v2.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod_v2.RDATA",sep = "_")

1.2 Benchmarking



QDAcv <- randomCV(theData,theOutcome,
                  MASS::qda,trainFraction = fraction,
                  repetitions = reps,
                  featureSelectionFunction = univariate_Wilcoxon,
                  featureSelection.control = list(limit = 0.10,thr = 0.95))


ADAcv <- randomCV(theData,theOutcome,
                  adaboost,
                  trainSampleSets=QDAcv$trainSamplesSets,
                  featureSelectionFunction = univariate_Wilcoxon,
                  featureSelection.control = list(limit = 0.10,thr = 0.95),asFactor = TRUE,nIter=10
)

BESScv <- randomCV(theData,theOutcome,BESS,trainSampleSets=QDAcv$trainSamplesSets)

BSWiMScv <- randomCV(theData,theOutcome,BSWiMS.model,trainSampleSets=QDAcv$trainSamplesSets)

GMVEBSWiMSCV <- randomCV(theData,theOutcome,GMVEBSWiMS,trainSampleSets=QDAcv$trainSamplesSets)

ClustClassCV <- randomCV(theData,theOutcome,ClustClass,trainSampleSets=QDAcv$trainSamplesSets,clustermethod=Mclust,clustermethod.control=list(G = 2))


par(mfrow = c(2,2),cex = 0.6);
cp <- BinaryBenchmark(referenceCV = list(QDA=QDAcv,BeSS=BESScv,ADABOOST=ADAcv,BSWiMS=BSWiMScv,GMVEBSWiMSCV=GMVEBSWiMSCV,Mclust_LASSO=ClustClassCV))

par(op );

save(cp, file="Boost.RDATA")
load(file="Boost.RDATA")

1.3 Reporting the results of the Benchmark procedure

Once done, we can compare CV test results using the plot() function. The plot function also generates summary tables of the CV results.

#ploting the results
op <-par();
prBenchmark <- plot(cp)


pander::pander(prBenchmark$metrics,caption = "Clasifier Performance",round = 3)
Clasifier Performance (continued below)
  RPART RF GMVEBSWiMSCV SVM ENS LASSO BeSS
BER 0.154 0.17 0.182 0.192 0.212 0.225 0.226
ACC 0.846 0.832 0.817 0.808 0.788 0.774 0.774
AUC 0.848 0.829 0.906 0.806 0.885 0.858 0.774
SEN 0.856 0.865 0.82 0.82 0.811 0.775 0.775
SPE 0.835 0.794 0.814 0.794 0.763 0.773 0.773
CIDX 0.777 0.82 0.827 0.79 0.886 0.844 0.737
  BSWiMS ADABOOST Mclust_LASSO KNN QDA
BER 0.226 0.238 0.256 0.267 0.344
ACC 0.774 0.764 0.75 0.74 0.654
AUC 0.865 0.88 0.834 0.846 0.795
SEN 0.775 0.802 0.838 0.856 0.631
SPE 0.773 0.722 0.649 0.608 0.68
CIDX 0.85 0.837 0.809 0.818 0.783 
#pander::pander(prBenchmark$metrics_filter,caption = "Average Filter Performance",round = 3)

par(op);
gplots::heatmap.2(t(as.matrix(cp$featureSelectionFrequency[1:50,])),trace = "none",mar = c(10,10),main = "Features",cexRow = 0.5,cexCol = 0.5)