1 FRESA.CAD Sonar Benchmark

1.1 Sonar, Mines vs. Rocks Data Set


data(Sonar)
Sonar$Class <- 1*(Sonar$Class == "M")

mFRESA <- FRESA.Model(formula = Class ~ 1,data = Sonar,repeats = 20)

1.2 Benchmark


cp <- CVBenchmark(theData = Sonar, theOutcome = "Class", reps = 100, topIncluded = 30 )


elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]),FeaLect=mean(cp$times$FeaLecttime[ elapcol ]))
featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Sonar Results

#The Times
pander::pander(cputimes)
  • Fresa: 2.394
  • LASSO: 0.4246
  • RF: 0.1119
  • SVM: 0.0209
  • FeaLect: 2.477
pander::pander(featsize)
  • Fresa: 16.69
  • LASSO: 16.08
  • Univ: 25.43

plotBEREvolution(cp,30,main="Balanced Error Rate", location="topright")


bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname="Balanced Error",thesets=thesets,themethod=theMethod,main="Balanced Error",offsets = c(0.5,0.05),args.legend = list(x = "topright"))


pander::pander(bp$barMatrix,caption = "Balanced Error Rate",round = 3)
Balanced Error Rate
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.238 0.22
LASSO 0.241 0.206
RF 0.203 0.196
SVM 0.182 0.191 
pander::pander(bp$ciTable,caption = "Balanced Error Rate with 95%CI",round = 3)
Balanced Error Rate with 95%CI
  Balanced Error lower upper
Default Classifier 0.238 0.16 0.332
Default Classifier 0.241 0.163 0.335
Default Classifier 0.203 0.132 0.291
Default Classifier 0.182 0.114 0.268
Filtered SVM Classifier 0.22 0.145 0.311
Filtered SVM Classifier 0.206 0.134 0.296
Filtered SVM Classifier 0.196 0.125 0.284
Filtered SVM Classifier 0.191 0.122 0.279 


bp <- barPlotCiError(as.matrix(cp$accciTable),metricname="Accuracy",thesets=thesets,themethod=theMethod,main="Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "Accuracy",round = 3)
Accuracy
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.764 0.784
LASSO 0.76 0.798
RF 0.803 0.808
SVM 0.822 0.812 
pander::pander(bp$ciTable,caption = "Accuracy with 95%CI",round = 3)
Accuracy with 95%CI
  Accuracy lower upper
Default Classifier 0.764 0.701 0.82
Default Classifier 0.76 0.696 0.816
Default Classifier 0.803 0.742 0.855
Default Classifier 0.822 0.763 0.872
Filtered SVM Classifier 0.784 0.721 0.838
Filtered SVM Classifier 0.798 0.737 0.85
Filtered SVM Classifier 0.808 0.747 0.859
Filtered SVM Classifier 0.812 0.753 0.863 


bp <- barPlotCiError(as.matrix(cp$aucTable),metricname="ROC AUC",thesets=thesets,themethod=theMethod,main="ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "ROC AUC",round = 3)
ROC AUC
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.818 0.855
LASSO 0.82 0.863
RF 0.794 0.892
SVM 0.914 0.887 
pander::pander(bp$ciTable,caption = "ROC AUC with 95%CI",round = 3)
ROC AUC with 95%CI ## Analysis of selected features
  ROC AUC lower upper
Default Classifier 0.818 0.751 0.871
Default Classifier 0.82 0.751 0.876
Default Classifier 0.794 0.715 0.869
Default Classifier 0.914 0.859 0.957
Filtered SVM Classifier 0.855 0.773 0.912
Filtered SVM Classifier 0.863 0.786 0.93
Filtered SVM Classifier 0.892 0.818 0.938
Filtered SVM Classifier 0.887 0.824 0.938 
gain <- length(mFRESA$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*mFRESA$BSWiMS.models$bagging$formulaNetwork,trace="none",mar=c(10,10),main="B:SWiMS Formula Network")

pander::pander(summary(mFRESA$BSWiMS.model,caption="Sonar",round = 3))

  • coefficients:

    Table continues below
      Estimate lower OR upper u.Accuracy r.Accuracy
    V11 1.73 4.806 5.641 6.622 0.7452 0.7197
    V36 -0.7629 0.3606 0.4663 0.603 0.6587 0.7243
    V12 1.121 2.556 3.069 3.686 0.7404 0.7142
    V45 1.848 5.3 6.348 7.602 0.6394 0.7572
    V49 4.672 43.66 107 262.1 0.6827 0.7337
    V37 -0.9108 0.2975 0.4022 0.5438 0.6346 0.7142
    V4 3.847 18.14 46.85 121 0.6058 0.7435
    V10 1.272 2.77 3.567 4.594 0.6971 0.7233
    V22 0.2324 1.22 1.262 1.305 0.5865 0.7051
    V47 1.817 4.418 6.151 8.564 0.6202 0.7144
    V9 0.957 1.99 2.604 3.407 0.6923 0.6998
    V21 0.5068 1.422 1.66 1.937 0.6298 0.7075
    V48 2.979 6.817 19.66 56.7 0.6875 0.7166
    V35 -0.5233 0.4655 0.5926 0.7543 0.5865 0.7132
    V44 0.9383 1.86 2.556 3.512 0.5913 0.7272
    V13 1.198 1.651 3.314 6.655 0.6683 0.701
    V23 0.04529 1.017 1.046 1.076 0.5577 0.7163
    V52 5.211 4.999 183.2 6716 0.6394 0.7232
    V46 0.5315 1.351 1.701 2.143 0.6106 0.6822
    V14 -0.5764 0.3878 0.5619 0.8143 0.5962 0.732
    V51 3.305 2.89 27.24 256.6 0.6683 0.704
    V20 0.1096 1.049 1.116 1.187 0.6394 0.6659
    V34 -0.07043 0.8942 0.932 0.9714 0.5865 0.6659
    V1 0.7944 1.354 2.213 3.617 0.601 0.7007
    V43 0.07518 1.035 1.078 1.123 0.5865 0.6474
    Table continues below
      full.Accuracy u.AUC r.AUC full.AUC IDI NRI
    V11 0.7827 0.7418 0.7173 0.7827 0.1213 0.7105
    V36 0.7827 0.6516 0.722 0.7827 0.09448 0.5792
    V12 0.7582 0.736 0.7129 0.7559 0.09728 0.8067
    V45 0.7788 0.6446 0.7558 0.7785 0.1154 0.7494
    V49 0.7582 0.6832 0.732 0.7559 0.1052 0.8231
    V37 0.7661 0.6258 0.7147 0.7648 0.09023 0.6
    V4 0.7702 0.6053 0.7412 0.7689 0.04784 0.4472
    V10 0.7673 0.6961 0.7216 0.766 0.05653 0.7073
    V22 0.7504 0.5782 0.7035 0.7484 0.0479 0.4601
    V47 0.7469 0.6201 0.7112 0.7447 0.0951 0.6052
    V9 0.7418 0.6929 0.697 0.7393 0.06433 0.5653
    V21 0.7419 0.6239 0.7052 0.7391 0.05855 0.6239
    V48 0.7522 0.6877 0.7124 0.7496 0.08497 0.6853
    V35 0.7406 0.5775 0.714 0.7378 0.06696 0.3532
    V44 0.7564 0.5924 0.7245 0.7555 0.07336 0.62
    V13 0.7391 0.6658 0.6985 0.7366 0.06626 0.6468
    V23 0.7548 0.544 0.716 0.7534 0.045 0.5125
    V52 0.7397 0.6394 0.7203 0.7375 0.04058 0.4548
    V46 0.7142 0.6124 0.678 0.7121 0.09229 0.6174
    V14 0.7572 0.5885 0.7286 0.7543 0.04007 0.4114
    V51 0.7225 0.6664 0.701 0.7207 0.04151 0.5077
    V20 0.726 0.6362 0.6644 0.7231 0.05709 0.6015
    V34 0.7163 0.5769 0.6671 0.7154 0.05185 0.3488
    V1 0.726 0.6027 0.6973 0.725 0.04452 0.3551
    V43 0.7179 0.5834 0.646 0.7154 0.05631 0.5037
      z.IDI z.NRI
    V11 5.498 5.514
    V36 4.72 4.359
    V12 4.647 6.356
    V45 5.058 6.032
    V49 4.763 6.671
    V37 4.578 4.547
    V4 3.588 3.404
    V10 3.757 5.507
    V22 3.202 3.413
    V47 4.723 4.748
    V9 3.899 4.363
    V21 3.529 4.732
    V48 4.411 5.32
    V35 3.941 2.605
    V44 3.957 4.783
    V13 3.801 4.944
    V23 3.125 3.817
    V52 2.836 3.406
    V46 4.518 4.78
    V14 3.045 3.032
    V51 2.887 3.815
    V20 3.484 4.565
    V34 3.331 2.566
    V1 3.169 2.694
    V43 3.576 3.787
  • Accuracy: 0.8125
  • tAUC: 0.8107

  • bootstrap: