1 FRESA.CAD Wisconsin Breast Cancer Benchmark

1.1 Wisconsin Breast Cancer Data Set

library(readr)
BreastCancer <- read_csv("./WSBC/wdbc.csv",col_names = FALSE)
BreastCancer <- as.data.frame(BreastCancer)

BreastCancer$Class <- 1*(BreastCancer$X2 == "M")
rownames(BreastCancer) <- BreastCancer$X1
BreastCancer$X1 <- NULL
BreastCancer$X2 <- NULL

ModelFresa <- FRESA.Model(formula = Class ~ 1,data = BreastCancer,repeats = 20)

1.2 Benchmark


cp <- CVBenchmark(theData = BreastCancer, theOutcome = "Class", reps = 100, fraction = 0.2, topIncluded = 31)


elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]),FeaLect=mean(cp$times$FeaLecttime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Results

#The Times
pander::pander(cputimes)
  • Fresa: 2.247
  • LASSO: 0.166
  • RF: 0.0434
  • SVM: 0.0114
  • FeaLect: 1.826
pander::pander(featsize)
  • Fresa: 17.82
  • LASSO: 8.64
  • Univ: 25.32

plotBEREvolution(cp,30,main="Balanced Error Rate", location="topright")


bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname="Balanced Error",thesets=thesets,themethod=theMethod,main="Balanced Error",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))


pander::pander(bp$barMatrix,caption = "Balanced Error Rate",round = 3)
Balanced Error Rate
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.038 0.05
LASSO 0.036 0.053
RF 0.053 0.062
SVM 0.062 0.062 
pander::pander(bp$ciTable,caption = "Balanced Error Rate with 95%CI",round = 3)
Balanced Error Rate with 95%CI
  Balanced Error lower upper
Default Classifier 0.038 0.019 0.068
Default Classifier 0.036 0.018 0.066
Default Classifier 0.053 0.03 0.087
Default Classifier 0.062 0.038 0.096
Filtered SVM Classifier 0.05 0.029 0.082
Filtered SVM Classifier 0.053 0.031 0.085
Filtered SVM Classifier 0.062 0.038 0.096
Filtered SVM Classifier 0.062 0.038 0.096 



bp <- barPlotCiError(as.matrix(cp$accciTable),metricname="Accuracy",thesets=thesets,themethod=theMethod,main="Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "Accuracy",round = 3)
Accuracy
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.967 0.96
LASSO 0.97 0.958
RF 0.953 0.951
SVM 0.951 0.951 
pander::pander(bp$ciTable,caption = "Accuracy with 95%CI",round = 3)
Accuracy with 95%CI
  Accuracy lower upper
Default Classifier 0.967 0.948 0.98
Default Classifier 0.97 0.953 0.983
Default Classifier 0.953 0.932 0.968
Default Classifier 0.951 0.93 0.967
Filtered SVM Classifier 0.96 0.94 0.974
Filtered SVM Classifier 0.958 0.938 0.973
Filtered SVM Classifier 0.951 0.93 0.967
Filtered SVM Classifier 0.951 0.93 0.967 


bp <- barPlotCiError(as.matrix(cp$aucTable),metricname="ROC AUC",thesets=thesets,themethod=theMethod,main="ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "ROC AUC",round = 3)
ROC AUC
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.989 0.992
LASSO 0.991 0.989
RF 0.937 0.991
SVM 0.991 0.991 
pander::pander(bp$ciTable,caption = "ROC AUC with 95%CI",round = 3)
ROC AUC with 95%CI
  ROC AUC lower upper
Default Classifier 0.989 0.98 0.994
Default Classifier 0.991 0.982 0.996
Default Classifier 0.937 0.902 0.96
Default Classifier 0.991 0.985 0.995
Filtered SVM Classifier 0.992 0.983 0.996
Filtered SVM Classifier 0.989 0.98 0.995
Filtered SVM Classifier 0.991 0.985 0.995
Filtered SVM Classifier 0.991 0.985 0.995

1.3 Analysis of selected features


gain <- length(ModelFresa$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*ModelFresa$BSWiMS.models$bagging$formulaNetwork,trace="none",mar=c(10,10),main="B:SWiMS Formula Network")

pander::pander(summary(ModelFresa$BSWiMS.model,caption="WDBC",round = 3))

  • coefficients:

    Table continues below
      Estimate lower OR upper u.Accuracy
    X25 0.04581 1.033 1.047 1.061 0.9125
    X24 0.05255 1.038 1.054 1.07 0.7182
    X27 15.28 19393 4311222 958406012 0.6956
    X4 0.09064 1.067 1.095 1.124 0.729
    X30 12.01 1963 164358 13758261 0.8974
    X26 0.005665 1.003 1.006 1.008 0.8983
    X3 -0.3646 0.5484 0.6945 0.8794 0.8577
    X23 0.3592 1.192 1.432 1.721 0.9014
    X29 2.128 0.4473 8.402 157.8 0.8408
    X10 6.158 65.85 472.6 3391 0.9124
    X31 3.793 1.211 44.4 1628 0.6635
    X17 82.83 7.277e+20 9.346e+35 1.2e+51 0.5373
    X6 -0.0009275 0.9954 0.9991 1.003 0.8603
    X5 -0.001276 0.9781 0.9987 1.02 0.8708
    X16 0.1544 1.081 1.167 1.26 0.855
    X19 -4.231 0.003152 0.01453 0.06702 0.6913
    X13 -9.181 6.639e-07 0.000103 0.01599 0.8047
    X9 11.7 957.8 120025 15041453 0.8798
    X18 -32.9 3.951e-23 5.143e-15 6.696e-07 0.6645
    X32 23.86 10764 2.296e+10 4.898e+16 0.6425
    X28 2.376 3.879 10.76 29.87 0.7844
    X22 -232.5 6.212e-165 1.098e-101 1.942e-38 0.5872
    X15 0.09419 1.06 1.099 1.139 0.808
    Table continues below
      r.Accuracy full.Accuracy u.AUC r.AUC full.AUC IDI
    X25 0.7631 0.9685 0.9125 0.7631 0.9685 0.549
    X24 0.9449 0.9685 0.7182 0.9449 0.9685 0.04423
    X27 0.9415 0.9685 0.6956 0.9415 0.9685 0.07089
    X4 0.9518 0.9735 0.729 0.9518 0.9735 0.04017
    X30 0.9338 0.9735 0.8974 0.9338 0.9735 0.09582
    X26 0.9385 0.9725 0.8983 0.9385 0.9725 0.06049
    X3 0.9607 0.9725 0.8577 0.9607 0.9725 0.0181
    X23 0.9186 0.9605 0.9014 0.9186 0.9605 0.09845
    X29 0.9483 0.9611 0.8408 0.9483 0.9611 0.03302
    X10 0.9382 0.9537 0.9124 0.9382 0.9537 0.0414
    X31 0.9537 0.9609 0.6635 0.9537 0.9609 0.01814
    X17 0.9462 0.9614 0.5373 0.9462 0.9614 0.02165
    X6 0.9465 0.9614 0.8603 0.9465 0.9614 0.02583
    X5 0.9524 0.9553 0.8708 0.9524 0.9553 0.02646
    X16 0.9301 0.9576 0.855 0.9301 0.9576 0.08099
    X19 0.9407 0.9549 0.6913 0.9407 0.9549 0.04113
    X13 0.9488 0.9583 0.8047 0.9488 0.9583 0.02754
    X9 0.9413 0.9578 0.8798 0.9413 0.9578 0.03792
    X18 0.9388 0.9573 0.6645 0.9388 0.9573 0.05232
    X32 0.9289 0.9586 0.6425 0.9289 0.9586 0.06985
    X28 0.9256 0.9548 0.7844 0.9256 0.9548 0.08011
    X22 0.9344 0.9586 0.5872 0.9344 0.9586 0.07334
    X15 0.9396 0.9546 0.808 0.9396 0.9546 0.03234
      NRI z.IDI z.NRI
    X25 1.792 30.08 54.77
    X24 1.357 6.069 25.41
    X27 1.311 7.858 24.83
    X4 1.486 5.496 30.95
    X30 1.188 9.246 22.92
    X26 1.228 7.043 22.58
    X3 0.7383 4 12.07
    X23 1.265 8.941 23.43
    X29 0.8531 4.934 14.79
    X10 1.182 5.29 21.32
    X31 0.5977 3.75 9.731
    X17 0.8372 4.467 12.93
    X6 0.6567 4.196 10.61
    X5 0.8142 4.229 12.59
    X16 1.427 7.675 28.56
    X19 1.04 5.426 18.64
    X13 1 4.314 16.81
    X9 1.071 5.408 19.09
    X18 1.455 6.725 31.21
    X32 1.359 7.273 26.14
    X28 1.528 7.778 33.4
    X22 1.419 8.244 28.91
    X15 1.07 5.163 17.48
  • Accuracy: 0.9719
  • tAUC: 0.9699

  • bootstrap: