1 FRESA.CAD Benchmark

1.1 Lymphoma Cancer Data Set


DLBCL_Lymohoma_A <- read.delim("./DLBCL/DLBCL_Lymohoma_A.txt")
#DLBCL_Lymohoma_A <- read.delim("./DLBCL_Lymohoma_A.txt")

Lymphoma <- DLBCL_Lymohoma_A[,-c(1,3,4)]
rownames(Lymphoma) <- DLBCL_Lymohoma_A[,1]

LYMPFRESA <- FRESA.Model(formula = Class ~ 1,data = Lymphoma,repeats = 20)

1.2 Benchmark


reps <- 50;
topIncluded <- 100;

cp <- CVBenchmark(theData = Lymphoma, theOutcome = "Class", reps = reps, fraction = 0.80, topIncluded = topIncluded)


elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]),FeaLect=mean(cp$times$FeaLecttime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Results

#The Times
pander::pander(cputimes)
  • Fresa: 16.61
  • LASSO: 0.7506
  • RF: 31.87
  • SVM: 4.954
  • FeaLect: 111.7
pander::pander(featsize)
  • Fresa: 21.82
  • LASSO: 26.48
  • Univ: 1180

plotBEREvolution(cp,40,main="Balanced Error Rate", location="topright")



bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname="Balanced Error",thesets=thesets,themethod=theMethod,main="Balanced Error",offsets = c(0.5,0.05),args.legend = list(x = "topright"))


pander::pander(bp$barMatrix,caption = "Balanced Error Rate",round = 3)
Balanced Error Rate
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.096 0.096
LASSO 0.088 0.105
RF 0.219 0.096
SVM 0.483 0.114 
pander::pander(bp$ciTable,caption = "Balanced Error Rate with 95%CI",round = 3)
Balanced Error Rate with 95%CI
  Balanced Error lower upper
Default Classifier 0.096 0.021 0.261
Default Classifier 0.088 0.017 0.244
Default Classifier 0.219 0.101 0.379
Default Classifier 0.483 0.44 0.586
Filtered SVM Classifier 0.096 0.021 0.261
Filtered SVM Classifier 0.105 0.03 0.259
Filtered SVM Classifier 0.096 0.019 0.257
Filtered SVM Classifier 0.114 0.03 0.274 


bp <- barPlotCiError(as.matrix(cp$accciTable),metricname="Accuracy",thesets=thesets,themethod=theMethod,main="Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "Accuracy",round = 3)
Accuracy
  Default Classifier Filtered SVM Classifier
B:SWiMS 0.909 0.909
LASSO 0.948 0.948
RF 0.883 0.935
SVM 0.273 0.935 
pander::pander(bp$ciTable,caption = "Accuracy with 95%CI",round = 3)
Accuracy with 95%CI
  Accuracy lower upper
Default Classifier 0.909 0.822 0.963
Default Classifier 0.948 0.872 0.986
Default Classifier 0.883 0.79 0.945
Default Classifier 0.273 0.177 0.386
Filtered SVM Classifier 0.909 0.822 0.963
Filtered SVM Classifier 0.948 0.872 0.986
Filtered SVM Classifier 0.935 0.855 0.979
Filtered SVM Classifier 0.935 0.855 0.979 


bp <- barPlotCiError(as.matrix(cp$aucTable),metricname="ROC AUC",thesets=thesets,themethod=theMethod,main="ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "ROC AUC",round = 3)
ROC AUC
  Default Classifier Filtered SVM Classifier
B:SWiMS 1 0.983
LASSO 1 1
RF 0.75 0.982
SVM 0.967 1 
pander::pander(bp$ciTable,caption = "ROC AUC with 95%CI",round = 3)
ROC AUC with 95%CI
  ROC AUC lower upper
Default Classifier 1 0.887 1
Default Classifier 1 0.94 1
Default Classifier 0.75 0.533 0.925
Default Classifier 0.967 0.629 1
Filtered SVM Classifier 0.983 0.856 1
Filtered SVM Classifier 1 0.912 1
Filtered SVM Classifier 0.982 0.925 1
Filtered SVM Classifier 1 0.925 1

2 Feature Analysis

pander::pander(summary(LYMPFRESA$BSWiMS.model,caption="Sonar",round = 3))
gain <- length(LYMPFRESA$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*LYMPFRESA$BSWiMS.models$bagging$formulaNetwork[1:30,1:30],trace="none",mar=c(10,10),main="B:SWiMS Formula Network")