Benchmarks for Wisconsin Breast Cancer Data Set

1 FRESA.CAD Wisconsin Breast Cancer Benchmark

1.1 Wisconsin Breast Cancer Data Set

library(readr)
BreastCancer <- read_csv("./WSBC/wdbc.csv",col_names = FALSE)
BreastCancer <- as.data.frame(BreastCancer)
BreastCancer$Class <- 1*(BreastCancer$X2 == "M")
rownames(BreastCancer) <- BreastCancer$X1
BreastCancer$X1 <- NULL
BreastCancer$X2 <- NULL
BreastCancer.mat <- as.data.frame(model.matrix(Class~.*.,BreastCancer))
BreastCancer.mat$`(Intercept)` <- NULL
BreastCancer.mat$Class <- as.numeric(BreastCancer$Class)

fnames <- colnames(BreastCancer.mat)
fnames <- str_replace_all(fnames," ","_")
fnames <- str_replace_all(fnames,"/","_")
fnames <- str_replace_all(fnames,":",".")
colnames(BreastCancer.mat) <- fnames

ExperimentName <- "WSBC"
bswimsReps <- 20;
theData <- BreastCancer.mat;
theOutcome <- "Class";
reps <- 22;
fraction <- 0.3;

BSIWIMSFileName <- paste(ExperimentName,"BSWiMSMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

sampleTrain <- sample(nrow(BreastCancer.mat),nrow(BreastCancer.mat)*0.30)
sampleData <- BreastCancer.mat[sampleTrain,]

1.2 Benchmarking

BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = sampleData,NumberofRepeats = bswimsReps)

save(BSWiMSMODEL,file = BSIWIMSFileName)


cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

save(cp,file = CVFileName)

1.3 Results

1.3.1 Classifier Results

hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45) 

[1] 2

#The Times
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
21.08	4.384	0.08818	0.4045	0.01091	0.04545	26.02

learningTime <- -1*cp$cpuElapsedTimes

bp <- barPlotCiError(as.matrix(cp$errorciTable),metricname = "Balanced Error",thesets = cp$thesets,themethod = cp$theMethod,main = "Balanced Error",offsets = c(0.5,0.05),scoreDirection = "<",args.legend = list(x = "topright"),col = terrain.colors(length(cp$theMethod)))

pander::pander(bp$ciTable$mean,caption = "Balanced Error Rate",round = 3)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
0.03052	0.03988	0.04695	0.03147	0.02727	0.03766	0.02491

testBalancedError <- -bp$ciTable$mean;
testBalancedErrormin <- min(-bp$ciTable$top95)
testBalancedErrormax <- max(-bp$ciTable$low95)

bp <- barPlotCiError(as.matrix(cp$accciTable),metricname = "Accuracy",thesets = cp$thesets,themethod = cp$theMethod,main = "Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"),col = terrain.colors(length(cp$theMethod)))

pander::pander(bp$ciTable$mean,caption = "Accuracy",round = 3)

ENS	SVM	BSWiMS	LASSO	KNN	RF	RPART
0.9772	0.9754	0.9701	0.9701	0.9684	0.9596	0.9543

testAccuracy <- bp$ciTable$mean;
testAccuracymin <- min(bp$ciTable$low95)
testAccuracymax <- max(bp$ciTable$top95)

bp <- barPlotCiError(as.matrix(cp$aucTable),metricname = "AUC",thesets = cp$thesets,themethod = cp$theMethod,main = "ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"),col = terrain.colors(length(cp$theMethod)))

pander::pander(bp$ciTable$mean,caption = "ROC AUC",round = 3)

ENS	BSWiMS	RF	LASSO	SVM	KNN	RPART
0.9947	0.9936	0.9938	0.9932	0.9943	0.9908	0.9838

testAUC <- bp$ciTable$mean;
testAUCmin <- min(bp$ciTable$low95)
testAUCmax <- max(bp$ciTable$top95)

bp <- barPlotCiError(as.matrix(cp$senTable),metricname = "Sensitivity",thesets = cp$thesets,themethod = cp$theMethod,main = "Sensititvity",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"),col = terrain.colors(length(cp$theMethod)))

pander::pander(bp$ciTable$mean,caption = "Sensitivity",round = 3)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
0.967	0.9623	0.9481	0.9623	0.9623	0.9387	0.967

testSEN <- bp$ciTable$mean;
testSENmin <- min(bp$ciTable$low95)
testSENmax <- max(bp$ciTable$top95)

bp <- barPlotCiError(as.matrix(cp$speTable),metricname = "Specificity",thesets = cp$thesets,themethod = cp$theMethod,main = "Specificity",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"),col = terrain.colors(length(cp$theMethod)))

pander::pander(bp$ciTable$mean,caption = "Specificity",round = 3)

SVM	KNN	ENS	BSWiMS	LASSO	RF	RPART
0.9832	0.986	0.9832	0.972	0.9748	0.958	0.958

testSPE <- bp$ciTable$mean;
testSPEmin <- min(bp$ciTable$low95)
testSPEmax <- max(bp$ciTable$top95)

1.3.2 Filter Results

pander::pander(cp$featsize)

Table continues below

BSWiMS	LASSO	RPART	RF.ref	IDI	NRI	t-test	Wilcoxon	Spearman
37.64	11.27	14.36	37.64	68.73	68.91	71.14	73.18	72.23

Kendall	mRMR
73.18	14.55

par(mfrow = c(1,2))
barplot(cp$jaccard[order(-cp$jaccard)],las = 2,cex.axis = 1,cex.names = 0.7,main = "Jaccard Index",ylab = "Jaccard")
selectJaccard <- cp$jaccard[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")]
unsize <- cp$featsize + 1
barplot(unsize[order(unsize)],las = 2,cex.axis = 1,cex.names = 0.7,log = "y",main = "Number of Features",ylab = "# of Features+1")

selectFilFeatsize <- -cp$featsize[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")]
par(mfrow = c(1,1))


bp <- barPlotCiError(as.matrix(cp$errorciTable_filter),metricname = "Balanced Error",thesets = cp$theFiltersets,themethod = cp$theClassMethod,main = "Balanced Error",scoreDirection = "<",args.legend = list(x = "topleft"),col = terrain.colors(length(cp$theClassMethod)))

pander::pander(bp$ciTable$mean,caption = "Balanced Error Rate",round = 3)

Balanced Error Rate (continued below)

	RPART	BSWiMS	mRMR	LASSO	RF.ref	t-test	NRI
SVM	0.029	0.034	0.027	0.027	0.034	0.043	0.05
RPART	0.046	0.047	0.046	0.044	0.046	0.056	0.058
KNN	0.032	0.038	0.032	0.034	0.03	0.071	0.07
Naive Bayes	0.037	0.028	0.033	0.024	0.043	0.099	0.108
NC RSS	0.055	0.051	0.052	0.044	0.056	0.099	0.105
NC Spearman	0.086	0.1	0.117	0.162	0.103	0.152	0.18

	IDI	Wilcoxon	Kendall	Spearman
SVM	0.053	0.06	0.06	0.062
RPART	0.062	0.044	0.044	0.034
KNN	0.073	0.077	0.077	0.077
Naive Bayes	0.111	0.108	0.108	0.11
NC RSS	0.112	0.122	0.122	0.119
NC Spearman	0.179	0.174	0.175	0.169

testFilBalancedError <- -apply(bp$ciTable$mean,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")];
testFilBalancedErrormax <- -min(apply(bp$ciTable$low95,2,min)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])
testFilBalancedErrormin <- -max(apply(bp$ciTable$top95,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])

bp <- barPlotCiError(as.matrix(cp$accciTable_filter),metricname = "Accuracy",thesets = cp$theFiltersets,themethod = cp$theClassMethod,main = "Accuracy",offsets = c(0.5,0.05),args.legend = list(x = "bottomleft"),col = terrain.colors(length(cp$theClassMethod)))

pander::pander(bp$ciTable$mean,caption = "Accuracy",round = 3)

Accuracy (continued below)

	RPART	mRMR	BSWiMS	LASSO	RF.ref	t-test	Spearman
SVM	0.975	0.975	0.97	0.977	0.972	0.96	0.944
RPART	0.956	0.958	0.956	0.956	0.956	0.949	0.968
KNN	0.972	0.974	0.968	0.972	0.975	0.933	0.93
Naive Bayes	0.965	0.968	0.974	0.979	0.958	0.9	0.888
NC RSS	0.938	0.944	0.942	0.951	0.937	0.895	0.87
NC Spearman	0.914	0.866	0.889	0.805	0.886	0.828	0.801

	Wilcoxon	Kendall	NRI	IDI
SVM	0.946	0.946	0.953	0.951
RPART	0.956	0.956	0.946	0.94
KNN	0.93	0.93	0.933	0.93
Naive Bayes	0.889	0.889	0.893	0.889
NC RSS	0.866	0.866	0.891	0.882
NC Spearman	0.798	0.796	0.791	0.793

testFilAccuracy <- apply(bp$ciTable$mean,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")];
testFilAccuracymin <- min(apply(bp$ciTable$low95,2,min)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])
testFilAccuracymax <- max(apply(bp$ciTable$top95,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])

bp <- barPlotCiError(as.matrix(cp$aucTable_filter),metricname = "AUC",thesets = cp$theFiltersets,themethod = cp$theClassMethod,main = "ROC AUC",offsets = c(0.5,0.05),args.legend = list(x = "bottomleft"),col = terrain.colors(length(cp$theClassMethod)))

pander::pander(bp$ciTable$mean,caption = "ROC AUC",round = 3)

ROC AUC (continued below)

	BSWiMS	RPART	LASSO	RF.ref	mRMR	t-test	Wilcoxon
SVM	0.995	0.992	0.995	0.992	0.994	0.991	0.989
RPART	0.984	0.984	0.987	0.982	0.982	0.983	0.991
KNN	0.991	0.992	0.993	0.99	0.992	0.982	0.98
NC RSS	0.994	0.991	0.994	0.99	0.992	0.965	0.962
Naive Bayes	0.99	0.993	0.993	0.986	0.991	0.946	0.932
NC Spearman	0.972	0.973	0.943	0.97	0.951	0.94	0.929

	Kendall	Spearman	NRI	IDI
SVM	0.989	0.989	0.989	0.989
RPART	0.991	0.987	0.981	0.977
KNN	0.98	0.98	0.982	0.981
NC RSS	0.962	0.962	0.954	0.952
Naive Bayes	0.932	0.932	0.938	0.939
NC Spearman	0.929	0.928	0.92	0.914

testFilAUC <- apply(bp$ciTable$mean,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")];
testFilAUCmin <- min(apply(bp$ciTable$low95,2,min)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])
testFilAUCmax <- max(apply(bp$ciTable$top95,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])

bp <- barPlotCiError(as.matrix(cp$senciTable_filter),metricname = "Sensitivity",thesets = cp$theFiltersets,themethod = cp$theClassMethod,main = "Sensitivity",offsets = c(0.5,0.05),args.legend = list(x = "bottomleft"),col = terrain.colors(length(cp$theClassMethod)))

pander::pander(bp$ciTable$mean,caption = "Sensitivity",round = 3)

Sensitivity (continued below)

	LASSO	mRMR	BSWiMS	RF.ref	RPART	t-test	Spearman
NC RSS	0.976	0.967	0.976	0.972	0.972	0.925	0.925
RPART	0.958	0.939	0.943	0.948	0.948	0.925	0.958
SVM	0.958	0.962	0.948	0.943	0.953	0.948	0.915
NC Spearman	0.967	0.948	0.943	0.939	0.915	0.929	0.948
Naive Bayes	0.962	0.962	0.967	0.953	0.958	0.906	0.901
KNN	0.943	0.948	0.939	0.948	0.953	0.91	0.896

	Wilcoxon	Kendall	NRI	IDI
NC RSS	0.925	0.925	0.91	0.91
RPART	0.958	0.958	0.929	0.929
SVM	0.92	0.92	0.939	0.934
NC Spearman	0.939	0.939	0.934	0.934
Naive Bayes	0.901	0.901	0.887	0.887
KNN	0.896	0.896	0.915	0.915

testFilSEN <- apply(bp$ciTable$mean,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")];
testFilSENmin <- min(apply(bp$ciTable$low95,2,min)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])
testFilSENmax <- max(apply(bp$ciTable$top95,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])

bp <- barPlotCiError(as.matrix(cp$speciTable_filter),metricname = "Specificity",thesets = cp$theFiltersets,themethod = cp$theClassMethod,main = "Specificity",offsets = c(0.5,0.05),args.legend = list(x = "bottomleft"),col = terrain.colors(length(cp$theClassMethod)))

pander::pander(bp$ciTable$mean,caption = "Specificity",round = 3)

Specificity (continued below)

	LASSO	BSWiMS	mRMR	RPART	RF.ref	t-test	Spearman
SVM	0.989	0.983	0.983	0.989	0.989	0.966	0.961
KNN	0.989	0.986	0.989	0.983	0.992	0.947	0.95
RPART	0.955	0.964	0.969	0.961	0.961	0.964	0.975
Naive Bayes	0.989	0.978	0.972	0.969	0.961	0.896	0.88
NC RSS	0.936	0.922	0.93	0.919	0.916	0.877	0.838
NC Spearman	0.709	0.857	0.818	0.913	0.854	0.768	0.714

	NRI	Wilcoxon	Kendall	IDI
SVM	0.961	0.961	0.961	0.961
KNN	0.944	0.95	0.95	0.938
RPART	0.955	0.955	0.955	0.947
Naive Bayes	0.896	0.882	0.882	0.891
NC RSS	0.88	0.832	0.832	0.866
NC Spearman	0.706	0.714	0.711	0.709

testFilSPE <- apply(bp$ciTable$mean,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")];
testFilSPEmin <- min(apply(bp$ciTable$low95,2,min)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])
testFilSPEmax <- max(apply(bp$ciTable$top95,2,max)[c("BSWiMS","LASSO","RF.ref","IDI","Kendall","mRMR")])

1.3.3 Radar Plots

op <- par(no.readonly = TRUE)

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

classRanks <- c(testBalancedErrormax,testSENmax,testSPEmax,testAccuracymax,testAUCmax,max(learningTime))
classRanks <- rbind(classRanks,c(testBalancedErrormin,0,0,0,0,min(learningTime)))

classRanks <- as.data.frame(rbind(classRanks,cbind(testBalancedError[names(learningTime)],testSEN[names(learningTime)],testSPE[names(learningTime)],testAccuracy[names(learningTime)],testAUC[names(learningTime)],learningTime)))
colnames(classRanks) <- c("B.Error","Sen","Spe","Accuracy","ROC AUC","CPU Time")

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filterRanks <- c(max(selectJaccard),max(selectFilFeatsize),testFilBalancedErrormax, testFilSENmax,testFilSPEmax, testFilAccuracymax,testFilAUCmax);

filterRanks <- rbind(filterRanks,c(0,min(selectFilFeatsize),testFilBalancedErrormin,0,0, 0,0));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(selectJaccard,selectFilFeatsize,testFilBalancedError,testFilSEN,testFilSPE ,testFilAccuracy,testFilAUC)));
colnames(filterRanks) <- c("Jaccard","Size","B.Error","Sen","Spe","Accuracy","ROC.ACU")

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0),rgb(1.0,0.0,1.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05), rgb(1.0,0.0,1.0,0.05)  )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)

par(mfrow = c(1,1))

detach("package:fmsb", unload=TRUE)

par(op)

1.3.4 Feature Analysis

rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(selFrequency,trace = "none",mar = c(10,10),main = "Features",cexRow = 0.5)

topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network")

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

• coefficients:

  Table continues below

  	Estimate	lower	OR	upper	u.Accuracy
  X26.X27	0.005039	1.004	1.005	1.006	0.9605
  X16.X29	0.1087	1.071	1.115	1.161	0.9315
  X16.X28	0.004034	1.004	1.004	1.004	0.9441
  X16.X30	0.04667	1.036	1.048	1.06	0.946
  X9.X26	0.002248	1.001	1.002	1.003	0.9377
  X10.X26	0.008635	1.003	1.009	1.014	0.9378
  X25	0.02404	1.01	1.024	1.039	0.9239
  X7.X26	0.2208	0.3598	1.247	4.323	0.9463
  X4.X26	5.68e-05	1	1	1	0.9168
  X26.X32	0.007347	1.004	1.007	1.011	0.9434
  X8.X26	0.0007643	1.001	1.001	1.001	0.9294
  X11.X26	0.0108	1.002	1.011	1.019	0.9444
  X12.X26	0.054	1.012	1.055	1.101	0.9478
  X26.X31	0.004806	1.003	1.005	1.007	0.9494
  X6.X27	0.01409	1.007	1.014	1.022	0.9435
  X26.X30	0.005429	1.002	1.005	1.009	0.9318
  X24.X26	0.0001418	0.9989	1	1.001	0.9201
  X6.X24	4.351e-05	1	1	1	0.8956
  X6.X31	0.00112	1	1.001	1.002	0.9327
  X26.X29	0.001579	0.9992	1.002	1.004	0.9231
  X25.X31	0.01783	1.009	1.018	1.027	0.9349
  X24.X30	0.03226	1.01	1.033	1.056	0.8791
  X15.X30	0.07242	1.044	1.075	1.107	0.9263
  X17.X26	0.02044	1.007	1.021	1.035	0.8643
  X19.X32	-58.25	1.337e-29	5.025e-26	1.889e-22	0.7426
  X7.X25	0.1597	1.085	1.173	1.269	0.9274
  X25.X27	0.2921	0.1928	1.339	9.305	0.9567
  X10.X27	1.559	3.047	4.756	7.422	0.9049
  X27.X31	16.37	11.19	12917185	1.492e+13	0.6969
  X8.X19	-29.82	2.634e-17	1.118e-13	4.749e-10	0.7767
  X9.X19	-6.034	0.0003703	0.002396	0.0155	0.802
  X4.X25	0.0003039	1	1	1	0.8991
  X8.X27	8.297	670	4012	24029	0.8201
  X27	7.478	22.41	1769	139669	0.6484
  X19.X29	-9.221	9.58e-11	9.893e-05	102.2	0.7794
  X11.X24	0.1321	1.047	1.141	1.244	0.7736
  X7.X27	4.29	14.05	72.93	378.6	0.6441
  X16.X24	0.000138	1	1	1	0.9081
  X24.X31	1.735	0.0002233	5.671	144008	0.7595
  X4.X27	0.4848	1.156	1.624	2.281	0.8066
  X23.X29	-0.003609	0.995	0.9964	0.9978	0.9127
  X24.X27	0.2817	0.9528	1.325	1.844	0.762
  X16.X27	0.008205	1.005	1.008	1.012	0.9333
  X12.X24	0.1049	1.032	1.111	1.195	0.6873
  X7.X24	0.4024	1.039	1.495	2.152	0.7829
  X24.X25	0.0003394	1	1	1.001	0.9166
  X17.X24	0.8775	0.942	2.405	6.14	0.6275
  X4.X7	0.4081	1.238	1.504	1.827	0.7787
  X4.X6	2.484e-06	1	1	1	0.881
  X10.X24	0.09498	1.059	1.1	1.142	0.9165
  X13.X24	0.001772	1.001	1.002	1.003	0.8784
  X22.X29	-10.38	1.798e-07	3.096e-05	0.005332	0.7588
  X24	0.01549	1.009	1.016	1.023	0.7226
  X4.X31	0.04665	0.9821	1.048	1.118	0.8054
  X4.X24	6.409e-05	1	1	1	0.7559

  Table continues below

  	r.Accuracy	full.Accuracy	u.AUC	r.AUC	full.AUC
  X26.X27	0.3471	0.9605	0.959	0.5	0.959
  X16.X29	0.7011	0.9528	0.9215	0.6975	0.9483
  X16.X28	0.5549	0.948	0.9327	0.6102	0.9397
  X16.X30	0.4328	0.9468	0.9418	0.5631	0.9431
  X9.X26	0.6199	0.9424	0.933	0.657	0.939
  X10.X26	0.6171	0.9482	0.9408	0.6697	0.9473
  X25	0.7631	0.9586	0.9264	0.7556	0.9575
  X7.X26	0.7692	0.976	0.9443	0.7615	0.9751
  X4.X26	0.7686	0.9679	0.9161	0.7694	0.9672
  X26.X32	0.6759	0.9524	0.9401	0.7275	0.9488
  X8.X26	0.7346	0.9529	0.9221	0.7372	0.9512
  X11.X26	0.7839	0.9716	0.9441	0.7827	0.9706
  X12.X26	0.7959	0.9748	0.9456	0.7901	0.9755
  X26.X31	0.7975	0.9716	0.9453	0.7957	0.9672
  X6.X27	0.7977	0.9679	0.9437	0.7902	0.9676
  X26.X30	0.7934	0.9613	0.9333	0.8017	0.9619
  X24.X26	0.8232	0.9729	0.9184	0.8342	0.9721
  X6.X24	0.8396	0.9601	0.8875	0.8328	0.9585
  X6.X31	0.8133	0.9434	0.9315	0.8036	0.9415
  X26.X29	0.8416	0.9519	0.9194	0.8412	0.9506
  X25.X31	0.8254	0.956	0.9331	0.8233	0.9527
  X24.X30	0.7873	0.9294	0.8728	0.8	0.925
  X15.X30	0.8075	0.9329	0.92	0.8364	0.9312
  X17.X26	0.7902	0.9233	0.8438	0.8175	0.9156
  X19.X32	0.9311	0.961	0.7136	0.9208	0.9578
  X7.X25	0.901	0.9707	0.9251	0.8968	0.9689
  X25.X27	0.9166	0.9847	0.951	0.9159	0.9843
  X10.X27	0.8735	0.9304	0.902	0.8549	0.9235
  X27.X31	0.9084	0.9747	0.6783	0.9047	0.9736
  X8.X19	0.8749	0.9339	0.748	0.8829	0.9278
  X9.X19	0.9346	0.9458	0.757	0.9266	0.943
  X4.X25	0.8992	0.9682	0.8958	0.9051	0.9666
  X8.X27	0.9186	0.9706	0.8172	0.9178	0.9679
  X27	0.9165	0.9654	0.6371	0.9155	0.9647
  X19.X29	0.9319	0.9539	0.7441	0.929	0.9531
  X11.X24	0.9455	0.9754	0.7669	0.9454	0.975
  X7.X27	0.9216	0.9539	0.6421	0.9241	0.9528
  X16.X24	0.9324	0.964	0.8957	0.9297	0.9612
  X24.X31	0.9406	0.9688	0.7405	0.9406	0.967
  X4.X27	0.9474	0.9763	0.8039	0.9442	0.9731
  X23.X29	0.9186	0.948	0.9094	0.9106	0.9463
  X24.X27	0.9464	0.972	0.7594	0.945	0.9709
  X16.X27	0.9275	0.9549	0.9238	0.9228	0.95
  X12.X24	0.9397	0.9623	0.6712	0.9396	0.9618
  X7.X24	0.9451	0.9728	0.7826	0.9434	0.9718
  X24.X25	0.9505	0.9707	0.9176	0.9469	0.9692
  X17.X24	0.9325	0.9552	0.6098	0.932	0.9515
  X4.X7	0.9393	0.964	0.7769	0.938	0.9626
  X4.X6	0.9281	0.9549	0.8736	0.9255	0.954
  X10.X24	0.9362	0.9544	0.9138	0.9341	0.9525
  X13.X24	0.9127	0.9402	0.8632	0.91	0.9363
  X22.X29	0.9431	0.9598	0.7154	0.9373	0.958
  X24	0.9314	0.9601	0.7273	0.9311	0.9605
  X4.X31	0.938	0.9556	0.7918	0.9398	0.9515
  X4.X24	0.9284	0.9598	0.7502	0.9234	0.9593

  	IDI	NRI	z.IDI	z.NRI	Frequency
  X26.X27	0.8791	1.838	40.42	35.28	1
  X16.X29	0.7821	1.789	32.5	30.97	1
  X16.X28	0.769	1.799	29.29	31.29	0.1
  X16.X30	0.7063	1.755	28.38	28.12	1
  X9.X26	0.7386	1.74	27.55	26.83	0.4
  X10.X26	0.603	1.784	20.73	30.31	1
  X25	0.501	1.778	17.23	32.04	1
  X7.X26	0.5657	1.889	17.12	Inf	1
  X4.X26	0.4567	1.722	16.84	32.1	0.9
  X26.X32	0.4587	1.744	16.73	27.44	1
  X8.X26	0.5473	1.718	16.7	25.2	0.15
  X11.X26	0.514	1.813	15.48	34.52	1
  X12.X26	0.4858	1.846	14.71	36.98	1
  X26.X31	0.47	1.797	14.14	31.45	1
  X6.X27	0.446	1.816	13.98	35.68	1
  X26.X30	0.397	1.736	13.42	28.43	1
  X24.X26	0.3668	1.81	13.33	1.498e+306	1
  X6.X24	0.3241	1.663	11.87	28.05	0.85
  X6.X31	0.325	1.38	11.75	18.52	0.45
  X26.X29	0.2789	1.565	11.1	21.72	0.85
  X25.X31	0.2896	1.576	10.81	20.33	0.9
  X24.X30	0.3002	1.527	10.13	18.72	0.3
  X15.X30	0.2215	1.419	9.046	16.06	0.25
  X17.X26	0.1917	1.143	7.417	11.46	0.5
  X19.X32	0.1458	1.794	7.277	31.96	0.4
  X7.X25	0.1913	1.778	7.211	30.72	1
  X25.X27	0.185	1.909	6.923	52.06	1
  X10.X27	0.175	1.213	6.866	12.29	0.1
  X27.X31	0.1649	1.671	6.718	25.15	0.5
  X8.X19	0.1268	1.677	6.663	26.5	0.5
  X9.X19	0.1123	1.774	6.333	30.01	0.15
  X4.X25	0.1437	1.717	6.287	28.79	0.9
  X8.X27	0.1168	1.857	5.984	38.89	0.2
  X27	0.1305	1.63	5.851	22.52	0.6
  X19.X29	0.1071	1.708	5.529	27.01	0.2
  X11.X24	0.1129	1.637	5.237	22.39	0.55
  X7.X27	0.1046	1.607	5.105	21.25	0.1
  X16.X24	0.1017	1.441	5.065	18.49	0.4
  X24.X31	0.1093	1.71	5.03	31.58	0.95
  X4.X27	0.102	1.726	5.015	34.81	1
  X23.X29	0.09881	0.985	4.959	9.304	0.1
  X24.X27	0.09699	1.742	4.914	29.93	1
  X16.X27	0.0926	1.303	4.854	14.31	0.15
  X12.X24	0.1026	1.583	4.818	22.1	0.2
  X7.X24	0.09283	1.707	4.789	28.68	1
  X24.X25	0.09945	1.752	4.746	33.15	1
  X17.X24	0.08342	1.495	4.739	19.74	0.35
  X4.X7	0.08841	1.702	4.632	25.68	0.85
  X4.X6	0.07132	1.379	4.219	14.41	0.15
  X10.X24	0.07324	1.502	4.188	18.52	0.8
  X13.X24	0.06966	0.991	4.12	8.709	0.1
  X22.X29	0.06055	1.58	3.952	20.66	0.1
  X24	0.06743	1.673	3.81	25.56	0.55
  X4.X31	0.06431	1.411	3.766	16.66	0.4
  X4.X24	0.05054	1.482	3.43	17.38	0.1

• Accuracy: 0.9882
• tAUC: 0.987
• sensitivity: 0.9831
• specificity: 0.991

• bootstrap:

hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)

[1] 2

vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LM",rankingTest = "Ztest",uniType = "Regression")[,c("cohortMean","cohortStd","kendall.r","kendall.p")] 
#> 100 : X17.X31


cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ,caption = "Features",round = 4)

Features

	cohortMean	cohortStd	kendall.r	kendall.p	Frequency
X14.X28	0.3014	0.2331	0.4354	0	0.5496
X11.X27	0.0242	0.0067	0.3637	0	0.5455
X14.X20	0.0151	0.0139	0.2869	0	0.5455
X17.X28	0.0018	0.0013	0.4052	0	0.5372
X31.X32	0.025	0.0103	0.3264	0	0.5331
X11.X24	4.667	1.376	0.4528	0	0.5248
X22.X24	0.0974	0.0677	0.2926	0	0.5248
X18.X21	6e-04	7e-04	0.2221	0	0.5165
X21.X28	0.0053	0.0051	0.4142	0	0.5124
X27.X32	0.0114	0.0042	0.3307	0	0.5041
X8.X21	0.0022	0.0021	0.3544	0	0.5041
X21.X24	0.5235	0.2432	0.1713	0	0.5
X14.X31	0.3486	0.1627	0.1582	0	0.5
X27.X31	0.0391	0.0136	0.3842	0	0.4959
X11.X32	0.0154	0.0051	0.3006	0	0.4917
X32	0.0839	0.0181	0.2546	0	0.4917
X11.X12	0.0115	0.0027	0.1684	0	0.4793
X4.X31	5.623	1.849	0.4652	0	0.4752
X17.X18	2e-04	2e-04	0.2264	0	0.4711
X14.X24	32.63	19.28	0.1664	0	0.4628
X17.X25	0.7333	0.3568	0.4342	0	0.4587
X20.X24	0.3062	0.1782	0.5016	0	0.4463
X31	0.2901	0.0619	0.3244	0	0.438
X27	0.1324	0.0228	0.348	0	0.4215
X7.X11	0.0177	0.0048	0.3263	0	0.4132
X21.X26	17.57	13.99	0.577	0	0.405
X21.X25	2.175	1.173	0.42	0	0.4008
X14.X19	0.0421	0.0622	0.3298	0	0.4008
X24.X26	23819	19106	0.6585	0	0.3802
X17.X26	5.889	4.48	0.5863	0	0.3802
X4.X7	1.857	0.4872	0.4685	0	0.3719
X22.X31	0.0011	8e-04	0.2518	0	0.3719
X4.X12	1.209	0.2914	0.3187	0	0.3678
X14.X26	1045	825.3	0.5585	0	0.3595
X24	25.68	6.146	0.3897	0	0.3595
X4.X27	2.561	0.7593	0.4799	0	0.3595
X17.X31	0.002	9e-04	0.1233	3e-04	0.3347
X19.X24	0.8375	0.7736	0.4537	0	0.3306
X14.X16	51.88	71.38	0.4813	0	0.3264
X24.X32	2.18	0.8164	0.4184	0	0.3223
X18.X24	0.6699	0.5264	0.3945	0	0.3223
X16.X21	0.879	1.453	0.4887	0	0.314
X8.X24	2.76	1.741	0.5655	0	0.3058
X21.X27	0.0027	0.0012	0.0977	0.0044	0.3058
X23.X27	2.177	0.8291	0.6563	0	0.3017
X16.X17	0.2943	0.4696	0.4886	0	0.3017
X26.X27	119.3	85.62	0.6679	0	0.2975
X14.X22	0.005	0.0058	0.1208	4e-04	0.2975
X10.X24	1.326	1.196	0.6612	0	0.2893
X21.X31	0.0062	0.0039	0.0839	0.0144	0.2893
X24.X30	3.088	2.166	0.651	0	0.2851
X22.X26	3.308	2.999	0.5629	0	0.2769
X4.X24	519.4	239.9	0.395	0	0.2686
X12.X24	1.61	0.4194	0.3426	0	0.2686
X21.X32	0.0017	9e-04	0.0764	0.0259	0.2686
X17.X21	2e-04	1e-04	-0.0604	0.0782	0.2686
X30	0.1146	0.0657	0.6391	0	0.2645
X4.X32	1.629	0.5511	0.4141	0	0.2603
X3.X4	277.4	109.3	0.5897	0	0.2562
X7.X26	86.51	61.38	0.6629	0	0.2521
X26.X30	128.8	150.6	0.6659	0	0.2479
X3	14.13	3.524	0.5991	0	0.2479
X14.X32	0.1017	0.0498	0.1298	2e-04	0.2397
X7.X24	2.477	0.7098	0.4801	0	0.2355
X13	0.4052	0.2773	0.5042	0	0.2355
X21	0.0205	0.0083	-0.0755	0.0278	0.2355
X24.X31	7.537	2.745	0.4609	0	0.2273
X6.X27	87.68	52.22	0.6578	0	0.2231
X26.X31	262.8	192.2	0.657	0	0.2149
X11.X31	0.0537	0.019	0.319	0	0.2149
X17.X27	0.001	5e-04	0.0993	0.0038	0.2149
X21.X22	1e-04	1e-04	0.09	0.0087	0.2149
X26.X29	304.1	376.6	0.6544	0	0.2107
X18.X22	1e-04	2e-04	0.2633	0	0.2107
X3.X31	4.134	1.498	0.6264	0	0.2066
X10.X14	0.06	0.0626	0.5805	0	0.2066
X14.X25	128.6	68.14	0.4106	0	0.2025
X25.X27	14.38	5.733	0.6563	0	0.1983
X4.X30	2.294	1.576	0.6533	0	0.1983
X23.X31	4.792	1.999	0.6399	0	0.1942
X26.X32	74.74	52.35	0.66	0	0.1942
X12.X26	54.37	34.1	0.6607	0	0.1942
X3.X27	1.88	0.6084	0.6492	0	0.1901
X4.X26	17826	14139	0.6508	0	0.1901
X6.X17	4.435	3.322	0.5213	0	0.1901
X23	16.27	4.833	0.6442	0	0.186
X3.X11	2.574	0.8233	0.6214	0	0.186
X9	0.0888	0.0797	0.5994	0	0.186
X4.X11	3.503	0.9729	0.4366	0	0.1818
X7.X27	0.013	0.004	0.3416	0	0.1818
X7	0.0964	0.0141	0.304	0	0.1818
X14.X21	0.0269	0.022	-0.0251	0.4641	0.1818
X24.X25	2829	1336	0.642	0	0.1777
X8.X13	0.0495	0.0591	0.5811	0	0.1777
X24.X27	3.43	1.147	0.4701	0	0.1777
X4	19.29	4.301	0.3776	0	0.1777
X25	107.3	33.6	0.6509	0	0.1736
X6.X14	784.1	579	0.4934	0	0.1736
X17.X32	6e-04	3e-04	0.091	0.008	0.1736
X16.X30	6.229	10.56	0.6609	0	0.1694
X22.X27	5e-04	4e-04	0.2369	0	0.1694
X21.X29	0.0057	0.0062	0.5315	0	0.1612
X7.X31	0.0283	0.0089	0.3703	0	0.1612
X23.X24	428.4	195.8	0.6349	0	0.157
X11.X26	162.3	114.5	0.6585	0	0.157
X10.X26	60.93	88.95	0.6629	0	0.157
X12.X27	0.0084	0.0022	0.2338	0	0.157
X6.X21	13.24	10.27	0.4998	0	0.1529
X4.X25	2121	977.1	0.6313	0	0.1488
X17.X22	0	0	0.0946	0.0058	0.1488
X5.X24	2407	995	0.6207	0	0.1446
X16.X20	0.5923	0.9704	0.5723	0	0.1446
X12	0.0628	0.0071	-0.0212	0.537	0.1446
X12.X23	1.013	0.2943	0.6531	0	0.1405
X13.X24	10.74	8.247	0.5678	0	0.1405
X4.X10	0.9925	0.8985	0.6565	0	0.1364
X6.X24	17436	11571	0.6355	0	0.1364
X28	0.2543	0.1573	0.496	0	0.1364
X17	0.007	0.003	-0.0427	0.2135	0.1364
X3.X7	1.37	0.4347	0.6328	0	0.1322
X17.X29	0.0019	0.0017	0.5143	0	0.1322
X10.X17	3e-04	4e-04	0.563	0	0.1281
X8.X31	0.0319	0.0225	0.5063	0	0.1281
X14.X30	0.1351	0.0999	0.5632	0	0.124
X23.X30	2.114	1.762	0.6604	0	0.1157
X23.X32	1.374	0.533	0.632	0	0.1157
X4.X20	0.2319	0.1406	0.4745	0	0.1157
X14.X17	0.0092	0.0076	-0.0065	0.8502	0.1157
X17.X30	8e-04	6e-04	0.5393	0	0.1157
X3.X30	1.791	1.4	0.6564	0	0.1116
X19.X26	31.32	37.14	0.5952	0	0.1116
X9.X26	108.8	165.9	0.6534	0	0.1116
X3.X24	369.2	147.6	0.6108	0	0.1033
X16.X24	1091	1311	0.6247	0	0.1033
X12.X21	0.0013	6e-04	-0.0664	0.0529	0.1033
X16.X26	56499	138558	0.6328	0	0.1033
X19.X31	0.0096	0.0105	0.4332	0	0.1033