1 FRESA.CAD ARCENE Benchmark

1.1 ARCENE


trainLabeled <- read.delim("C:/Users/Jose Tamez/Dropbox/Documents/FRESACAD/FresaPaper/FresaPaper/Arcene/ARCENE/trainSet.txt")
validLabeled <- read.delim("C:/Users/Jose Tamez/Dropbox/Documents/FRESACAD/FresaPaper/FresaPaper/Arcene/ARCENE/arcene_valid.txt")

trainLabeled$Labels <-  1*(trainLabeled$Labels > 0)
validLabeled$Labels <-  1*(validLabeled$Labels > 0)

sum(trainLabeled$Labels)
sum(validLabeled$Labels)

arcene <- rbind(trainLabeled,validLabeled)
arcene <- as.data.frame(arcene)

ExperimentName <- "ARCENE"
bswimsReps <- 10;
theData <- arcene;
theOutcome <- "Labels";
reps <- 30;
fraction <- 0.5;

BSWiMSFileName <- paste(ExperimentName,"FRESAMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

1.2 Benchmarking



BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = bswimsReps)

#save(BSWiMSMODEL,file = BSWiMSFileName)

#load(file = BSWiMSFileName)

par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

par(mfrow = c(1,1),cex=1.0);

#save(cp,file = CVFileName)

#load(file = CVFileName)

1.3 Results

1.3.1 Classifier Results


hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


#The Times
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
46.84	88.02	8.344	1.709	0.04567	0.029	145

learningTime <- -1*cp$cpuElapsedTimes
par(mfrow = c(2,1),cex=1.0);
pr <- plot(cp)

par(mfrow = c(1,1),cex=1.0);

1.3.2 Radar Plots

op <- par(no.readonly = TRUE)

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

mNames <- names(cp$cpuElapsedTimes)

classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")

classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filnames <- c("BSWiMS","LASSO","RF.ref","IDI","t-test","Kendall","mRMR")

filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));

filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.6,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


detach("package:fmsb", unload=TRUE)

par(mfrow = c(1,1))
par(op)

1.3.3 Feature Analysis



rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(selFrequency,trace = "none",mar = c(10,10),main = "Features",cexRow = 0.5)



topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network")

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

coefficients:

Table continues below
	Estimate	lower	OR	upper	u.Accuracy	r.Accuracy
V2804	0.001731	1.001	1.002	1.002	0.675	0.7113
V2134	-0.0001151	0.9999	0.9999	0.9999	0.675	0.64
V7976	0.0002679	1	1	1	0.595	0.695
V8405	-0.0002958	0.9996	0.9997	0.9998	0.485	0.765
V7856	0.0001637	1	1	1	0.675	0.745
V7217	-7.409e-05	0.9999	0.9999	0.9999	0.67	0.725
V3365	-0.002719	0.9965	0.9973	0.998	0.685	0.792
V8685	-0.001581	0.998	0.9984	0.9989	0.56	0.7471
V33	0.0002248	1	1	1	0.56	0.75
V7136	0.0005582	1	1.001	1.001	0.565	0.795
V2747	0.0002463	1	1	1	0.54	0.7233
V6487	-0.000342	0.9996	0.9997	0.9998	0.5	0.83
V7195	0.0002212	1	1	1	0.575	0.815
V3725	0.0003455	1	1	1	0.605	0.7525
V2294	0.001125	1.001	1.001	1.001	0.69	0.778
V4070	0.0004281	1	1	1.001	0.69	0.7388
V5417	-0.001072	0.9986	0.9989	0.9993	0.57	0.7406
V9678	0.0003424	1	1	1	0.56	0.71
V2556	0.001534	1.001	1.002	1.002	0.71	0.8012
V9275	-0.003488	0.9953	0.9965	0.9977	0.565	0.7925
V9215	-0.00316	0.9958	0.9968	0.9979	0.58	0.7889
V9585	-0.003931	0.9948	0.9961	0.9974	0.685	0.7856
V6408	0.0004817	1	1	1.001	0.565	0.8
V8055	-0.002111	0.9972	0.9979	0.9986	0.55	0.7817
V4414	-0.001526	0.998	0.9985	0.999	0.635	0.82
V5	-0.0008247	0.9989	0.9992	0.9995	0.665	0.7158
V6511	3.936e-05	1	1	1	0.635	0.72
V2256	-0.001718	0.9977	0.9983	0.9989	0.585	0.7775
V2297	0.0003623	1	1	1	0.53	0.83
V6146	0.0004681	1	1	1.001	0.615	0.749
V2242	0.0002177	1	1	1	0.58	0.825
V872	-0.001849	0.9975	0.9982	0.9988	0.675	0.7629
V5761	-0.001969	0.9973	0.998	0.9987	0.56	0.7645
V6508	-0.0005087	0.9993	0.9995	0.9997	0.695	0.7825
V6959	-0.001035	0.9986	0.999	0.9993	0.58	0.7769
V5391	-0.0004055	0.9994	0.9996	0.9997	0.685	0.772
V8156	-0.001637	0.9978	0.9984	0.999	0.58	0.7863
V533	-0.002485	0.9966	0.9975	0.9984	0.67	0.7763
V7272	-0.003809	0.9948	0.9962	0.9976	0.66	0.79
V1967	-7.074e-05	0.9999	0.9999	1	0.69	0.77
V9635	-0.001213	0.9983	0.9988	0.9992	0.495	0.7675
V9965	-0.0007328	0.999	0.9993	0.9995	0.535	0.7486
V7891	0.001116	1.001	1.001	1.002	0.685	0.7906
V1142	0.0001568	1	1	1	0.56	0.77
V8502	-0.001785	0.9976	0.9982	0.9989	0.665	0.765
V7319	-0.001188	0.9984	0.9988	0.9993	0.48	0.7567
V4584	-0.001676	0.9977	0.9983	0.9989	0.51	0.783
V434	0.0008116	1.001	1.001	1.001	0.58	0.785
V1787	-0.002176	0.997	0.9978	0.9986	0.685	0.7707
V6594	0.0005537	1	1.001	1.001	0.665	0.8
V9947	0.0007902	1	1.001	1.001	0.56	0.785
V3557	-0.0003415	0.9995	0.9997	0.9998	0.695	0.7725
V729	-0.001916	0.9974	0.9981	0.9988	0.57	0.8081
V8517	0.000817	1.001	1.001	1.001	0.53	0.8025
V3826	0.0001171	1	1	1	0.595	0.79
V1975	-0.002527	0.9965	0.9975	0.9984	0.68	0.7922
V4290	-0.0024	0.9967	0.9976	0.9985	0.665	0.7811
V9395	-0.00499	0.9931	0.995	0.9969	0.555	0.8056
V7748	-0.004433	0.9939	0.9956	0.9973	0.66	0.7955
V9432	0.0001444	1	1	1	0.695	0.774
V7928	-0.002296	0.9968	0.9977	0.9986	0.64	0.79
V3222	-0.0008078	0.9989	0.9992	0.9995	0.565	0.8117
V6958	-0.0005404	0.9992	0.9995	0.9997	0.55	0.8
V5489	-2.66e-05	1	1	1	0.65	0.74
V6163	-0.002178	0.997	0.9978	0.9987	0.49	0.7989
V34	-0.001229	0.9983	0.9988	0.9993	0.57	0.7808
V5801	0.0004546	1	1	1.001	0.705	0.7767
V5400	-0.003018	0.9958	0.997	0.9982	0.635	0.7821
V3082	-0.001001	0.9986	0.999	0.9994	0.64	0.82
V5473	-0.001055	0.9985	0.9989	0.9994	0.665	0.755
V7857	-0.004158	0.9942	0.9959	0.9975	0.675	0.7939
V4194	-0.003791	0.9947	0.9962	0.9978	0.655	0.8106
V7062	-0.0004498	0.9994	0.9996	0.9997	0.57	0.82
V6584	-0.002064	0.9971	0.9979	0.9988	0.535	0.7939
V5378	-0.001219	0.9983	0.9988	0.9993	0.505	0.777
V4069	-0.001157	0.9984	0.9988	0.9993	0.585	0.7879
V9818	-0.004235	0.994	0.9958	0.9975	0.57	0.8175
V7513	-0.002074	0.9971	0.9979	0.9988	0.63	0.775
V8245	0.0004198	1	1	1.001	0.58	0.8
V3152	-0.0002194	0.9997	0.9998	0.9999	0.61	0.765
V6121	0.002913	1.002	1.003	1.004	0.56	0.8017
V782	-0.001775	0.9975	0.9982	0.999	0.695	0.7793
V1451	0.0001727	1	1	1	0.555	0.81
V9402	-5.874e-05	0.9999	0.9999	1	0.62	0.76
V3339	-0.0002783	0.9996	0.9997	0.9998	0.635	0.785
V6688	-0.00391	0.9945	0.9961	0.9977	0.555	0.8
V8585	-0.000278	0.9996	0.9997	0.9998	0.68	0.7967
V8972	-0.0002523	0.9996	0.9997	0.9999	0.58	0.8
V2783	-0.0003039	0.9996	0.9997	0.9998	0.495	0.7417
V1198	-0.0002973	0.9996	0.9997	0.9998	0.53	0.7625
V6685	0.0009784	1.001	1.001	1.001	0.65	0.8367
V6141	0.001125	1.001	1.001	1.002	0.56	0.81
V86	-0.003706	0.9947	0.9963	0.9979	0.675	0.8022
V6180	-0.0001023	0.9999	0.9999	0.9999	0.505	0.805
V8267	-0.0002378	0.9997	0.9998	0.9999	0.66	0.73
V9213	-0.003122	0.9955	0.9969	0.9982	0.56	0.769
V6239	-0.001936	0.9972	0.9981	0.9989	0.655	0.8125
V1046	-0.003657	0.9947	0.9963	0.998	0.56	0.7969
V1943	-0.0005084	0.9993	0.9995	0.9997	0.55	0.83
V7033	0.00126	1.001	1.001	1.002	0.56	0.795
V6164	-0.0003667	0.9995	0.9996	0.9998	0.48	0.7675
V4285	-0.0004476	0.9994	0.9996	0.9998	0.56	0.825
V2515	-0.002237	0.9968	0.9978	0.9988	0.555	0.8075
V3652	-0.0004469	0.9994	0.9996	0.9998	0.475	0.82
V3206	-0.002835	0.9959	0.9972	0.9985	0.65	0.8069
V819	-0.000173	0.9997	0.9998	0.9999	0.525	0.76
V256	-0.002418	0.9965	0.9976	0.9987	0.505	0.8089
V8193	-0.3104	0.636	0.7332	0.8451	0.56	0.805
V5721	-0.00137	0.998	0.9986	0.9993	0.56	0.7883
V3189	-0.0004502	0.9993	0.9995	0.9998	0.635	0.84
V7220	-0.001782	0.9974	0.9982	0.999	0.68	0.7879
V8038	-0.000609	0.9991	0.9994	0.9997	0.54	0.79
V723	-0.00123	0.9982	0.9988	0.9993	0.65	0.8067
V3545	0.0001813	1	1	1	0.59	0.7917
V3161	0.002229	1.001	1.002	1.003	0.585	0.8211
V1248	-0.002856	0.9958	0.9971	0.9985	0.58	0.8065
V6780	-0.001004	0.9985	0.999	0.9995	0.55	0.8225
V4017	-0.001817	0.9973	0.9982	0.999	0.56	0.8008
V7302	0.001906	1.001	1.002	1.003	0.56	0.8117
V2750	-0.001798	0.9974	0.9982	0.9991	0.56	0.8263
V5005	0.0008678	1	1.001	1.001	0.695	0.8115
V4406	-0.0003498	0.9995	0.9997	0.9998	0.59	0.7887
V4554	0.0005107	1	1.001	1.001	0.665	0.8225
V130	-0.00655	0.9904	0.9935	0.9966	0.56	0.8017
V9329	-0.0004471	0.9993	0.9996	0.9998	0.47	0.795
V4301	-0.003461	0.9949	0.9965	0.9982	0.56	0.7894
V4185	-0.001098	0.9984	0.9989	0.9994	0.55	0.7783
V9117	0.0003784	1	1	1.001	0.59	0.775
V9031	0.0001496	1	1	1	0.605	0.7917
V2438	-0.0006591	0.999	0.9993	0.9997	0.56	0.7867
V1512	4.763e-05	1	1	1	0.655	0.79
V311	0.0001537	1	1	1	0.54	0.795
V3170	-0.007533	0.9888	0.9925	0.9962	0.56	0.825
V5774	0.001076	1.001	1.001	1.002	0.59	0.8013
V3125	0.0001505	1	1	1	0.63	0.845
V762	0.0005802	1	1.001	1.001	0.59	0.7817
V9070	-0.001411	0.9979	0.9986	0.9993	0.575	0.8212
V3592	-0.000166	0.9998	0.9998	0.9999	0.57	0.775
V9735	0.0001448	1	1	1	0.685	0.7817
V4900	-0.0003899	0.9994	0.9996	0.9998	0.65	0.805
V6111	-0.0001112	0.9998	0.9999	0.9999	0.54	0.785
V4580	-0.001852	0.9972	0.9982	0.9991	0.56	0.791
V3591	3.78e-05	1	1	1	0.625	0.78
V7315	-0.002665	0.996	0.9973	0.9987	0.56	0.84
V1184	-0.0008565	0.9987	0.9991	0.9996	0.555	0.798
V7435	0.0001141	1	1	1	0.55	0.81
V1936	0.000672	1	1.001	1.001	0.69	0.8131
V1715	-6.636e-05	0.9999	0.9999	1	0.64	0.785
V4198	-0.0007624	0.9988	0.9992	0.9996	0.635	0.8325
V306	7.197e-05	1	1	1	0.61	0.805
V2657	-0.0002251	0.9997	0.9998	0.9999	0.525	0.825
V8650	0.0004814	1	1	1.001	0.6	0.82
V188	-0.0003125	0.9995	0.9997	0.9999	0.56	0.755
V1293	0.0004298	1	1	1.001	0.605	0.81
V5148	-8.625e-05	0.9999	0.9999	1	0.615	0.835
V4564	-0.002259	0.9965	0.9977	0.9989	0.56	0.8415
V630	-0.0006414	0.999	0.9994	0.9997	0.56	0.765
V6123	0.0004872	1	1	1.001	0.57	0.88
V8428	-0.0002019	0.9997	0.9998	0.9999	0.56	0.8
V5321	-0.006559	0.99	0.9935	0.997	0.56	0.828
V7446	5.311e-05	1	1	1	0.555	0.8
V1336	0.0001589	1	1	1	0.54	0.8225
V662	-0.0008147	0.9987	0.9992	0.9996	0.65	0.7983
V2358	0.00127	1.001	1.001	1.002	0.585	0.7931
V8611	0.0005966	1	1.001	1.001	0.565	0.86
V7402	-0.0007037	0.9989	0.9993	0.9997	0.56	0.7883
V4738	-0.0002374	0.9996	0.9998	0.9999	0.56	0.82
V9704	-0.0003618	0.9994	0.9996	0.9998	0.56	0.8425
V4352	-0.003316	0.9949	0.9967	0.9985	0.62	0.83
V5672	0.001928	1.001	1.002	1.003	0.595	0.8181
V3708	-0.000139	0.9998	0.9999	0.9999	0.54	0.775
V2597	-0.05887	0.9117	0.9428	0.975	0.56	0.795
V2856	-0.0008149	0.9987	0.9992	0.9997	0.56	0.8575
V3271	0.0003543	1	1	1.001	0.56	0.775
V8749	-0.0002151	0.9997	0.9998	0.9999	0.56	0.79
V7899	-0.0006421	0.999	0.9994	0.9997	0.595	0.825
V1202	0.0003698	1	1	1.001	0.57	0.825
V5995	0.0001477	1	1	1	0.6	0.815
V1529	-0.0003031	0.9995	0.9997	0.9999	0.615	0.8

Table continues below
	full.Accuracy	u.AUC	r.AUC	full.AUC	IDI	NRI
V2804	0.8437	0.6636	0.7121	0.8431	0.2765	1.065
V2134	0.78	0.6879	0.6664	0.789	0.2488	1.023
V7976	0.835	0.5751	0.6978	0.8369	0.2193	0.7792
V8405	0.855	0.4598	0.7634	0.8523	0.2061	0.8442
V7856	0.835	0.6745	0.7419	0.832	0.2267	1.008
V7217	0.805	0.6834	0.7155	0.804	0.2023	0.8929
V3365	0.866	0.6859	0.7912	0.8659	0.1882	1.088
V8685	0.8414	0.5	0.7509	0.841	0.1946	0.9281
V33	0.835	0.5	0.7415	0.832	0.184	1.114
V7136	0.8675	0.5276	0.7938	0.8683	0.165	0.9497
V2747	0.8167	0.5114	0.727	0.8168	0.1698	0.9637
V6487	0.895	0.472	0.8324	0.8965	0.1395	1.054
V7195	0.86	0.5633	0.8105	0.858	0.132	0.862
V3725	0.8237	0.5913	0.7437	0.8232	0.1411	0.6506
V2294	0.848	0.6843	0.7784	0.8475	0.1385	1.006
V4070	0.8287	0.683	0.7354	0.8291	0.1465	0.8929
V5417	0.8162	0.5613	0.7435	0.8169	0.1448	0.5913
V9678	0.78	0.5	0.7155	0.789	0.1561	0.4448
V2556	0.8669	0.7045	0.7999	0.8647	0.1268	0.9314
V9275	0.8445	0.5678	0.7943	0.8442	0.1353	0.8542
V9215	0.8456	0.586	0.7884	0.8463	0.1362	0.8784
V9585	0.8433	0.7041	0.7858	0.8447	0.1251	0.9509
V6408	0.845	0.5227	0.8044	0.8458	0.0979	1.008
V8055	0.8422	0.5361	0.7794	0.8418	0.1403	0.6861
V4414	0.895	0.6619	0.8222	0.8965	0.1287	1.062
V5	0.8042	0.6838	0.715	0.8065	0.1153	0.6891
V6511	0.8	0.6193	0.7123	0.7983	0.1313	0.5438
V2256	0.8312	0.5905	0.7779	0.8323	0.1262	0.7837
V2297	0.905	0.4781	0.8287	0.9054	0.1019	0.8214
V6146	0.828	0.599	0.7459	0.8274	0.1243	0.6445
V2242	0.86	0.5593	0.8243	0.858	0.1321	0.7386
V872	0.8136	0.6952	0.7648	0.8156	0.1146	0.8822
V5761	0.8325	0.5499	0.7657	0.8338	0.1278	0.5852
V6508	0.8325	0.7045	0.7827	0.8328	0.1209	0.8247
V6959	0.83	0.569	0.7783	0.829	0.1325	0.817
V5391	0.812	0.6956	0.7709	0.8131	0.1249	0.7453
V8156	0.84	0.586	0.785	0.8386	0.1212	0.7094
V533	0.8162	0.6907	0.7765	0.8177	0.1145	0.972
V7272	0.83	0.683	0.79	0.832	0.1081	1.005
V1967	0.81	0.6964	0.7703	0.8084	0.125	0.862
V9635	0.825	0.47	0.7684	0.8264	0.1268	0.6613
V9965	0.8064	0.5373	0.7492	0.8063	0.1188	0.6521
V7891	0.8422	0.6871	0.792	0.8417	0.1243	0.8959
V1142	0.835	0.5	0.7691	0.8369	0.1041	0.4951
V8502	0.825	0.6692	0.764	0.8246	0.117	0.7936
V7319	0.8242	0.4554	0.76	0.8262	0.1135	0.6974
V4584	0.8415	0.4968	0.7849	0.8431	0.1167	0.8314
V434	0.84	0.5373	0.78	0.8389	0.1161	1.073
V1787	0.8164	0.7041	0.7715	0.8175	0.1032	0.7195
V6594	0.8375	0.6656	0.8056	0.8385	0.119	0.8547
V9947	0.85	0.5	0.7812	0.849	0.1127	0.6867
V3557	0.8313	0.7021	0.7701	0.8305	0.1144	0.804
V729	0.8644	0.5625	0.8075	0.8637	0.1154	0.8559
V8517	0.8367	0.4927	0.8062	0.8357	0.1141	0.8157
V3826	0.835	0.5775	0.7796	0.832	0.1108	0.7711
V1975	0.8378	0.6997	0.7911	0.8393	0.09847	0.7691
V4290	0.8356	0.6692	0.7798	0.8357	0.1062	0.917
V9395	0.8633	0.5722	0.8058	0.8623	0.106	0.8323
V7748	0.855	0.6684	0.794	0.8553	0.1094	1.034
V9432	0.816	0.6912	0.7739	0.8157	0.1065	0.6958
V7928	0.84	0.664	0.7902	0.8409	0.09316	0.8723
V3222	0.835	0.569	0.8176	0.8373	0.1014	0.7167
V6958	0.88	0.558	0.8007	0.877	0.09668	0.8636
V5489	0.81	0.6388	0.7374	0.8097	0.1013	0.8052
V6163	0.8411	0.4752	0.7987	0.8414	0.1047	0.8405
V34	0.8317	0.5674	0.781	0.8312	0.1068	0.6358
V5801	0.8289	0.7062	0.775	0.8286	0.1029	0.7527
V5400	0.82	0.6607	0.7836	0.8217	0.08791	0.817
V3082	0.845	0.6445	0.8222	0.8446	0.1018	1.039
V5473	0.8207	0.668	0.7529	0.8197	0.09776	0.8676
V7857	0.855	0.683	0.7937	0.8551	0.09351	0.9102
V4194	0.8467	0.6798	0.8115	0.8489	0.09143	0.9817
V7062	0.86	0.5625	0.8174	0.8616	0.1022	0.7565
V6584	0.8394	0.5325	0.7961	0.8397	0.09806	0.6804
V5378	0.82	0.4911	0.7785	0.8193	0.09894	0.649
V4069	0.8357	0.5905	0.7906	0.8364	0.09602	0.6484
V9818	0.865	0.5747	0.8167	0.8656	0.0971	0.8281
V7513	0.8163	0.655	0.7745	0.818	0.08366	0.8271
V8245	0.85	0.5507	0.7983	0.849	0.08513	0.8263
V3152	0.795	0.6481	0.7658	0.7987	0.07757	0.7662
V6121	0.8267	0.5	0.8059	0.8278	0.1054	0.8582
V782	0.8207	0.7143	0.7808	0.8229	0.08893	0.816
V1451	0.88	0.5101	0.8084	0.8807	0.07435	0.7873
V9402	0.81	0.6059	0.7638	0.8097	0.08419	0.6055
V3339	0.835	0.6644	0.78	0.8332	0.08901	0.8003
V6688	0.8465	0.5722	0.8002	0.8455	0.09174	0.6794
V8585	0.8183	0.6912	0.7953	0.8167	0.09872	0.6818
V8972	0.85	0.558	0.7971	0.849	0.07835	0.7646
V2783	0.8067	0.4846	0.7417	0.8067	0.08785	0.5758
V1198	0.8175	0.5207	0.7642	0.8176	0.08443	0.6055
V6685	0.8667	0.6425	0.8363	0.8676	0.09109	0.7668
V6141	0.85	0.5	0.817	0.8539	0.09448	0.9026
V86	0.8478	0.683	0.8008	0.848	0.08562	0.8604
V6180	0.815	0.4789	0.8052	0.8153	0.08739	0.6688
V8267	0.7825	0.6794	0.7334	0.7869	0.07351	0.7606
V9213	0.817	0.5	0.7713	0.8186	0.06388	0.7045
V6239	0.8408	0.6774	0.8107	0.8404	0.0813	0.8704
V1046	0.825	0.5	0.7998	0.827	0.06806	0.6998
V1943	0.86	0.5666	0.83	0.858	0.07337	0.4026
V7033	0.84	0.5	0.795	0.8407	0.09396	0.9383
V6164	0.8175	0.4785	0.7687	0.8157	0.07855	0.3141
V4285	0.87	0.5	0.8255	0.8718	0.05261	0.2646
V2515	0.8433	0.5686	0.8084	0.8433	0.08014	0.6426
V3652	0.86	0.446	0.8186	0.8616	0.08252	0.4935
V3206	0.8375	0.6741	0.8067	0.8386	0.07807	0.8689
V819	0.8025	0.5211	0.7608	0.8048	0.07987	0.4326
V256	0.8422	0.4935	0.8103	0.8426	0.08274	0.6263
V8193	0.83	0.5	0.8109	0.8344	0.06928	0.9037
V5721	0.8133	0.5	0.7915	0.8163	0.07376	0.7332
V3189	0.875	0.6388	0.8401	0.875	0.07794	0.9578
V7220	0.8243	0.6887	0.7866	0.8236	0.07111	0.7347
V8038	0.815	0.5491	0.7918	0.8147	0.07941	0.75
V723	0.8317	0.6729	0.8022	0.8314	0.06866	0.7446
V3545	0.8233	0.5694	0.7969	0.824	0.07315	0.5471
V3161	0.8644	0.5601	0.8203	0.8637	0.09108	0.7738
V1248	0.848	0.5946	0.8076	0.8488	0.07748	0.5222
V6780	0.8625	0.5653	0.8239	0.8644	0.07341	0.6193
V4017	0.8333	0.5	0.8013	0.8327	0.08015	0.6521
V7302	0.8317	0.5	0.8148	0.8367	0.08563	0.6778
V2750	0.845	0.5	0.8251	0.8446	0.0729	0.6818
V5005	0.85	0.696	0.8105	0.8509	0.07523	0.7638
V4406	0.8425	0.5816	0.7904	0.842	0.07207	0.5544
V4554	0.855	0.6607	0.8214	0.8535	0.0687	0.6031
V130	0.8317	0.5	0.803	0.8329	0.05331	0.7006
V9329	0.845	0.4489	0.7975	0.8446	0.06619	0.5901
V4301	0.8206	0.5	0.7923	0.8228	0.0554	0.5422
V4185	0.8275	0.5666	0.7794	0.8281	0.06949	0.5146
V9117	0.815	0.5487	0.7735	0.8141	0.07214	0.5925
V9031	0.8317	0.5864	0.7909	0.8314	0.07009	0.5227
V2438	0.8267	0.5718	0.7884	0.8262	0.04978	0.3111
V1512	0.805	0.6384	0.7979	0.8088	0.07687	0.5471
V311	0.8317	0.5126	0.8011	0.8331	0.06349	0.4957
V3170	0.8528	0.5	0.8248	0.8533	0.04999	0.699
V5774	0.8287	0.5548	0.8031	0.8273	0.08117	0.3742
V3125	0.875	0.5893	0.8433	0.875	0.06658	0.6851
V762	0.8117	0.5548	0.7856	0.812	0.07672	0.4416
V9070	0.845	0.5864	0.8212	0.8433	0.05857	0.5804
V3592	0.82	0.5832	0.7821	0.8247	0.06189	0.5284
V9735	0.8108	0.6847	0.7813	0.8094	0.06037	0.5939
V4900	0.825	0.6534	0.8082	0.8237	0.06958	0.8101
V6111	0.805	0.554	0.7849	0.8015	0.06269	0.4205
V4580	0.815	0.5	0.7917	0.8163	0.05587	0.6821
V3591	0.825	0.6067	0.778	0.8267	0.07208	0.4789
V7315	0.855	0.5	0.8364	0.8523	0.05135	1.01
V1184	0.8375	0.5199	0.8002	0.8387	0.05713	0.6221
V7435	0.85	0.5142	0.806	0.8478	0.05891	0.526
V1936	0.84	0.6818	0.8146	0.8406	0.0709	0.6707
V1715	0.83	0.6335	0.7812	0.8263	0.05182	0.7127
V4198	0.8375	0.6437	0.834	0.8379	0.06831	0.8515
V306	0.825	0.5921	0.8088	0.8304	0.06751	0.5958
V2657	0.85	0.4785	0.8316	0.849	0.06131	0.7938
V8650	0.8525	0.5686	0.821	0.8531	0.06078	0.5584
V188	0.79	0.5	0.7593	0.7918	0.05803	0.7857
V1293	0.855	0.556	0.8097	0.8559	0.05516	0.8003
V5148	0.85	0.6027	0.8356	0.8478	0.05354	0.7175
V4564	0.871	0.5	0.8406	0.8697	0.05402	0.5823
V630	0.8	0.5	0.7622	0.7983	0.05273	0.9302
V6123	0.905	0.5235	0.8795	0.9054	0.04677	0.7825
V8428	0.8217	0.5	0.806	0.8233	0.05404	0.5244
V5321	0.8515	0.5	0.8288	0.8511	0.05899	0.8244
V7446	0.845	0.5296	0.8007	0.8446	0.05159	0.237
V1336	0.835	0.4882	0.8245	0.8362	0.04044	0.3328
V662	0.8117	0.6607	0.8013	0.8132	0.06277	0.8122
V2358	0.8369	0.5588	0.7922	0.8349	0.05944	0.6045
V8611	0.89	0.5751	0.858	0.886	0.04615	0.5455
V7402	0.81	0.5	0.7936	0.8137	0.0543	0.7476
V4738	0.85	0.5	0.8186	0.8478	0.05658	0.9497
V9704	0.8575	0.5	0.8423	0.8569	0.04287	0.418
V4352	0.8605	0.6266	0.8302	0.8607	0.0563	0.8571
V5672	0.85	0.5629	0.8178	0.8489	0.05895	0.4625
V3708	0.83	0.554	0.7772	0.8312	0.04545	0.3734
V2597	0.83	0.5	0.8024	0.8373	0.04677	0.6737
V2856	0.87	0.5	0.8557	0.8705	0.04507	0.7054
V3271	0.805	0.5	0.7784	0.8088	0.06241	0.6218
V8749	0.825	0.5	0.7869	0.8243	0.04254	0.2354
V7899	0.845	0.6347	0.8279	0.8458	0.04885	0.8539
V1202	0.87	0.5223	0.8218	0.8718	0.04967	0.5049
V5995	0.83	0.5637	0.8129	0.8275	0.0459	0.3247
V1529	0.83	0.6526	0.7971	0.8312	0.02992	0.6396

	z.IDI	z.NRI	Frequency
V2804	8.653	9.036	0.4
V2134	8.496	8.988	0.1
V7976	7.633	6.311	0.2
V8405	7.577	6.619	0.1
V7856	7.478	8.175	0.1
V7217	7.267	7.043	0.1
V3365	7.011	9.483	1
V8685	7.003	7.49	0.7
V33	6.781	9.486	0.1
V7136	6.547	7.747	0.2
V2747	6.524	7.818	0.3
V6487	6.23	9.213	0.1
V7195	6.114	6.781	0.1
V3725	6.056	4.879	0.4
V2294	6.011	8.291	0.5
V4070	5.972	7.096	0.4
V5417	5.947	4.439	0.8
V9678	5.946	3.834	0.1
V2556	5.874	7.737	0.8
V9275	5.823	6.873	1
V9215	5.823	7.214	0.9
V9585	5.803	8.085	0.9
V6408	5.778	8.175	0.1
V8055	5.773	5.188	0.9
V4414	5.729	9.133	0.1
V5	5.701	5.543	0.6
V6511	5.692	3.967	0.1
V2256	5.641	6.343	0.8
V2297	5.628	6.363	0.1
V6146	5.611	4.936	0.5
V2242	5.586	5.578	0.1
V872	5.523	7.541	0.7
V5761	5.521	4.414	1
V6508	5.521	6.561	0.4
V6959	5.45	6.432	0.8
V5391	5.443	5.869	0.5
V8156	5.425	5.727	0.8
V533	5.358	8.432	0.4
V7272	5.35	9.055	0.9
V1967	5.347	6.781	0.1
V9635	5.344	5.015	0.4
V9965	5.329	5.209	0.7
V7891	5.309	7.161	0.9
V1142	5.303	3.702	0.1
V8502	5.267	6.366	0.8
V7319	5.262	5.526	0.6
V4584	5.252	6.623	1
V434	5.248	9.009	0.1
V1787	5.242	5.893	0.7
V6594	5.229	6.73	0.4
V9947	5.213	5.155	0.1
V3557	5.185	6.295	0.4
V729	5.175	6.859	0.8
V8517	5.166	6.425	0.6
V3826	5.154	5.889	0.1
V1975	5.124	6.396	0.9
V4290	5.115	7.502	0.9
V9395	5.081	6.793	0.9
V7748	5.079	8.795	1
V9432	5.077	5.288	0.5
V7928	5.008	7.762	0.3
V3222	4.981	5.713	0.3
V6958	4.973	6.947	0.1
V5489	4.92	6.383	0.1
V6163	4.908	6.833	0.9
V34	4.907	4.81	0.6
V5801	4.896	5.836	0.9
V5400	4.87	7.22	0.7
V3082	4.848	8.967	0.2
V5473	4.845	6.955	0.7
V7857	4.832	7.445	0.9
V4194	4.814	8.622	0.9
V7062	4.792	5.733	0.1
V6584	4.785	5.19	0.9
V5378	4.782	5.229	0.5
V4069	4.779	5.052	0.7
V9818	4.767	6.597	1
V7513	4.738	7.163	0.4
V8245	4.735	6.402	0.2
V3152	4.719	7.401	0.1
V6121	4.704	6.942	0.3
V782	4.698	6.981	0.7
V1451	4.697	6.005	0.1
V9402	4.696	4.508	0.1
V3339	4.688	7.107	0.1
V6688	4.682	5.427	1
V8585	4.584	5.193	0.3
V8972	4.578	5.803	0.1
V2783	4.538	4.377	0.3
V1198	4.534	4.515	0.2
V6685	4.521	6.04	0.3
V6141	4.509	7.134	0.1
V86	4.502	6.958	0.9
V6180	4.502	5.051	0.1
V8267	4.48	6.186	0.2
V9213	4.475	6.92	0.5
V6239	4.44	7.639	0.6
V1046	4.438	6.139	0.8
V1943	4.416	3.113	0.1
V7033	4.408	7.459	0.2
V6164	4.367	2.366	0.2
V4285	4.364	2.105	0.1
V2515	4.349	5.133	0.6
V3652	4.34	3.629	0.1
V3206	4.329	7.499	0.8
V819	4.311	3.432	0.2
V256	4.288	4.754	0.9
V8193	4.28	8.357	0.3
V5721	4.254	6.151	0.3
V3189	4.24	7.882	0.1
V7220	4.23	5.779	0.7
V8038	4.228	5.99	0.2
V723	4.226	6.45	0.3
V3545	4.217	4.065	0.3
V3161	4.212	5.968	0.9
V1248	4.206	4.152	1
V6780	4.176	4.793	0.2
V4017	4.167	5.017	0.6
V7302	4.149	5.384	0.3
V2750	4.148	5.141	0.4
V5005	4.144	5.935	1
V4406	4.122	4.245	0.4
V4554	4.116	4.449	0.2
V130	4.093	6.903	0.6
V9329	4.081	4.423	0.2
V4301	4.078	4.94	0.8
V4185	4.065	4.053	0.6
V9117	4.041	4.635	0.1
V9031	4.018	3.854	0.3
V2438	3.989	2.491	0.3
V1512	3.972	4.014	0.1
V311	3.956	3.644	0.3
V3170	3.955	6.884	0.9
V5774	3.949	2.85	0.4
V3125	3.93	5.219	0.1
V762	3.914	3.367	0.3
V9070	3.912	4.621	0.4
V3592	3.907	4.324	0.1
V9735	3.889	4.481	0.6
V4900	3.885	6.484	0.2
V6111	3.881	3.247	0.1
V4580	3.875	5.75	0.5
V3591	3.869	3.499	0.1
V7315	3.829	8.963	0.1
V1184	3.821	4.986	1
V7435	3.787	3.826	0.1
V1936	3.768	5.037	0.8
V1715	3.751	5.377	0.1
V4198	3.75	7.022	0.2
V306	3.743	4.432	0.1
V2657	3.724	6.103	0.1
V8650	3.723	4.102	0.2
V188	3.713	6.667	0.1
V1293	3.707	6.438	0.1
V5148	3.692	5.433	0.1
V4564	3.669	4.626	1
V630	3.669	8.427	0.1
V6123	3.668	6.248	0.1
V8428	3.648	3.92	0.3
V5321	3.638	6.985	1
V7446	3.633	1.681	0.1
V1336	3.626	2.373	0.2
V662	3.607	6.847	0.3
V2358	3.603	4.488	0.8
V8611	3.592	3.984	0.1
V7402	3.573	6.598	0.3
V4738	3.572	7.923	0.1
V9704	3.53	3.215	0.2
V4352	3.526	7.023	1
V5672	3.479	3.449	0.8
V3708	3.443	3.087	0.1
V2597	3.436	7.258	0.1
V2856	3.39	5.645	0.2
V3271	3.325	4.7	0.1
V8749	3.311	1.763	0.1
V7899	3.242	7.585	0.1
V1202	3.14	4.133	0.1
V5995	2.841	2.612	0.1
V1529	2.671	5.744	0.1

Accuracy: 0.925
tAUC: 0.9269
sensitivity: 0.9432
specificity: 0.9107
bootstrap:



hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")] 

cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)

Features (continued below)
	controlMean	controlStd	caseMean	caseStd	ROCAUC
V5005	239.3	83.62	314.7	72.85	0.772
V312	121.6	94.73	47.18	48.01	0.7499
V1936	243.1	79.21	316	79.48	0.748
V2556	255.3	92.59	324.8	64.88	0.7475
V9735	377.8	135	484	95.31	0.7453
V5801	385	144.3	503.2	107	0.7443
V7891	233.4	94.59	307.9	70.05	0.7441
V4070	234.6	95.72	311.7	81.57	0.7383
V6594	210.4	98.05	288.8	76.94	0.7377
V2640	221.1	182.6	358	204.6	0.7359
V5	26.58	30.65	4.932	14.67	0.734
V5802	370.1	128.1	463.9	93.28	0.7323
V1476	31.02	28.2	8.773	16.77	0.7307
V4290	36.72	31.4	12.12	20.9	0.7294
V7748	32.71	29.2	9.5	17.81	0.7284
V698	34.61	30.34	10.58	19.17	0.7281
V1975	26.74	30.6	4.716	14.5	0.7274
V471	114.8	113.9	213.1	128.8	0.7273
V533	25.49	31.11	4.33	14.07	0.7273
V2294	276.6	92.11	338.8	62.78	0.7254
V436	29.61	27.29	8.352	16.02	0.7252
V4973	28.23	32.2	4.75	14.45	0.7237
V1400	256	124.4	377.1	166.8	0.7226
V5473	41.37	33.29	16.44	25.3	0.7225
V86	28.55	26.42	8.114	15.47	0.7206
V2644	238.2	135.1	346.1	113.9	0.7203
V3629	222	125.3	348.8	172	0.7197
V7734	246.9	125.5	342.7	110.9	0.7188
V7655	35.48	30.77	13.34	20.64	0.718
V7857	27.68	25.6	7.989	15.11	0.7178

	WilcoxRes.p	Frequency
V5005	0	0.6267
V312	0	0.28
V1936	0	0.4867
V2556	0	0.4567
V9735	0	0.3167
V5801	0	0.1833
V7891	0	0.2
V4070	0	0.47
V6594	0	0.1033
V2640	0	0.13
V5	0	0.2833
V5802	0	0.1767
V1476	0	0.1167
V4290	0	0.1433
V7748	0	0.16
V698	0	0.1867
V1975	0	0.1533
V471	0	0.11
V533	0	0.1633
V2294	0	0.26
V436	0	0.1033
V4973	0	0.1267
V1400	0	0.2433
V5473	0	0.1233
V86	0	0.3267
V2644	0	0.1333
V3629	0	0.1833
V7734	0	0.1133
V7655	0	0.1333
V7857	0	0.1367

ARCENE Benchmarks

José Tamez-Peña

Sep 28, 2018

1 FRESA.CAD ARCENE Benchmark

1.1 ARCENE

1.2 Benchmarking

1.3 Results

1.3.1 Classifier Results

1.3.2 Radar Plots

1.3.3 Feature Analysis