1 FRESA.CAD Benchmark

1.1 Leukemia Cancer Data Set


LeukemiaA <- read.delim("./Leukemia/Leukemia.txt")
#LeukemiaA <- read.delim("./Leukemia.txt")
Leukemia <- LeukemiaA[,-1]
rownames(Leukemia) <- LeukemiaA[,1]


ExperimentName <- "Leukemia"
bswimsReps <- 20;
theData <- Leukemia;
theOutcome <- "Status";
reps <- 75;
fraction <- 0.8;

theData[,1:ncol(theData)] <- sapply(theData,as.numeric)

BSWiMSFileName <- paste(ExperimentName,"FRESAMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

1.2 Benchmarking



BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = bswimsReps)
save(BSWiMSMODEL,file = BSWiMSFileName)

load(file = BSWiMSFileName)

par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

par(mfrow = c(1,1),cex=1.0);

save(cp,file = CVFileName)

load(file = CVFileName)

1.3 Results

1.3.1 Classifier Results


hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


#The Times
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
11.13	36.38	4.249	0.7745	0.01707	0.01307	52.57

learningTime <- -1*cp$cpuElapsedTimes

par(mfrow = c(2,1));
pr <- plot(cp)

par(mfrow = c(1,1));

1.3.2 Radar Plots

op <- par(no.readonly = TRUE)

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

mNames <- names(cp$cpuElapsedTimes)

classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")

classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filnames <- c("BSWiMS","LASSO","RF.ref","IDI","t-test","Kendall","mRMR")

filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));

filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.6,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


detach("package:fmsb", unload=TRUE)

par(mfrow = c(1,1))
par(op)

1.3.3 Feature Analysis



rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(selFrequency,trace = "none",mar = c(10,10),main = "Features",cexRow = 0.5)



topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network")

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

coefficients:

Table continues below
	Estimate	lower	OR	upper	u.Accuracy
M23197_at	0.02094	1.021	1.021	1.021	0.9405
M11722_at	-0.0005205	0.9995	0.9995	0.9995	0.9
U46499_at	0.006506	1.007	1.007	1.007	0.9264
M27891_at	0.0002253	1	1	1	0.94
X95735_at	0.0002531	1	1	1	0.9456
M31523_at	-0.000957	0.999	0.999	0.9991	0.9259
M63379_at	0.03053	1.027	1.031	1.035	0.8694
L09209_s_at	0.007382	1.007	1.007	1.008	0.909
M92287_at	-0.0001924	0.9998	0.9998	0.9998	0.8912
M84526_at	0.0002038	1	1	1	0.9417
D88422_at	0.000485	1	1	1.001	0.912
U05259_rna1_at	-0.0005299	0.9994	0.9995	0.9996	0.8444
J05243_at	-0.09255	0.898	0.9116	0.9254	0.8711
X62654_rna1_at	0.006698	1.005	1.007	1.008	0.876
D88270_at	-3.932e-05	1	1	1	0.8403
HG1612.HT1612_at	-0.003471	0.9958	0.9965	0.9973	0.822
M62982_at	0.06068	1.047	1.063	1.078	0.5926
X51521_at	-0.000733	0.9991	0.9993	0.9994	0.7639
Y07604_at	0.007428	1.006	1.007	1.009	0.8206
X59417_at	-0.0006307	0.9992	0.9994	0.9995	0.8755
U16954_at	-0.05569	0.9321	0.9458	0.9598	0.7778
X68560_at	-0.02605	0.9676	0.9743	0.981	0.7216
M13690_s_at	0.01347	1.01	1.014	1.017	0.7812
U09578_at	0.03882	1.027	1.04	1.053	0.7639

Table continues below
	r.Accuracy	full.Accuracy	u.AUC	r.AUC	full.AUC
M23197_at	0.3472	0.9405	0.9352	0.5	0.9352
M11722_at	0.3472	0.9	0.914	0.5	0.914
U46499_at	0.3472	0.9264	0.9152	0.5	0.9152
M27891_at	0.3472	0.94	0.9371	0.5	0.9371
X95735_at	0.3472	0.9456	0.9412	0.5	0.9412
M31523_at	0.3472	0.9259	0.9319	0.5	0.9319
M63379_at	0.4944	0.9444	0.8582	0.5522	0.9387
L09209_s_at	0.4398	0.9329	0.8996	0.5581	0.9263
M92287_at	0.3472	0.8912	0.9008	0.5	0.9008
M84526_at	0.3472	0.9417	0.918	0.5	0.918
D88422_at	0.3472	0.912	0.896	0.5	0.896
U05259_rna1_at	0.5194	0.8991	0.8696	0.6157	0.914
J05243_at	0.5486	0.9151	0.8841	0.6337	0.9233
X62654_rna1_at	0.7205	0.9491	0.8697	0.771	0.9479
D88270_at	0.5625	0.8935	0.8777	0.6462	0.9138
HG1612.HT1612_at	0.7668	0.9394	0.8287	0.7718	0.9456
M62982_at	0.8704	0.9954	0.587	0.8612	0.9954
X51521_at	0.4352	0.8009	0.7812	0.5617	0.8153
Y07604_at	0.6291	0.8976	0.8251	0.6936	0.8993
X59417_at	0.7671	0.9579	0.894	0.7942	0.9624
U16954_at	0.8307	0.9517	0.7892	0.8462	0.9528
X68560_at	0.7454	0.9233	0.7404	0.7809	0.9314
M13690_s_at	0.7633	0.9236	0.7712	0.7895	0.9294
U09578_at	0.8773	0.9861	0.7505	0.8842	0.9878

	IDI	NRI	z.IDI	z.NRI	Frequency
M23197_at	0.835	1.752	99262102	Inf	1
M11722_at	0.7371	1.682	22600307	1.498e+306	1
U46499_at	0.6978	1.686	11563925	Inf	1
M27891_at	0.8164	1.765	21.93	20.32	1
X95735_at	0.8067	1.78	21.06	21.37	1
M31523_at	0.8023	1.726	20.43	18.36	1
M63379_at	0.7148	1.765	17.3	Inf	0.25
L09209_s_at	0.7003	1.709	16.46	Inf	1
M92287_at	0.717	1.618	16.22	14.34	1
M84526_at	0.7123	1.656	15.88	15.84	1
D88422_at	0.7122	1.588	15.86	14.05	1
U05259_rna1_at	0.5823	1.667	12.09	23.74	0.25
J05243_at	0.5724	1.67	12.08	Inf	0.6
X62654_rna1_at	0.4229	1.66	9.312	Inf	0.95
D88270_at	0.4438	1.433	9.185	11.31	0.1
HG1612.HT1612_at	0.4479	1.742	9.113	Inf	0.55
M62982_at	0.4257	1.872	8.302	41.2	0.15
X51521_at	0.4161	1.281	8.126	Inf	0.3
Y07604_at	0.3738	1.582	7.668	18.23	0.4
X59417_at	0.3533	1.635	7.647	2.996e+306	0.5
U16954_at	0.3663	1.475	7.44	Inf	0.35
X68560_at	0.3469	1.703	7.401	Inf	0.35
M13690_s_at	0.3344	1.571	6.966	Inf	0.4
U09578_at	0.2862	1.929	6.092	1.498e+307	0.1

Accuracy: 1
tAUC: 1
sensitivity: 1
specificity: 1
bootstrap:



hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")] 

cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)

Features (continued below)
	controlMean	controlStd	caseMean	caseStd	ROCAUC
M23197_at	174.7	84.97	872.8	523.3	0.9889
X95735_at	378.1	532.7	3286	1682	0.9787
M27891_at	185.5	451.9	9120	7377	0.9779
M31523_at	1402	834.7	313.4	133.7	0.977
U46499_at	143.8	464.4	1333	741.7	0.9715
L09209_s_at	541.9	606	2691	1545	0.9702
D88422_at	150.6	129.3	2790	3002	0.954
J05243_at	806.5	515.9	165.6	155.7	0.9506
M92287_at	4255	2465	975.4	466.7	0.9481
M63138_at	1672	722.6	5609	3228	0.9472
X62654_rna1_at	641.9	416.5	2226	1549	0.9464
M11722_at	3936	3192	182.4	307.1	0.9438
M63379_at	315.6	111.4	1595	2290	0.934
M84526_at	-137.7	177.1	5125	4234	0.9315
M31211_s_at	512	252.3	159.6	121.4	0.9302
X61587_at	331.1	548.3	1778	1111	0.9298
M96326_rna1_at	657.3	601.7	8311	8056	0.9277
M16038_at	371.4	267	1536	896.9	0.9268
X59417_at	4199	2259	1112	545.5	0.9234
Z15115_at	3903	2266	1489	585.8	0.92
M19507_at	443	784.6	8605	8301	0.92
X62320_at	744.7	333.4	3648	3539	0.9166
HG1612.HT1612_at	3768	1903	1369	907.8	0.9166
M83652_s_at	53.09	271.1	1374	1384	0.9149
X17042_at	1717	1738	6495	2751	0.9149
M22960_at	456.8	354.5	1802	1041	0.9119
U05259_rna1_at	4161	2997	411.6	389.7	0.9106
L47738_at	1011	761.8	242.5	201.6	0.9106
M55150_at	740.9	343.1	1595	578.6	0.9013
U16954_at	348	569.4	-50.64	102.3	0.9004

	WilcoxRes.p	Frequency
M23197_at	0	0.872
X95735_at	0	0.6627
M27891_at	0	0.656
M31523_at	0	0.664
U46499_at	0	0.8
L09209_s_at	0	0.3307
D88422_at	0	0.4027
J05243_at	0	0.6987
M92287_at	0	0.7587
M63138_at	0	0.3773
X62654_rna1_at	0	0.308
M11722_at	0	0.78
M63379_at	0	0.3787
M84526_at	0	0.7067
M31211_s_at	0	0.4973
X61587_at	0	0.1787
M96326_rna1_at	0	0.448
M16038_at	0	0.5307
X59417_at	0	0.6907
Z15115_at	0	0.2573
M19507_at	0	0.536
X62320_at	0	0.2413
HG1612.HT1612_at	0	0.5547
M83652_s_at	0	0.4427
X17042_at	0	0.62
M22960_at	0	0.4533
U05259_rna1_at	0	0.62
L47738_at	0	0.392
M55150_at	0	0.56
U16954_at	0	0.2547

Benchmarks for Leukemia Cancer Data Set

José Tamez-Peña

Sep 28, 2018

1 FRESA.CAD Benchmark

1.1 Leukemia Cancer Data Set

1.2 Benchmarking

1.3 Results

1.3.1 Classifier Results

1.3.2 Radar Plots

1.3.3 Feature Analysis