1 FRESA.CAD Benchmark

1.1 Lymphoma Cancer Data Set


DLBCL_Lymohoma_A <- read.delim("./DLBCL/DLBCL_Lymohoma_A.txt")
#DLBCL_Lymohoma_A <- read.delim("./DLBCL_Lymohoma_A.txt")

Lymphoma <- DLBCL_Lymohoma_A[,-c(1,3,4)]
rownames(Lymphoma) <- DLBCL_Lymohoma_A[,1]

ExperimentName <- "Lymphoma"
bswimsReps <- 20;
theData <- Lymphoma;
theOutcome <- "Class";
reps <- 75;
fraction <- 0.8;
theData[,1:ncol(theData)] <- sapply(theData,as.numeric)

BSWiMSFileName <- paste(ExperimentName,"FRESAMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

1.2 Benchmarking



BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = bswimsReps)

save(BSWiMSMODEL,file = BSWiMSFileName)

load(file = BSWiMSFileName)

par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

save(cp,file = CVFileName)
par(mfrow = c(1,1),cex=1.0);


load(file = CVFileName)

1.3 Results

1.3.1 Classifier Results


hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


#The Times
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
18.11	33.83	3.882	0.7125	0.0164	0.018	56.56

learningTime <- -1*cp$cpuElapsedTimes
par(mfrow = c(2,1),cex=1.0);
pr <- plot(cp)

par(mfrow = c(1,1));

1.3.2 Radar Plots

op <- par(no.readonly = TRUE)

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

mNames <- names(cp$cpuElapsedTimes)

classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")

classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filnames <- c("BSWiMS","LASSO","RF.ref","IDI","t-test","Kendall","mRMR")

filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));

filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.6,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


detach("package:fmsb", unload=TRUE)

par(mfrow = c(1,1))
par(op)

1.3.3 Feature Analysis



rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(selFrequency,trace = "none",mar = c(10,10),main = "Features",cexRow = 0.5)



topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network")

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

coefficients:

Table continues below
	Estimate	lower	OR	upper	u.Accuracy
U28386_at	0.09328	1.087	1.098	1.108	0.8197
D55716_at	0.0003205	1	1	1	0.8618
X02152_at	2.333e-05	1	1	1	0.8719
HG4074.HT4344_at	0.0002645	1	1	1	0.8495
HG1980.HT2023_at	0.00272	1.002	1.003	1.003	0.8334
M63835_at	0.1114	1.101	1.118	1.135	0.8471
M63138_at	0.01215	1.01	1.012	1.014	0.825
X01060_at	0.003381	1.003	1.003	1.004	0.8033
X56494_at	0.002457	1.002	1.002	1.003	0.8338
D82348_at	0.00542	1.005	1.005	1.006	0.8313
M14328_s_at	1.572e-05	1	1	1	0.8308
X17620_at	0.0001138	1	1	1	0.827
J03909_at	7.186e-05	1	1	1	0.8366
M57710_at	0.000157	1	1	1	0.8044
U14518_at	0.1204	1.107	1.128	1.15	0.8259
D79997_at	0.000536	1	1.001	1.001	0.7994
L17131_rna1_at	1.81e-05	1	1	1	0.8043
M13792_at	6.623e-05	1	1	1	0.809
X16396_at	0.005733	1.005	1.006	1.007	0.8017
M20471_at	9.661e-06	1	1	1	0.7709
X62078_at	7.913e-05	1	1	1	0.7973
L25876_at	0.02112	1.017	1.021	1.025	0.7684
L33842_rna1_at	0.0001454	1	1	1	0.803
HG417.HT417_s_at	1.874e-05	1	1	1	0.7788
M60830_at	-0.1237	0.8611	0.8836	0.9068	0.6141
U72342_at	-0.01106	0.9864	0.989	0.9916	0.6166
V00594_s_at	1.017e-05	1	1	1	0.788
U46006_s_at	-0.02968	0.9636	0.9708	0.978	0.7854
M94880_f_at	-0.007183	0.991	0.9928	0.9946	0.6853
D78134_at	-0.01851	0.9771	0.9817	0.9863	0.7751
Z21966_at	-0.07024	0.9154	0.9322	0.9492	0.8291
X16983_at	-0.03717	0.954	0.9635	0.9731	0.7912
Z35227_at	-0.002803	0.9965	0.9972	0.998	0.7213
D87119_at	-0.001772	0.9977	0.9982	0.9987	0.797
D83597_at	-0.002808	0.9964	0.9972	0.998	0.7347

Table continues below
	r.Accuracy	full.Accuracy	u.AUC	r.AUC	full.AUC
U28386_at	0.7376	0.9377	0.8254	0.6841	0.9437
D55716_at	0.7532	0.8618	0.8696	0.5	0.8696
X02152_at	0.7532	0.8719	0.897	0.5	0.897
HG4074.HT4344_at	0.7532	0.8495	0.8776	0.5	0.8776
HG1980.HT2023_at	0.7606	0.8786	0.8565	0.6003	0.8931
M63835_at	0.74	0.8712	0.8799	0.5431	0.8996
M63138_at	0.72	0.9185	0.8492	0.6534	0.9324
X01060_at	0.7471	0.8096	0.84	0.5053	0.8448
X56494_at	0.7654	0.876	0.8285	0.5983	0.875
D82348_at	0.7675	0.8841	0.8407	0.6103	0.8938
M14328_s_at	0.7532	0.8308	0.822	0.5	0.822
X17620_at	0.7511	0.8386	0.838	0.5231	0.8499
J03909_at	0.7509	0.8412	0.862	0.5107	0.8653
M57710_at	0.736	0.835	0.8433	0.5531	0.8673
U14518_at	0.7813	0.8766	0.8468	0.6349	0.8903
D79997_at	0.7534	0.8134	0.8573	0.5255	0.8647
L17131_rna1_at	0.7514	0.8191	0.817	0.5275	0.8306
M13792_at	0.7532	0.809	0.8451	0.5	0.8451
X16396_at	0.7586	0.8661	0.8155	0.6166	0.8787
M20471_at	0.743	0.8534	0.8138	0.6108	0.8787
X62078_at	0.7737	0.8613	0.8181	0.638	0.8751
L25876_at	0.7609	0.9439	0.8096	0.772	0.9524
L33842_rna1_at	0.7615	0.8216	0.8315	0.5503	0.8452
HG417.HT417_s_at	0.7685	0.8375	0.789	0.6102	0.8441
M60830_at	0.8219	0.9944	0.5972	0.8135	0.9963
U72342_at	0.8114	0.9629	0.6006	0.8318	0.9669
V00594_s_at	0.7668	0.9282	0.8405	0.7654	0.9426
U46006_s_at	0.8015	0.9579	0.7885	0.8109	0.9679
M94880_f_at	0.8249	0.9596	0.714	0.8551	0.9687
D78134_at	0.8271	0.9632	0.7907	0.8412	0.9656
Z21966_at	0.8124	0.9347	0.845	0.805	0.9392
X16983_at	0.8222	0.957	0.7745	0.8296	0.9652
Z35227_at	0.7751	0.9221	0.701	0.7893	0.9282
D87119_at	0.8178	0.9362	0.7753	0.8255	0.9435
D83597_at	0.8002	0.9343	0.7148	0.8255	0.9365

	IDI	NRI	z.IDI	z.NRI	Frequency
U28386_at	0.6654	1.796	19.11	Inf	1
D55716_at	0.7236	1.531	17.35	13.56	1
X02152_at	0.6466	1.597	14.98	14.98	1
HG4074.HT4344_at	0.6439	1.518	14.62	13.29	1
HG1980.HT2023_at	0.6169	1.601	14.12	Inf	1
M63835_at	0.618	1.576	14.11	Inf	1
M63138_at	0.5851	1.697	13.5	Inf	1
X01060_at	0.576	1.375	13.04	6.42e+305	1
X56494_at	0.5675	1.5	12.8	Inf	1
D82348_at	0.5657	1.543	12.69	Inf	1
M14328_s_at	0.5768	1.313	12.66	9.91	1
X17620_at	0.5698	1.428	12.6	11.85	0.9
J03909_at	0.5584	1.49	12.52	6.42e+305	1
M57710_at	0.5586	1.468	12.34	13.58	1
U14518_at	0.5521	1.578	12.28	Inf	1
D79997_at	0.5467	1.449	11.89	12.57	1
L17131_rna1_at	0.549	1.331	11.83	10.21	0.4
M13792_at	0.5372	1.397	11.69	10.99	0.85
X16396_at	0.5246	1.478	11.66	Inf	0.55
M20471_at	0.5074	1.48	11.03	14.16	0.1
X62078_at	0.502	1.534	11.01	16.37	0.2
L25876_at	0.4648	1.574	10.85	Inf	1
L33842_rna1_at	0.4999	1.406	10.85	11.55	0.55
HG417.HT417_s_at	0.479	1.315	10.26	10.98	0.25
M60830_at	0.4322	2	9.383	Inf	0.1
U72342_at	0.3791	1.839	8.364	Inf	0.15
V00594_s_at	0.3472	1.579	8.173	15.35	0.35
U46006_s_at	0.3337	1.67	7.812	Inf	0.15
M94880_f_at	0.3353	1.832	7.753	Inf	1
D78134_at	0.3462	1.828	7.728	Inf	0.95
Z21966_at	0.3331	1.734	7.583	Inf	0.45
X16983_at	0.3102	1.562	7.346	Inf	0.55
Z35227_at	0.326	1.593	7.304	Inf	0.65
D87119_at	0.3027	1.607	7.097	Inf	0.5
D83597_at	0.2885	1.588	6.858	Inf	0.35

Accuracy: 0.974
tAUC: 0.9828
sensitivity: 0.9655
specificity: 1
bootstrap:



hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")]

100 : X65867_at


cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)

Features (continued below)
	controlMean	controlStd	caseMean	caseStd	ROCAUC
D55716_at	616	242.2	2040	1404	0.9655
HG4074.HT4344_at	283.3	195.1	1397	1117	0.9428
M63835_at	-259.3	213	478.7	762.2	0.9383
X02152_at	3182	1552	10785	4792	0.9283
D82348_at	786.9	652.9	2923	1683	0.9283
M14328_s_at	4807	2440	11842	3903	0.9229
U14518_at	89.47	94.07	322.9	189	0.9224
X56494_at	2418	1270	7903	4091	0.9211
HG1980.HT2023_at	1845	882.5	6680	4117	0.9201
X62078_at	403.2	535.9	2299	1602	0.9183
U28386_at	437.6	324.7	1847	1263	0.9138
X17620_at	1041	629.7	3357	2033	0.912
M63138_at	2736	1101	7142	3569	0.9093
X01060_at	484.4	218	2327	2077	0.9083
HG417.HT417_s_at	3115	1590	9939	5682	0.9083
M13792_at	974.9	422.2	2909	2188	0.9065
L17131_rna1_at	2437	1408	7290	3390	0.9056
X16396_at	458.1	447.5	1827	1133	0.9029
D31887_at	94.47	238	648	514.5	0.9025
X69433_at	584.8	349	1672	1016	0.8988
D13633_at	108.5	94.2	396.8	259.1	0.8984
D79997_at	24.79	88.78	534.4	497.8	0.8975
HG2279.HT2375_at	3406	1399	7706	3226	0.8956
L25876_at	350.1	165.3	1174	845.2	0.8943
M57710_at	1317	558.8	5526	3684	0.8929
U29680_at	889.4	396.5	2945	2063	0.8929
L33842_rna1_at	983.4	359.8	2489	1338	0.8907
Z21966_at	248.9	106.6	59.79	133.1	0.8902
J03909_at	1972	906.5	6983	3983	0.8893
U23143_at	402.9	160.8	870.4	500.3	0.8893

	WilcoxRes.p	Frequency
D55716_at	0	0.7133
HG4074.HT4344_at	0	0.312
M63835_at	0	0.5307
X02152_at	0	0.5093
D82348_at	0	0.3027
M14328_s_at	0	0.364
U14518_at	0	0.5147
X56494_at	0	0.2413
HG1980.HT2023_at	0	0.184
X62078_at	0	0.12
U28386_at	0	0.104
X17620_at	0	0.1533
M63138_at	0	0.6613
X01060_at	0	0.1853
HG417.HT417_s_at	0	0.208
M13792_at	0	0.564
L17131_rna1_at	0	0.2133
X16396_at	0	0.1093
D31887_at	0	0.412
X69433_at	0	0.384
D13633_at	0	0.6267
D79997_at	0	0.276
HG2279.HT2375_at	0	0.1907
L25876_at	0	0.3387
M57710_at	0	0.576
U29680_at	0	0.5067
L33842_rna1_at	0	0.6253
Z21966_at	0	0.756
J03909_at	0	0.5533
U23143_at	0	0.2987

Benchmarks for Lymphoma Cancer Data Set

José Tamez-Peña

Sep. 28, 2018

1 FRESA.CAD Benchmark

1.1 Lymphoma Cancer Data Set

1.2 Benchmarking

1.3 Results

1.3.1 Classifier Results

1.3.2 Radar Plots

1.3.3 Feature Analysis