1 FRESA.CAD Benchmark

1.1 Colon Cancer Data Set



# Load the colon data form the rda package
data(colon,package = "rda")
#FRESA.CAD requires a data frame. One of the columns must be the class
Colon <- as.data.frame(cbind(Class = colon.y, colon.x))
#The class should be 0 for controls and 1 for cases
Colon$Class <- Colon$Class - 1


ExperimentName <- "Colon"
bswimsReps <- 20;
theData <- Colon;
theOutcome <- "Class";
reps <- 75;
fraction <- 0.8;
theData[,1:ncol(theData)] <- sapply(theData,as.numeric)


BSWiMSFileName <- paste(ExperimentName,"BSWIMSMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

1.2 Benchmarking


BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = 1)


save(BSWiMSMODEL,file = BSWiMSFileName)

load(file = BSWiMSFileName)

par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

save(cp,file = CVFileName)
par(mfrow = c(1,1),cex=1.0);


load(file = CVFileName)

1.3 Results

1.3.1 Classifier Results


hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45)


#The Times
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
4.397	1.305	0.2892	0.1812	0.0056	0.0096	5.898

learningTime <- -1*cp$cpuElapsedTimes

pr <- plot(cp)

1.3.2 Radar Plots

op <- par(no.readonly = TRUE)

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

mNames <- names(cp$cpuElapsedTimes)

classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")

classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filnames <- c("BSWiMS","LASSO_MIN","RF","IDI","tStudent","kendall","mRMR.classic")

filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));

filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.6,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


detach("package:fmsb", unload=TRUE)

par(mfrow = c(1,1))
par(op)

1.3.3 Feature Analysis



rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(as.matrix(selFrequency),trace = "none",mar = c(10,10),main = "Features",cexRow = 0.25,cexCol = 0.5)



topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network",cexRow = 0.5,cexCol = 0.5)

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

coefficients:

Table continues below
	Estimate	lower	OR	upper	u.Accuracy	r.Accuracy
V494	-0.1099	0.8821	0.8959	0.9099	0.8226	0.6452
V378	-0.2057	0.7908	0.8141	0.8381	0.8333	0.6452
V1773	0.4673	1.481	1.596	1.719	0.828	0.6882
V514	0.1331	1.121	1.142	1.164	0.828	0.6452
V246	-0.09924	0.8929	0.9055	0.9183	0.8387	0.6452
V268	-0.2298	0.7693	0.7947	0.8209	0.8548	0.6344
V1424	-0.06845	0.9234	0.9338	0.9444	0.8118	0.6452
V898	-0.06184	0.9304	0.94	0.9498	0.8226	0.6452
V250	-0.0712	0.918	0.9313	0.9447	0.8333	0.6452
V1636	-0.2533	0.7287	0.7762	0.8269	0.8817	0.672
V1731	0.106	1.089	1.112	1.135	0.7742	0.6452
V67	-0.238	0.7507	0.7882	0.8276	0.7903	0.6344
V965	0.08831	1.066	1.092	1.119	0.8065	0.6452
V1111	0.5024	1.411	1.653	1.935	0.7419	0.8333
V626	0.2552	1.188	1.291	1.402	0.7742	0.7796
V813	-0.433	0.5626	0.6486	0.7478	0.8333	0.7419
V1211	-0.3072	0.6592	0.7355	0.8207	0.6882	0.828
V1885	-0.1881	0.7734	0.8285	0.8876	0.8065	0.7419
V1772	0.1991	1.132	1.22	1.315	0.8065	0.8333
V1550	0.1213	1.077	1.129	1.183	0.7258	0.8011
V740	-0.1646	0.7942	0.8483	0.906	0.7796	0.7742
V577	0.2862	1.186	1.331	1.495	0.672	0.8817
V138	-0.3505	0.6095	0.7043	0.814	0.7419	0.8065
V1347	0.2346	1.145	1.264	1.396	0.6344	0.8548
V766	-0.08942	0.8802	0.9145	0.95	0.8387	0.8118
V781	0.1088	1.061	1.115	1.171	0.8011	0.7258
V139	0.1519	1.078	1.164	1.257	0.8011	0.8548
V1844	-0.1053	0.853	0.9001	0.9498	0.8333	0.8065
V1583	0.1119	1.053	1.118	1.187	0.8548	0.8011
V1043	0.1425	1.068	1.153	1.246	0.8118	0.8387
V1994	0.149	1.066	1.161	1.264	0.6344	0.7903

Table continues below
	full.Accuracy	u.AUC	r.AUC	full.AUC	IDI	NRI
V494	0.8226	0.8352	0.5	0.8352	0.5924	1.467
V378	0.8333	0.8333	0.5	0.8333	0.6011	1.367
V1773	0.8925	0.8189	0.7038	0.8894	0.5652	1.65
V514	0.828	0.8428	0.5	0.8428	0.5179	1.467
V246	0.8387	0.8375	0.5	0.8375	0.5114	1.45
V268	0.871	0.8466	0.6178	0.8659	0.4945	1.5
V1424	0.8118	0.8167	0.5	0.8167	0.4932	1.283
V898	0.8226	0.842	0.5	0.842	0.4589	1.4
V250	0.8333	0.8231	0.5	0.8231	0.4404	1.117
V1636	0.8871	0.8879	0.6572	0.8955	0.4371	1.667
V1731	0.7742	0.7739	0.5	0.7739	0.423	1.2
V67	0.8172	0.7864	0.6348	0.8208	0.4036	1.3
V965	0.8065	0.8193	0.5	0.8193	0.3629	1.317
V1111	0.8387	0.7625	0.8265	0.8477	0.3407	1.567
V626	0.914	0.7807	0.7814	0.9163	0.2847	0.9833
V813	0.8387	0.8265	0.7625	0.8477	0.3228	1.55
V1211	0.8925	0.7038	0.8189	0.8894	0.2796	1.467
V1885	0.8925	0.7989	0.7591	0.8996	0.2406	1.233
V1772	0.8978	0.8125	0.8299	0.9004	0.2383	1.1
V1550	0.8656	0.7193	0.8049	0.8754	0.2116	1.067
V740	0.914	0.7814	0.7807	0.9163	0.204	1.2
V577	0.8871	0.6572	0.8879	0.8955	0.2159	1.05
V138	0.8925	0.7591	0.7989	0.8996	0.1948	1.05
V1347	0.871	0.6178	0.8466	0.8659	0.2056	0.9333
V766	0.8871	0.8239	0.8133	0.8886	0.1811	0.7333
V781	0.8656	0.8049	0.7193	0.8754	0.1806	0.9833
V139	0.8763	0.8049	0.8466	0.8837	0.1373	0.6833
V1844	0.8978	0.8299	0.8125	0.9004	0.1545	0.9333
V1583	0.8763	0.8466	0.8049	0.8837	0.1297	0.7167
V1043	0.8871	0.8133	0.8239	0.8886	0.1444	0.5833
V1994	0.8172	0.6348	0.7864	0.8208	0.1182	0.7333

	z.IDI	z.NRI	Frequency
V494	10.86	9.782	0.05
V378	10.56	8.425	0.05
V1773	10.34	13.42	0.05
V514	9.951	9.916	0.05
V246	9.569	9.49	0.05
V268	9.047	10.41	0.05
V1424	8.999	7.813	0.05
V898	8.97	9.536	0.05
V250	8.062	6.15	0.05
V1636	7.861	13.82	0.05
V1731	7.602	6.808	0.05
V67	7.17	8.011	0.05
V965	7.018	7.978	0.05
V1111	6.225	11.49	0.05
V626	5.983	5.905	0.05
V813	5.914	11	0.05
V1211	5.486	10.23	0.05
V1885	5.344	7.062	0.05
V1772	5.183	6.063	0.05
V1550	4.907	6.257	0.05
V740	4.846	7.309	0.05
V577	4.81	5.73	0.05
V138	4.72	5.626	0.05
V1347	4.555	5.072	0.05
V766	4.544	3.621	0.05
V781	4.27	5.537	0.05
V139	3.865	3.323	0.05
V1844	3.787	4.958	0.05
V1583	3.619	3.463	0.05
V1043	3.577	2.752	0.05
V1994	3.283	3.669	0.05

Accuracy: 0.9194
tAUC: 0.917
sensitivity: 0.925
specificity: 0.9091
bootstrap:



hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.5,srtCol = 45)


vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")]

100 : V610


cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)

Features (continued below)
	controlMean	controlStd	caseMean	caseStd	ROCAUC
V378	0.741	0.4201	-0.0097	0.3183	0.9341
V494	1.207	0.5516	0.0745	0.516	0.9205
V1636	-0.07	0.4481	-1.098	0.5804	0.9045
V1424	1.121	0.8332	-0.3798	0.8767	0.8909
V626	0.2895	0.5038	1.138	0.482	0.8898
V250	2.091	0.7986	0.7251	0.637	0.8886
V1772	-1.061	0.4778	-0.2618	0.5116	0.8852
V1043	-0.5278	0.3595	0.2742	0.6335	0.8795
V1844	0.3261	0.5442	-0.8701	0.8825	0.8761
V1773	-1.319	0.45	-0.6071	0.4575	0.8727
V514	0.6087	0.3459	1.367	0.5933	0.8648
V766	1.239	1.037	-0.0012	0.6195	0.8648
V740	1.026	0.6136	0.1875	0.6158	0.8648
V1885	-0.8892	0.5478	-1.622	0.4309	0.8602
V67	2.144	0.4965	1.444	0.4422	0.8568
V813	0.3412	0.397	-0.2195	0.3996	0.8568
V246	1.583	0.7959	0.5808	0.5056	0.8557
V268	1.596	0.8016	0.6233	0.5027	0.8557
V823	1.855	0.7811	0.5968	0.8911	0.8557
V1495	0.4171	0.8022	-0.8496	0.859	0.8557
V1583	-1.312	0.5631	-0.5262	0.518	0.8523
V139	1.312	0.3371	1.895	0.4572	0.8523
V898	1.226	0.6475	-0.0314	0.9014	0.8511
V965	-0.3606	0.4413	0.3658	0.5746	0.8511
V1731	-0.6813	0.569	0.1531	0.5689	0.85
V138	1.237	0.2782	0.8257	0.3015	0.8489
V825	0.2442	0.4281	-0.3768	0.4734	0.8443
V781	0.3102	0.5331	1.052	0.5429	0.8432
V1061	-0.5987	0.4012	0.0075	0.4601	0.8386
V392	0.4058	0.4568	1.024	0.4519	0.8386

	WilcoxRes.p	Frequency
V378	0	0.9013
V494	0	0.9173
V1636	0	0.8227
V1424	0	0.7027
V626	0	0.7427
V250	0	0.7613
V1772	0	0.7493
V1043	0	0.608
V1844	0	0.5773
V1773	0	0.836
V514	0	0.5947
V766	0	0.2533
V740	0	0.556
V1885	0	0.6853
V67	0	0.6653
V813	0	0.6107
V246	0	0.2667
V268	0	0.2213
V823	0	0.252
V1495	0	0.1467
V1583	0	0.7547
V139	0	0.2693
V898	0	0.7267
V965	0	0.4933
V1731	0	0.536
V138	0	0.5507
V825	0	0.1787
V781	0	0.4493
V1061	0	0.3307
V392	0	0.528

Benchmarks for Colon Cancer Data Set

José Tamez-Peña

Sep 4, 2019

1 FRESA.CAD Benchmark

1.1 Colon Cancer Data Set

1.2 Benchmarking

1.3 Results

1.3.1 Classifier Results

1.3.2 Radar Plots

1.3.3 Feature Analysis