Benchmarks for Colon Cancer Data Set
José Tamez-Peña
Sep 26, 2018
#FRESA.CAD Benchmark
##Colon Cancer Data Set
ColonData <- read.delim("./Colon/cancerColonb.txt")
Colon <- ColonData[,-1]
rownames(Colon) <- ColonData[,1]
colonC <- Colon
colonC$Class <- as.factor(Colon$Class)
ls <- LASSO_1SE(Class~.,Colon)
ExperimentName <- "Colon"
bswimsReps <- 20;
theData <- Colon;
theOutcome <- "Class";
reps <- 75;
fraction <- 0.8;
theData[,1:ncol(theData)] <- sapply(theData,as.numeric)
#sf <- univariate_Wilcoxon(data=theData, Outcome="Class",pvalue=0.5)
#theData <- theData[,c("Class",names(sf))]
BSWiMSFileName <- paste(ExperimentName,"BSWIMSMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")
##Benchmarking
BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = 1)
save(BSWiMSMODEL,file = BSWiMSFileName)
load(file = BSWiMSFileName)
par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)
save(cp,file = CVFileName)
par(mfrow = c(1,1),cex=1.0);
load(file = CVFileName)
##Results
###Classifier Results
hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45) 
#The Times
par(mfrow = c(2,1),cex=1.0);
pander::pander(cp$cpuElapsedTimes)| BSWiMS | RF | RPART | LASSO | SVM | KNN | ENS |
|---|---|---|---|---|---|---|
| 7.519 | 2.197 | 0.4525 | 0.3317 | 0.009333 | 0.02173 | 10.08 |
par(mfrow = c(1,1),cex=1.0);
learningTime <- -1*cp$cpuElapsedTimes
pr <- plot(cp)
0.0.1 Radar Plots
op <- par(no.readonly = TRUE)
library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))
mNames <- names(cp$cpuElapsedTimes)
classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")
classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU
colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex = 0.5 ,title = "Prediction Model")
legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)
filnames <- c("BSWiMS","LASSO_MIN","RF","IDI","tStudent","kendall","mRMR.classic")
filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));
filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));
filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE
colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex = 0.6,title = "Filter Method" )
legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)
detach("package:fmsb", unload=TRUE)
par(mfrow = c(1,1))
par(op)0.0.2 Feature Analysis
rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.1,]
gplots::heatmap.2(as.matrix(selFrequency),trace = "none",mar = c(10,10),main = "Features",cexRow = 0.25,cexCol = 0.5)
topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network",cexRow = 0.5,cexCol = 0.5)
pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))coefficients:
Table continues below Estimate lower OR upper u.Accuracy r.Accuracy U09564 0.003282 1.003 1.003 1.004 0.6935 0.6505 Z50753 -0.001931 0.9978 0.9981 0.9983 0.7473 0.5968 X14958 0.0007181 1.001 1.001 1.001 0.7473 0.629 M22382 0.0004268 1 1 1 0.7742 0.6613 T57619 0.0002193 1 1 1 0.6828 0.6613 H22579.1 0.003206 1.003 1.003 1.004 0.5968 0.7473 J02854 -0.0003617 0.9996 0.9996 0.9997 0.8011 0.6398 R08183 0.0003633 1 1 1 0.7258 0.6613 T92451 -0.0001153 0.9999 0.9999 0.9999 0.8011 0.629 T51261 0.0009718 1.001 1.001 1.001 0.672 0.7312 T47383 -0.002818 0.9964 0.9972 0.998 0.6505 0.6935 D16431 0.0005371 1 1.001 1.001 0.6398 0.8011 U30825 0.001474 1.001 1.001 1.002 0.672 0.7204 H64807 -0.003421 0.9955 0.9966 0.9977 0.6613 0.6828 H78386 -0.002665 0.9965 0.9973 0.9982 0.629 0.7473 U19969 -0.002472 0.9967 0.9975 0.9984 0.7312 0.672 T56604 0.0006288 1 1.001 1.001 0.6774 0.6935 M80815 -0.00102 0.9986 0.999 0.9993 0.7204 0.672 T61661 -0.0001551 0.9998 0.9998 0.9999 0.6613 0.7258 T71025 -0.0001133 0.9998 0.9999 0.9999 0.7581 0.7366 J04102 0.001628 1.001 1.002 1.002 0.629 0.8011 X12369 -0.0002573 0.9996 0.9997 0.9998 0.6935 0.6774 X63629 0.00242 1.002 1.002 1.003 0.7688 0.7903 M63391 -7.52e-05 0.9999 0.9999 1 0.8548 0.7688 M76378.1 -0.000138 0.9998 0.9999 0.9999 0.8118 0.7204 H43887 -0.0001683 0.9998 0.9998 0.9999 0.7473 0.7258 R87126 -0.0003646 0.9995 0.9996 0.9998 0.8333 0.7903 M76378.2 -0.0003254 0.9995 0.9997 0.9998 0.7903 0.7688 X12671 0.0002818 1 1 1 0.7204 0.8118 H08393 0.002445 1.001 1.002 1.004 0.7903 0.8333 H20709 -6.328e-05 0.9999 0.9999 1 0.6613 0.7742 H40095 0.0002385 1 1 1 0.7366 0.7581 R36977 0.0009141 1 1.001 1.001 0.7688 0.8548 T47377 0.0001731 1 1 1 0.7258 0.7473 M76378 -7.332e-05 0.9999 0.9999 1 0.8226 0.6613 J05032 0.0005921 1 1.001 1.001 0.6613 0.8226 Table continues below full.Accuracy u.AUC r.AUC full.AUC IDI NRI U09564 0.8763 0.7148 0.6167 0.8871 0.6291 1.7 Z50753 0.9086 0.7462 0.6295 0.9189 0.6093 1.567 X14958 0.8763 0.7564 0.6 0.8837 0.5319 1.567 M22382 0.8548 0.7841 0.6489 0.8636 0.5457 1.45 T57619 0.8172 0.6928 0.6352 0.814 0.5411 1.433 H22579.1 0.9086 0.6295 0.7462 0.9189 0.4839 1.55 J02854 0.8925 0.7742 0.6595 0.8928 0.4352 1.367 R08183 0.8387 0.7466 0.6455 0.8545 0.4322 1.467 T92451 0.8387 0.7777 0.6307 0.8443 0.3997 1.233 T51261 0.828 0.7015 0.7098 0.8462 0.3897 1.083 T47383 0.8763 0.6167 0.7148 0.8871 0.3733 1.433 D16431 0.8925 0.6595 0.7742 0.8928 0.3333 1.333 U30825 0.8333 0.6947 0.7015 0.8299 0.3535 1.167 H64807 0.8172 0.6352 0.6928 0.814 0.3141 1.133 H78386 0.8763 0.6 0.7564 0.8837 0.2727 1.233 U19969 0.828 0.7098 0.7015 0.8462 0.2897 1.267 T56604 0.8656 0.7057 0.667 0.8754 0.2668 1.1 M80815 0.8333 0.7015 0.6947 0.8299 0.2818 1.3 T61661 0.8387 0.6455 0.7466 0.8545 0.2618 1.117 T71025 0.8387 0.7614 0.7652 0.8443 0.2316 1.1 J04102 0.8387 0.6307 0.7777 0.8443 0.2416 1.133 X12369 0.8656 0.667 0.7057 0.8754 0.2212 0.9667 X63629 0.871 0.7799 0.7591 0.8727 0.2519 1.317 M63391 0.8925 0.8261 0.7833 0.8894 0.2519 1.2 M76378.1 0.8441 0.7723 0.7424 0.8417 0.2289 1.067 H43887 0.8226 0.7189 0.7466 0.825 0.2012 0.8667 R87126 0.8817 0.8231 0.817 0.8845 0.2061 1.217 M76378.2 0.871 0.7591 0.7799 0.8727 0.2111 1.183 X12671 0.8441 0.7424 0.7723 0.8417 0.2076 1 H08393 0.8817 0.817 0.8231 0.8845 0.2103 0.9833 H20709 0.8548 0.6489 0.7841 0.8636 0.1647 1.1 H40095 0.8387 0.7652 0.7614 0.8443 0.1704 0.9833 R36977 0.8925 0.7833 0.8261 0.8894 0.1682 0.65 T47377 0.8226 0.7466 0.7189 0.825 0.1725 0.7833 M76378 0.8333 0.7807 0.6898 0.8299 0.1444 0.55 J05032 0.8333 0.6898 0.7807 0.8299 0.1367 0.6333 z.IDI z.NRI Frequency U09564 12.04 14.88 0.05556 Z50753 11.09 11.96 0.05556 X14958 10.12 11.34 0.05556 M22382 9.895 9.52 0.05556 T57619 9.749 9.539 0.05556 H22579.1 8.617 11.23 0.05556 J02854 8.452 8.987 0.05556 R08183 7.899 9.73 0.05556 T92451 7.541 7.23 0.05556 T51261 7.054 6.123 0.05556 T47383 7.01 9.441 0.05556 D16431 6.994 8.48 0.05556 U30825 6.378 6.64 0.05556 H64807 6.107 6.395 0.05556 H78386 6.054 7.359 0.05556 U19969 5.726 7.677 0.05556 T56604 5.71 6.737 0.05556 M80815 5.475 8.043 0.05556 T61661 5.388 6.056 0.05556 T71025 5.3 6.055 0.05556 J04102 5.192 6.377 0.05556 X12369 5.181 5.504 0.05556 X63629 5.156 9.358 0.05556 M63391 5.129 7.411 0.05556 M76378.1 4.845 6.219 0.05556 H43887 4.841 4.793 0.05556 R87126 4.617 7.011 0.05556 M76378.2 4.522 7.023 0.05556 X12671 4.499 5.243 0.05556 H08393 4.42 5.268 0.05556 H20709 4.199 6.226 0.05556 H40095 4.134 5.369 0.05556 R36977 4.039 3.241 0.05556 T47377 3.905 3.928 0.05556 M76378 3.794 2.634 0.05556 J05032 3.412 3.226 0.05556 Accuracy: 0.9032
tAUC: 0.9045
sensitivity: 0.9
specificity: 0.9091
bootstrap:
hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.5,srtCol = 45)
vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")] 100 : T52185 200 : D16431
cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)| controlMean | controlStd | caseMean | caseStd | ROCAUC | |
|---|---|---|---|---|---|
| R87126 | 806.5 | 546.1 | 271.7 | 185.5 | 0.8841 |
| H08393 | 55.01 | 24.97 | 132.6 | 80.18 | 0.875 |
| R36977 | 125.7 | 63.3 | 375.6 | 340.1 | 0.8648 |
| M22382 | 405.2 | 220.3 | 1142 | 868.5 | 0.8648 |
| M26383 | 114.9 | 168.6 | 398.1 | 475.6 | 0.8534 |
| H40095 | 319.1 | 215.5 | 800.5 | 529.5 | 0.8409 |
| X63629 | 57.35 | 31.71 | 154.8 | 108.2 | 0.8352 |
| J05032 | 81.3 | 55.47 | 194.7 | 120.5 | 0.833 |
| X12671 | 333.3 | 263.5 | 850.5 | 582.4 | 0.8295 |
| Z50753 | 447.6 | 216.3 | 233.9 | 119 | 0.8284 |
| J02854 | 907.3 | 682.6 | 222.9 | 266.8 | 0.8261 |
| U09564 | 116.4 | 63.94 | 262.9 | 176.3 | 0.8239 |
| H43887 | 850.5 | 547.3 | 330.4 | 414.5 | 0.8216 |
| M63391 | 2303 | 1538 | 597.1 | 568.4 | 0.8205 |
| M76378.2 | 1052 | 844.1 | 260.3 | 172 | 0.8136 |
| M36634 | 214.5 | 146.9 | 93.32 | 96.29 | 0.8125 |
| T86473 | 156.4 | 94.12 | 407 | 323.7 | 0.8114 |
| X14958 | 350.8 | 209.1 | 716.1 | 434.5 | 0.8102 |
| M76378 | 1283 | 798.5 | 465.9 | 357.5 | 0.8068 |
| T47377 | 259.1 | 265.6 | 930.2 | 813.4 | 0.8057 |
| U30825 | 173.4 | 60.48 | 353.9 | 235.7 | 0.8023 |
| M26697 | 852.3 | 478.3 | 1848 | 1101 | 0.8023 |
| R84411 | 338.9 | 225.6 | 747.4 | 483.9 | 0.7989 |
| T71025 | 1892 | 697.2 | 1103 | 675.2 | 0.7977 |
| X54942 | 123 | 86.76 | 337.1 | 299.6 | 0.7932 |
| R08183 | 356.8 | 248.3 | 840.3 | 593.4 | 0.7932 |
| M76378.1 | 1306 | 805.3 | 490.9 | 375.2 | 0.792 |
| D31885 | 354.3 | 252.2 | 768.2 | 563.3 | 0.7886 |
| H06524 | 362.9 | 272.6 | 144.6 | 183 | 0.7875 |
| X55715 | 851.7 | 464.2 | 1555 | 833.8 | 0.7852 |
| WilcoxRes.p | Frequency | |
|---|---|---|
| R87126 | 0 | 0.852 |
| H08393 | 0 | 0.8867 |
| R36977 | 0 | 0.5187 |
| M22382 | 0 | 0.4813 |
| M26383 | 1e-04 | 0.608 |
| H40095 | 0 | 0.548 |
| X63629 | 0 | 0.8347 |
| J05032 | 0 | 0.4213 |
| X12671 | 0 | 0.5707 |
| Z50753 | 0 | 0.8027 |
| J02854 | 0 | 0.7293 |
| U09564 | 0 | 0.412 |
| H43887 | 0 | 0.6627 |
| M63391 | 0 | 0.7893 |
| M76378.2 | 0 | 0.5373 |
| M36634 | 0 | 0.3187 |
| T86473 | 0 | 0.356 |
| X14958 | 0 | 0.3373 |
| M76378 | 0 | 0.3693 |
| T47377 | 1e-04 | 0.7773 |
| U30825 | 1e-04 | 0.312 |
| M26697 | 1e-04 | 0.1707 |
| R84411 | 1e-04 | 0.4813 |
| T71025 | 0 | 0.792 |
| X54942 | 1e-04 | 0.3973 |
| R08183 | 1e-04 | 0.5 |
| M76378.1 | 0 | 0.2613 |
| D31885 | 2e-04 | 0.1613 |
| H06524 | 1e-04 | 0.224 |
| X55715 | 2e-04 | 0.3253 |