i— title: “BRCA Risk of Metastasis” author: “José Tamez-Peña” date: “Sep 20, 2017” output: word_document: fig_height: 6 fig_width: 8 toc: yes —

BRCA Analysis

library("FRESA.CAD")
library("R.matlab")
library("epiR")

#BRCAdata <- readMat("./BRCA_2002/DMMPLCN_080423.mat")
BRCAdata <- readMat("./DMMPLCN_080423.mat")


a=as.numeric(Sys.time());
set.seed(a);


summaryBRCA <- function(data)
{
  sumBC <- NULL
  sumBC$age <- c(mean(data$Age,na.rm=TRUE),sd(data$Age,na.rm=TRUE))
  sumBC$size <- c(mean(data$size,na.rm=TRUE),sd(data$size,na.rm=TRUE))
  sumBC$grade <- table(data$grade)
  sumBC$ER <- table(data$er)
  sumBC$type <- table(data$typeBRCA)
  sumBC$t.dmfs <- c(mean(data$t.dmfs,na.rm=TRUE),sd(data$t.dmfs,na.rm=TRUE))
  sumBC$e.dmfs <- table(data$e.dmfs)
  sumBC$t.sos <- c(mean(data$t.sos,na.rm=TRUE),sd(data$t.sos,na.rm=TRUE))
  sumBC$e.sos <- table(data$e.sos)
  sumBC$ln <- table(data$ln)
  cat(sprintf("Age: \t %5.1f (%4.1f)\n",sumBC$age[1],sumBC$age[2]))
  cat(sprintf("Size: \t %5.1f (%4.1f)\n",sumBC$size[1],sumBC$size[2]))
  cat("Grade: \t ",sumBC$grade,"\n")
  cat(sprintf("ER: \t  %d (%d) \n",sumBC$ER[1],sumBC$ER[2]))
  cat("Type: \t ",sumBC$type,"\n")
  cat(sprintf("Nodes: \t  %d (%d) \n",sumBC$ln[1],sumBC$ln[2]))
  cat(sprintf("DM Event:\t %d (%d) \n",sumBC$e.dmfs[1],sumBC$e.dmfs[2]))
  cat(sprintf("SOS Event:\t %d (%d) \n",sumBC$e.sos[1],sumBC$e.sos[2]))
  return (sumBC)
}

Data Preparation

op <- par(no.readonly = TRUE)


BRCAdata2 <- BRCAdata$DMMPLCN
DataExpresion <- as.data.frame(BRCAdata2[3])
subjectsIDs <- unlist(BRCAdata2[4])
genesIDs <- unlist(BRCAdata2[5])
ngenesIDs <- gsub("-","_",genesIDs,fixed = TRUE,perl=FALSE);
ngenesIDs <- gsub("/","_",ngenesIDs,fixed = TRUE,perl=FALSE);
TgenesIDs <- unlist(BRCAdata2[6])
names(TgenesIDs) <- ngenesIDs 

colnames(DataExpresion) <- subjectsIDs
rownames(DataExpresion) <- paste("N",ngenesIDs,sep="_")

#FRESA.CAD works with tranposed data frames
DataExpresionV <- as.data.frame(t(DataExpresion))

otr <- as.data.frame(BRCAdata2[7])
clinical  <- otr$X1.1
subjID <- as.character(unlist(clinical$Simplified.ID))
t.dmfs <- unlist(clinical$t.dmfs)/12
e.dmfs <- unlist(clinical$e.dmfs)
t.sos <- unlist(clinical$t.sos)/12
e.sos <- unlist(clinical$e.sos)
events <- unlist(clinical$Events)
ln <- unlist(clinical$LN)
er <- unlist(clinical$ER.IHC)
lumA <- unlist(clinical$LumA.HU)
lumB <- unlist(clinical$LumB.HU)
HER2 <- unlist(clinical$Her2.HU)
Basal <- unlist(clinical$Basal.HU)
Nrmal <- unlist(clinical$Normal.HU)
typeBRCA <- unlist(clinical$which.max)
size <- unlist(clinical$Size)
sig70 <- unlist(clinical$CIN70.bin)
rsig70 <- unlist(clinical$CIN70)

otr <- as.data.frame(BRCAdata2[8])
clinical2  <- otr$X1.1
ids2 <- as.character(unlist(clinical2$Simplified.ID))
age <- as.vector(clinical2$Age)
names(age) <- ids2
grade <- as.vector(clinical2$Grade)
names(grade) <- ids2
ER2 <- as.vector(clinical2$ER)
names(ER2) <- ids2
PGR <- as.vector(clinical2$PGR)
names(PGR) <- ids2
Node <- as.vector(clinical2$Node)
names(Node) <- ids2


#Small data frame with clinical info:
clicalDF <- data.frame(t.dmfs,e.dmfs,t.sos,e.sos,events,ln,er,typeBRCA,size)
rownames(clicalDF) <- subjID
clicalDF$Age <- age[subjID] 
clicalDF$grade <- grade[subjID]
clicalDF$PGR <- PGR[subjID]



# Events are DM events in less than ten years 
Event <- 1*((t.dmfs<10)&(e.dmfs==1))
sum(Event,na.rm=TRUE)
#> [1] 123
# Censor events greater than ten years
c.t.dmfs <- (t.dmfs>10)&(e.dmfs==1)

#Set the event column
DataExpresionV$Event <- 1*Event

#The Subjects with no event information
included <- !is.na(Event)


#Lets get the subjects with overall LOGITival data
#Mark events that are less than five years
LOGITEvent <- 1*(as.vector((t.sos<10)*e.sos))
sum(LOGITEvent,na.rm=TRUE)
#> [1] 108
LOGITExclude <-  as.vector(is.na(t.sos) | !is.na(Event))
DataExpresionLOGIT <- DataExpresionV



DataExpresionLOGIT$Event <- LOGITEvent

DataExpresionLOGIT <- DataExpresionLOGIT[!LOGITExclude,]
DataExpresionLOGIT$ct.dmfs <- 0;
DataExpresionLOGIT$ct.sos <-  t.sos[!LOGITExclude]
clicalDF$SEvent <- LOGITEvent
clicalDF$DMEvent <- Event
LOGITD <- clicalDF[!LOGITExclude,]
sum(LOGITD$SEvent)
#> [1] 80

sum(DataExpresionLOGIT$Event)
#> [1] 80

Summary of data sets

op <- par(no.readonly = TRUE)

smry <- summaryBRCA(clicalDF)

Age: 56.6 (13.7) Size: 25.3 (13.6) Grade: 158 358 277 ER: 200 (581) Type: 187 259 179 241 92 Nodes: 527 (249) DM Event: 374 (135) SOS Event: 399 (111)

pander::pander(smry,caption="Clinical")

• age: 56.63 and 13.73
• size: 25.28 and 13.57

• grade:

  1	2	3
  158	358	277

• ER:

  0	1
  200	581

• type:

  1	2	3	4	5
  187	259	179	241	92

• t.dmfs: 6.905 and 4.562

• e.dmfs:

  0	1
  374	135

• t.sos: 7.208 and 3.407

• e.sos:

  0	1
  399	111

• ln:

  0	1
  527	249

smry <- summaryBRCA(LOGITD) 

Age: 62.5 (14.0) Size: 22.2 (10.5) Grade: 88 176 111 ER: 31 (197) Type: 70 110 86 82 40 Nodes: 147 (76) DM Event: NA (NA) SOS Event: 305 (83)

#pander::pander(smry,caption="SOS Test Set")

roc.signature70 <- plotModels.ROC(cbind(DataExpresionLOGIT$Event,rsig70[!LOGITExclude]),main="70 Signature")

epi.signature70 <- epi.tests(roc.signature70$predictionTable)
pander::pander(epi.signature70$tab,caption="70 Signature")

70 Signature

	Outcome +	Outcome -	Total
Test +	57	137	194
Test -	23	171	194
Total	80	308	388

pander::pander(summary(epi.signature70),caption="Diangostic Summary 70 Signature")

Diangostic Summary 70 Signature

	est	lower	upper
aprev	0.5	0.4491	0.5509
tprev	0.2062	0.167	0.2499
se	0.7125	0.6005	0.8082
sp	0.5552	0.4978	0.6115
diag.acc	0.5876	0.5368	0.6371
diag.or	3.093	1.814	5.275
nnd	3.736	2.382	10.18
youden	0.2677	0.09822	0.4198
ppv	0.2938	0.2307	0.3633
npv	0.8814	0.8274	0.9233
plr	1.602	1.329	1.931
nlr	0.5178	0.3616	0.7416

DataExpresionLOGIT$ct.dmfs <- NULL
DataExpresionLOGIT$ct.sos <- NULL


#sf <- univariate_Logit(data=DataExpresionLOGIT, Outcome="Event", pvalue=0.2, adjustMethod="BH",uniTest=c("zIDI"))
sf <- univariate_Wilcoxon(data=DataExpresionLOGIT, Outcome="Event",pvalue=0.5)

theData <- DataExpresionLOGIT[,c("Event",names(sf))]



ExperimentName <- "BRCA2002"
bswimsReps <- 10;
theOutcome <- "Event";
reps <- 20;
fraction <- 0.3;

BSWiMSFileName <- paste(ExperimentName,"BSWIMSMethod.RDATA",sep = "_")
CVFileName <- paste(ExperimentName,"CVMethod.RDATA",sep = "_")

Benchmarking

BSWiMSMODEL <- BSWiMS.model(formula = paste(theOutcome," ~ 1"),data = theData,NumberofRepeats = bswimsReps)

save(BSWiMSMODEL,file = BSWiMSFileName)

load(file = BSWiMSFileName)

BSWiMSMODEL$formula.list

par(mfrow = c(2,2),cex=0.6);
cp <- BinaryBenchmark(theData,theOutcome,reps,fraction)

save(cp,file = CVFileName)
par(mfrow = c(1,1),cex=1.0);

load(file = CVFileName)

cp$TheCVEvaluations$Reference$medianTest
cp$TheCVEvaluations$Reference$selectedFeaturesSet

Results

Classifier Results

hm <- heatMaps(Outcome = "Outcome",data = cp$testPredictions,title = "Heat Map",Scale = FALSE,hCluster = "col",cexRow = 0.25,cexCol = 0.75,srtCol = 45) 

#The Times
par(mfrow = c(2,1),cex=1.0);
pander::pander(cp$cpuElapsedTimes)

BSWiMS	RF	RPART	LASSO	SVM	KNN	ENS
58.15	107.4	10	1.726	0.034	0.0385	177.4

par(mfrow = c(1,1),cex=1.0);
learningTime <- -1*cp$cpuElapsedTimes
pr <- plot(cp)

Radar Plots

library(fmsb)
par(mfrow = c(1,2),xpd = TRUE,pty = "s",mar = c(1,1,1,1))

mNames <- names(cp$cpuElapsedTimes)

classRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],min(cp$cpuElapsedTimes))
classRanks <- rbind(classRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,max(cp$cpuElapsedTimes)))
classRanks <- as.data.frame(rbind(classRanks,cbind(t(pr$metrics[c("BER","ACC","AUC","SEN","SPE"),mNames]),cp$cpuElapsedTimes)))
colnames(classRanks) <- c("BER","ACC","AUC","SEN","SPE","CPU")

classRanks$BER <- -classRanks$BER
classRanks$CPU <- -classRanks$CPU

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(classRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.5 ,title = "Prediction Model")

legend("topleft",legend = rownames(classRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)


filnames <- c("BSWiMS","LASSO","RF.ref","IDI","t-test","Kendall","mRMR")

filterRanks <- c(pr$minMaxMetrics$BER[1],pr$minMaxMetrics$ACC[2],pr$minMaxMetrics$AUC[2],pr$minMaxMetrics$SEN[2],pr$minMaxMetrics$SPE[2],max(cp$jaccard),min(cp$featsize));

filterRanks <- rbind(filterRanks,c(pr$minMaxMetrics$BER[2],0,0,0,0,min(cp$jaccard),max(cp$featsize)));

filterRanks <- as.data.frame(rbind(filterRanks,cbind(t(pr$metrics_filter[c("BER","ACC","AUC","SEN","SPE"),filnames]),cp$jaccard[filnames],cp$featsize[filnames])));
colnames(filterRanks) <- c("BER","ACC","AUC","SEN","SPE","Jaccard","SIZE")
filterRanks$BER <- -filterRanks$BER
filterRanks$SIZE <- -filterRanks$SIZE

colors_border = c( rgb(1.0,0.0,0.0,1.0), rgb(0.0,1.0,0.0,1.0) , rgb(0.0,0.0,1.0,1.0), rgb(0.2,0.2,0.0,1.0), rgb(0.0,1.0,1.0,1.0), rgb(1.0,0.0,1.0,1.0), rgb(0.0,0.0,0.0,1.0) )
colors_in = c( rgb(1.0,0.0,0.0,0.05), rgb(0.0,1.0,0.0,0.05) , rgb(0.0,0.0,1.0,0.05),rgb(1.0,1.0,0.0,0.05), rgb(0.0,1.0,1.0,0.05) , rgb(1.0,0.0,1.0,0.05), rgb(0.0,0.0,0.0,0.05) )
radarchart(filterRanks,axistype = 0,maxmin = T,pcol = colors_border,pfcol = colors_in,plwd = c(6,2,2,2,2,2,2),plty = 1, cglcol = "grey", cglty = 1,axislabcol = "black",cglwd = 0.8, vlcex  = 0.6,title = "Filter Method" )


legend("topleft",legend = rownames(filterRanks[-c(1,2),]),bty = "n",pch = 20,col = colors_in,text.col = colors_border,cex = 0.5,pt.cex = 2)

detach("package:fmsb", unload=TRUE)

par(mfrow = c(1,1))
par(op)

Feature Analysis

rm <- rowMeans(cp$featureSelectionFrequency)
selFrequency <- cp$featureSelectionFrequency[rm > 0.20,]
gplots::heatmap.2(selFrequency,trace = "none",mar = c(10,10),main = "Features",cexRow = 0.3)

topFeat <- min(ncol(BSWiMSMODEL$bagging$formulaNetwork),30);
gplots::heatmap.2(BSWiMSMODEL$bagging$formulaNetwork[1:topFeat,1:topFeat],trace="none",mar = c(10,10),main = "B:SWiMS Formula Network",cexRow = 0.5,cexCol = 0.5)

pander::pander(summary(BSWiMSMODEL$bagging$bagged.model,caption="Colon",round = 3))

• coefficients:

  Table continues below

  	Estimate	lower	OR	upper
  N_204359_at	-0.002428	0.997	0.9976	0.9981
  N_AFFX_HUMGAPDH_M33197_3_at	0.001089	1.001	1.001	1.001
  N_213156_at	-0.004981	0.9938	0.995	0.9962
  N_200737_at	0.01462	1.011	1.015	1.018
  N_218447_at	0.003676	1.003	1.004	1.005
  N_203380_x_at	-0.204	0.7754	0.8154	0.8575
  N_202779_s_at	0.0479	1.037	1.049	1.062
  N_201990_s_at	-0.05419	0.9346	0.9473	0.96
  N_222077_s_at	0.0266	1.02	1.027	1.034
  N_201584_s_at	0.009478	1.007	1.01	1.012
  N_202561_at	-0.00329	0.9959	0.9967	0.9975
  N_202160_at	-0.0579	0.9301	0.9437	0.9576
  N_209236_at	-0.01605	0.9801	0.9841	0.9881
  N_218589_at	-0.004104	0.9949	0.9959	0.9969
  N_213058_at	-0.05693	0.9311	0.9447	0.9584
  N_219238_at	-0.01771	0.978	0.9824	0.9869
  N_202535_at	0.01424	1.011	1.014	1.018
  N_203799_at	-0.04771	0.9415	0.9534	0.9654
  N_212188_at	-0.009187	0.9885	0.9909	0.9933
  N_212176_at	-0.02099	0.9738	0.9792	0.9847
  N_212549_at	-0.06001	0.9268	0.9418	0.9569
  N_212914_at	-0.01002	0.9874	0.99	0.9927
  N_209540_at	-0.001477	0.9981	0.9985	0.9989
  N_209773_s_at	0.04328	1.032	1.044	1.056
  N_221063_x_at	0.02367	1.017	1.024	1.031
  N_218901_at	-0.07063	0.9142	0.9318	0.9498
  N_213158_at	-0.009904	0.9875	0.9901	0.9928
  N_35776_at	-0.00954	0.9879	0.9905	0.9931
  N_203852_s_at	0.01563	1.011	1.016	1.02
  N_218087_s_at	-0.004979	0.9937	0.995	0.9964
  N_201037_at	0.006319	1.005	1.006	1.008
  N_213698_at	-0.01764	0.9777	0.9825	0.9873
  N_218887_at	0.01591	1.011	1.016	1.021
  N_218174_s_at	-0.02707	0.9659	0.9733	0.9808
  N_209598_at	-0.09769	0.8821	0.9069	0.9324
  N_205898_at	-0.03855	0.9517	0.9622	0.9728
  N_210236_at	0.1257	1.094	1.134	1.176
  N_202205_at	0.00588	1.004	1.006	1.008
  N_206163_at	-0.106	0.8722	0.8994	0.9274
  N_222368_at	-0.1584	0.8153	0.8535	0.8936
  N_219933_at	0.08661	1.063	1.09	1.118
  N_208074_s_at	0.05172	1.037	1.053	1.069
  N_208677_s_at	0.0772	1.056	1.08	1.105
  N_218996_at	0.003445	1.002	1.003	1.004
  N_200738_s_at	0.002376	1.002	1.002	1.003
  N_222129_at	-0.007842	0.9898	0.9922	0.9945
  N_214171_s_at	-0.06589	0.9178	0.9362	0.955
  N_209735_at	-0.00506	0.9934	0.995	0.9965
  N_217922_at	-0.004413	0.9943	0.9956	0.9969
  N_214074_s_at	0.01586	1.011	1.016	1.021
  N_212289_at	-0.03809	0.9515	0.9626	0.9739
  N_201989_s_at	-0.008645	0.9888	0.9914	0.994
  N_209724_s_at	-0.1423	0.8303	0.8674	0.9061
  N_201578_at	-0.002934	0.9962	0.9971	0.998
  N_217714_x_at	0.01135	1.008	1.011	1.015
  N_213727_x_at	-0.08	0.9003	0.9231	0.9465
  N_203339_at	-0.115	0.8598	0.8913	0.924
  N_218629_at	-0.4037	0.5883	0.6678	0.7581
  N_202066_at	0.06034	1.042	1.062	1.083
  N_221045_s_at	-0.03255	0.9581	0.968	0.978
  N_204735_at	-0.01148	0.985	0.9886	0.9922
  N_208968_s_at	0.02417	1.017	1.024	1.032
  N_212614_at	-0.02675	0.9653	0.9736	0.982
  N_202308_at	-0.006064	0.992	0.994	0.9959
  N_208228_s_at	-0.01228	0.9839	0.9878	0.9917
  N_216538_at	-0.01405	0.9816	0.986	0.9905
  N_220005_at	-0.1108	0.8637	0.8951	0.9276
  N_219429_at	0.03266	1.022	1.033	1.044
  N_203894_at	-0.01944	0.9746	0.9807	0.9869
  N_210519_s_at	0.02336	1.016	1.024	1.031
  N_206772_at	-0.03586	0.9536	0.9648	0.9761
  N_215171_s_at	0.00638	1.004	1.006	1.008
  N_214477_at	0.1555	1.111	1.168	1.229
  N_214906_x_at	-0.01982	0.974	0.9804	0.9867
  N_205357_s_at	-0.04519	0.9418	0.9558	0.97
  N_213375_s_at	-0.05724	0.9265	0.9444	0.9626
  N_218614_at	-0.1177	0.8544	0.889	0.9249
  N_219832_s_at	0.1462	1.101	1.157	1.216
  N_201228_s_at	0.1332	1.092	1.142	1.195
  N_203639_s_at	-0.04258	0.9445	0.9583	0.9723
  N_203638_s_at	-0.01033	0.9862	0.9897	0.9932
  N_212676_at	-0.003221	0.9957	0.9968	0.9979
  N_201841_s_at	0.003155	1.002	1.003	1.004
  N_219218_at	-0.1321	0.8372	0.8763	0.9171
  N_214330_at	-0.008145	0.9891	0.9919	0.9947
  N_202459_s_at	-0.03964	0.9479	0.9611	0.9745
  N_213639_s_at	-0.06367	0.9177	0.9383	0.9594
  N_201387_s_at	-0.009922	0.9867	0.9901	0.9936
  N_219416_at	-0.01144	0.9846	0.9886	0.9927
  N_221878_at	-0.007103	0.9904	0.9929	0.9955
  N_218322_s_at	-0.0332	0.9559	0.9673	0.979
  N_213013_at	0.01354	1.009	1.014	1.019
  N_59644_at	-0.0128	0.9826	0.9873	0.9919
  N_221348_at	0.01599	1.01	1.016	1.022
  N_202838_at	-0.007308	0.99	0.9927	0.9954
  N_208016_s_at	-0.003728	0.9949	0.9963	0.9977
  N_218933_at	0.1197	1.077	1.127	1.179
  N_201467_s_at	0.001171	1.001	1.001	1.002
  N_219582_at	-0.01301	0.9821	0.9871	0.9921
  N_202771_at	0.002224	1.001	1.002	1.003
  N_221185_s_at	-0.03102	0.9576	0.9695	0.9814
  N_203185_at	-0.006076	0.9914	0.9939	0.9965
  N_203958_s_at	-0.007446	0.9891	0.9926	0.9961

  Table continues below

  	u.Accuracy	r.Accuracy	full.Accuracy
  N_204359_at	0.6175	0.5556	0.6528
  N_AFFX_HUMGAPDH_M33197_3_at	0.6403	0.2062	0.6403
  N_213156_at	0.6233	0.6151	0.6732
  N_200737_at	0.6232	0.6418	0.7086
  N_218447_at	0.6384	0.5806	0.6624
  N_203380_x_at	0.6132	0.645	0.6965
  N_202779_s_at	0.648	0.6304	0.6929
  N_201990_s_at	0.6445	0.6528	0.6939
  N_222077_s_at	0.6278	0.6161	0.6676
  N_201584_s_at	0.6476	0.506	0.669
  N_202561_at	0.6182	0.5847	0.6539
  N_202160_at	0.6163	0.6365	0.6867
  N_209236_at	0.6175	0.6926	0.7143
  N_218589_at	0.6546	0.5932	0.6834
  N_213058_at	0.6509	0.5439	0.6737
  N_219238_at	0.5909	0.6616	0.6819
  N_202535_at	0.6973	0.6528	0.7198
  N_203799_at	0.624	0.5954	0.6916
  N_212188_at	0.6524	0.6357	0.6878
  N_212176_at	0.608	0.6016	0.6706
  N_212549_at	0.6444	0.545	0.6719
  N_212914_at	0.6246	0.6249	0.6668
  N_209540_at	0.6123	0.6079	0.6491
  N_209773_s_at	0.6135	0.6459	0.6886
  N_221063_x_at	0.5891	0.655	0.7147
  N_218901_at	0.6479	0.6674	0.7099
  N_213158_at	0.6428	0.6031	0.674
  N_35776_at	0.6182	0.5885	0.662
  N_203852_s_at	0.5762	0.6543	0.7091
  N_218087_s_at	0.6322	0.6179	0.6674
  N_201037_at	0.6307	0.6554	0.7143
  N_213698_at	0.6464	0.6314	0.7016
  N_218887_at	0.5887	0.6355	0.7066
  N_218174_s_at	0.5527	0.7058	0.7338
  N_209598_at	0.5592	0.6648	0.6867
  N_205898_at	0.6515	0.6715	0.6996
  N_210236_at	0.6913	0.6588	0.7062
  N_202205_at	0.6075	0.6524	0.6848
  N_206163_at	0.5731	0.6678	0.6735
  N_222368_at	0.6249	0.6756	0.7057
  N_219933_at	0.638	0.6496	0.6966
  N_208074_s_at	0.627	0.6478	0.6842
  N_208677_s_at	0.6132	0.6792	0.7073
  N_218996_at	0.6303	0.6487	0.6955
  N_200738_s_at	0.6451	0.63	0.6848
  N_222129_at	0.6314	0.6128	0.6629
  N_214171_s_at	0.5486	0.6679	0.6999
  N_209735_at	0.6068	0.6418	0.6686
  N_217922_at	0.63	0.6451	0.6848
  N_214074_s_at	0.7012	0.6394	0.6681
  N_212289_at	0.6328	0.631	0.6737
  N_201989_s_at	0.662	0.6337	0.6985
  N_209724_s_at	0.5958	0.6564	0.7002
  N_201578_at	0.5806	0.6384	0.6624
  N_217714_x_at	0.6325	0.6554	0.7014
  N_213727_x_at	0.6234	0.6503	0.6923
  N_203339_at	0.6697	0.6649	0.7001
  N_218629_at	0.5382	0.6844	0.7205
  N_202066_at	0.6247	0.6638	0.7054
  N_221045_s_at	0.62	0.6861	0.6942
  N_204735_at	0.6086	0.6306	0.6719
  N_208968_s_at	0.6494	0.6378	0.6869
  N_212614_at	0.6119	0.6727	0.6999
  N_202308_at	0.5615	0.7047	0.7147
  N_208228_s_at	0.5732	0.6314	0.6806
  N_216538_at	0.5895	0.6686	0.6911
  N_220005_at	0.6501	0.673	0.7017
  N_219429_at	0.6751	0.6481	0.6885
  N_203894_at	0.5932	0.6594	0.6826
  N_210519_s_at	0.6226	0.6433	0.6831
  N_206772_at	0.5588	0.6636	0.6929
  N_215171_s_at	0.6226	0.6694	0.6985
  N_214477_at	0.5713	0.6631	0.6932
  N_214906_x_at	0.5976	0.658	0.6819
  N_205357_s_at	0.5571	0.6898	0.7207
  N_213375_s_at	0.6479	0.6657	0.6972
  N_218614_at	0.6061	0.6867	0.7155
  N_219832_s_at	0.6196	0.6591	0.6874
  N_201228_s_at	0.5896	0.6594	0.68
  N_203639_s_at	0.5593	0.6607	0.6959
  N_203638_s_at	0.5994	0.6395	0.665
  N_212676_at	0.6079	0.6281	0.6767
  N_201841_s_at	0.6189	0.655	0.7014
  N_219218_at	0.6338	0.6824	0.6989
  N_214330_at	0.6009	0.6215	0.6933
  N_202459_s_at	0.5783	0.66	0.6997
  N_213639_s_at	0.594	0.6688	0.7068
  N_201387_s_at	0.5188	0.7128	0.7066
  N_219416_at	0.6095	0.6357	0.6742
  N_221878_at	0.6162	0.6487	0.6813
  N_218322_s_at	0.5844	0.6681	0.6989
  N_213013_at	0.5983	0.6675	0.6797
  N_59644_at	0.5663	0.6922	0.6933
  N_221348_at	0.6274	0.6524	0.6649
  N_202838_at	0.602	0.6447	0.6797
  N_208016_s_at	0.4687	0.69	0.7143
  N_218933_at	0.6118	0.689	0.7043
  N_201467_s_at	0.6344	0.6256	0.669
  N_219582_at	0.5302	0.697	0.7032
  N_202771_at	0.6222	0.634	0.6653
  N_221185_s_at	0.5598	0.6621	0.6898
  N_203185_at	0.577	0.6561	0.6605
  N_203958_s_at	0.5876	0.6922	0.6951

  Table continues below

  	u.AUC	r.AUC	full.AUC	IDI
  N_204359_at	0.6394	0.5793	0.6716	0.1155
  N_AFFX_HUMGAPDH_M33197_3_at	0.6624	0.5	0.6624	0.1027
  N_213156_at	0.6236	0.6171	0.6775	0.09689
  N_200737_at	0.6222	0.6326	0.6876	0.09546
  N_218447_at	0.65	0.5971	0.6763	0.09261
  N_203380_x_at	0.6067	0.6679	0.7203	0.09379
  N_202779_s_at	0.6376	0.6289	0.6895	0.09298
  N_201990_s_at	0.6366	0.6313	0.6898	0.09173
  N_222077_s_at	0.6406	0.6183	0.6806	0.09336
  N_201584_s_at	0.6426	0.5869	0.6686	0.09192
  N_202561_at	0.6412	0.6241	0.6802	0.08814
  N_202160_at	0.6139	0.6281	0.6956	0.08808
  N_209236_at	0.6414	0.6649	0.7156	0.08424
  N_218589_at	0.6509	0.6155	0.712	0.08966
  N_213058_at	0.6727	0.6011	0.6835	0.08849
  N_219238_at	0.6128	0.6805	0.7038	0.08273
  N_202535_at	0.6035	0.6544	0.7118	0.08156
  N_203799_at	0.6368	0.6358	0.7016	0.08265
  N_212188_at	0.6548	0.6516	0.6913	0.08298
  N_212176_at	0.6042	0.6468	0.6936	0.07897
  N_212549_at	0.6616	0.6141	0.6851	0.07941
  N_212914_at	0.6265	0.6137	0.6762	0.0816
  N_209540_at	0.6494	0.6261	0.6686	0.07962
  N_209773_s_at	0.612	0.6621	0.6965	0.08446
  N_221063_x_at	0.5832	0.6333	0.7026	0.08097
  N_218901_at	0.6596	0.6664	0.7089	0.08073
  N_213158_at	0.6326	0.6173	0.6699	0.08141
  N_35776_at	0.6257	0.5974	0.6847	0.07307
  N_203852_s_at	0.5672	0.6494	0.7183	0.07372
  N_218087_s_at	0.6492	0.6214	0.6737	0.0755
  N_201037_at	0.618	0.6864	0.7156	0.07667
  N_213698_at	0.6248	0.6297	0.6957	0.07538
  N_218887_at	0.5922	0.6435	0.7094	0.06991
  N_218174_s_at	0.5682	0.6957	0.7444	0.06923
  N_209598_at	0.6251	0.6738	0.7164	0.07319
  N_205898_at	0.6825	0.6818	0.7037	0.07307
  N_210236_at	0.6286	0.669	0.7034	0.0682
  N_202205_at	0.61	0.6542	0.6838	0.07044
  N_206163_at	0.6306	0.6663	0.6961	0.07183
  N_222368_at	0.6515	0.6725	0.712	0.06935
  N_219933_at	0.6312	0.6571	0.6955	0.072
  N_208074_s_at	0.6392	0.6461	0.6862	0.06873
  N_208677_s_at	0.631	0.6835	0.7181	0.06441
  N_218996_at	0.6191	0.6261	0.6978	0.06336
  N_200738_s_at	0.6469	0.6162	0.6719	0.06649
  N_222129_at	0.6132	0.618	0.6595	0.06518
  N_214171_s_at	0.5412	0.693	0.7257	0.06747
  N_209735_at	0.6208	0.6574	0.6776	0.06213
  N_217922_at	0.6162	0.6469	0.6719	0.06346
  N_214074_s_at	0.649	0.6477	0.6683	0.06345
  N_212289_at	0.655	0.6389	0.6783	0.06071
  N_201989_s_at	0.6602	0.6397	0.697	0.06057
  N_209724_s_at	0.5827	0.6567	0.6984	0.06094
  N_201578_at	0.5971	0.65	0.6763	0.05928
  N_217714_x_at	0.6397	0.6712	0.72	0.05777
  N_213727_x_at	0.6264	0.6617	0.6973	0.05961
  N_203339_at	0.675	0.6814	0.7154	0.05642
  N_218629_at	0.5589	0.6988	0.7348	0.05958
  N_202066_at	0.5791	0.6767	0.7007	0.06024
  N_221045_s_at	0.6441	0.6709	0.7029	0.05896
  N_204735_at	0.6255	0.6492	0.6803	0.05834
  N_208968_s_at	0.6453	0.6576	0.6796	0.05748
  N_212614_at	0.6135	0.6868	0.7257	0.05798
  N_202308_at	0.5738	0.6871	0.7026	0.06198
  N_208228_s_at	0.6021	0.6551	0.6932	0.05645
  N_216538_at	0.6026	0.6915	0.6997	0.05849
  N_220005_at	0.6902	0.688	0.714	0.05935
  N_219429_at	0.6358	0.6472	0.6827	0.05622
  N_203894_at	0.6157	0.6588	0.6983	0.05387
  N_210519_s_at	0.6148	0.661	0.6881	0.05501
  N_206772_at	0.593	0.6625	0.7019	0.05344
  N_215171_s_at	0.609	0.6946	0.697	0.05632
  N_214477_at	0.5626	0.6663	0.7068	0.05541
  N_214906_x_at	0.6143	0.6755	0.7038	0.05158
  N_205357_s_at	0.658	0.7011	0.7341	0.05245
  N_213375_s_at	0.6637	0.6706	0.6995	0.05543
  N_218614_at	0.638	0.6966	0.7244	0.05505
  N_219832_s_at	0.6063	0.6634	0.6927	0.05205
  N_201228_s_at	0.5953	0.665	0.6899	0.05103
  N_203639_s_at	0.5959	0.6617	0.699	0.05046
  N_203638_s_at	0.5967	0.6439	0.6767	0.04978
  N_212676_at	0.6136	0.6395	0.6748	0.04939
  N_201841_s_at	0.6232	0.6677	0.72	0.04958
  N_219218_at	0.6252	0.6912	0.7127	0.05015
  N_214330_at	0.6098	0.636	0.6918	0.04732
  N_202459_s_at	0.6081	0.6645	0.7051	0.04959
  N_213639_s_at	0.609	0.6863	0.7152	0.04782
  N_201387_s_at	0.5819	0.6915	0.6896	0.04982
  N_219416_at	0.6493	0.6562	0.7055	0.04458
  N_221878_at	0.6313	0.6433	0.6727	0.0461
  N_218322_s_at	0.6267	0.6797	0.7104	0.04397
  N_213013_at	0.5718	0.6835	0.6912	0.04647
  N_59644_at	0.5946	0.672	0.6978	0.04307
  N_221348_at	0.6034	0.6522	0.6773	0.04475
  N_202838_at	0.6214	0.6642	0.6945	0.04246
  N_208016_s_at	0.6085	0.6792	0.7169	0.03662
  N_218933_at	0.615	0.6908	0.7038	0.04184
  N_201467_s_at	0.6124	0.6267	0.6785	0.04112
  N_219582_at	0.5699	0.712	0.7073	0.04183
  N_202771_at	0.6332	0.6485	0.6735	0.04026
  N_221185_s_at	0.5809	0.6829	0.7027	0.04019
  N_203185_at	0.5994	0.6948	0.6989	0.03583
  N_203958_s_at	0.612	0.7169	0.7207	0.02724

  	NRI	z.IDI	z.NRI	Frequency
  N_204359_at	0.5584	8.901	7.271	0.1
  N_AFFX_HUMGAPDH_M33197_3_at	0.6521	8.423	8.626	0.1
  N_213156_at	0.4235	8.062	5.396	0.2
  N_200737_at	0.506	7.982	6.504	0.2
  N_218447_at	0.5872	7.959	7.639	0.1
  N_203380_x_at	0.4917	7.948	6.329	1
  N_202779_s_at	0.4877	7.947	6.269	1
  N_201990_s_at	0.5671	7.927	7.371	0.8
  N_222077_s_at	0.5233	7.88	6.757	0.8
  N_201584_s_at	0.5355	7.833	6.928	0.4
  N_202561_at	0.5594	7.82	7.257	0.1
  N_202160_at	0.5144	7.791	6.646	0.4
  N_209236_at	0.6605	7.765	8.772	0.1
  N_218589_at	0.5826	7.725	7.564	0.1
  N_213058_at	0.5982	7.716	7.836	1
  N_219238_at	0.5566	7.637	7.208	0.1
  N_202535_at	0.4485	7.532	6.081	0.2
  N_203799_at	0.4717	7.462	6.053	1
  N_212188_at	0.5728	7.423	7.435	0.2
  N_212176_at	0.432	7.378	5.512	0.3
  N_212549_at	0.576	7.365	7.512	1
  N_212914_at	0.4515	7.36	5.76	0.3
  N_209540_at	0.5677	7.279	7.451	0.1
  N_209773_s_at	0.3909	7.271	4.962	1
  N_221063_x_at	0.5325	7.26	6.878	0.1
  N_218901_at	0.5058	7.237	6.522	1
  N_213158_at	0.4956	7.217	6.362	0.4
  N_35776_at	0.4708	7.18	6.02	0.2
  N_203852_s_at	0.4592	7.128	5.872	0.1
  N_218087_s_at	0.5478	7.11	7.115	0.4
  N_201037_at	0.564	7.083	7.328	0.1
  N_213698_at	0.4492	7.062	5.762	0.4
  N_218887_at	0.5427	6.955	7.013	0.1
  N_218174_s_at	0.4147	6.936	5.273	0.1
  N_209598_at	0.4217	6.902	5.505	1
  N_205898_at	0.4776	6.884	6.141	1
  N_210236_at	0.4701	6.845	6.17	1
  N_202205_at	0.461	6.803	5.883	0.1
  N_206163_at	0.4156	6.771	5.383	0.9
  N_222368_at	0.5658	6.768	7.359	1
  N_219933_at	0.403	6.727	5.14	1
  N_208074_s_at	0.4465	6.715	5.714	1
  N_208677_s_at	0.5232	6.678	6.754	0.6
  N_218996_at	0.5	6.62	6.42	0.1
  N_200738_s_at	0.4703	6.607	6.021	0.1
  N_222129_at	0.4031	6.503	5.118	0.2
  N_214171_s_at	0.5751	6.498	7.455	0.1
  N_209735_at	0.385	6.471	4.892	0.1
  N_217922_at	0.5297	6.459	6.832	0.1
  N_214074_s_at	0.5223	6.454	6.907	0.3
  N_212289_at	0.5659	6.42	7.378	0.7
  N_201989_s_at	0.5631	6.388	7.299	0.1
  N_209724_s_at	0.3083	6.383	3.885	1
  N_201578_at	0.3609	6.373	4.57	0.1
  N_217714_x_at	0.4555	6.343	5.811	0.1
  N_213727_x_at	0.4308	6.277	5.497	0.9
  N_203339_at	0.4215	6.259	5.379	1
  N_218629_at	0.4631	6.24	5.923	1
  N_202066_at	0.3316	6.239	4.261	1
  N_221045_s_at	0.5303	6.219	6.853	0.3
  N_204735_at	0.3973	6.207	5.069	0.3
  N_208968_s_at	0.4827	6.185	6.195	0.5
  N_212614_at	0.4703	6.14	6.027	0.1
  N_202308_at	0.3813	6.14	4.828	0.1
  N_208228_s_at	0.397	6.112	5.063	0.4
  N_216538_at	0.3701	6.098	4.677	0.1
  N_220005_at	0.6193	6.088	8.185	1
  N_219429_at	0.4287	6.081	5.527	0.6
  N_203894_at	0.4341	6.048	5.558	0.3
  N_210519_s_at	0.3909	6.029	4.968	0.8
  N_206772_at	0.4852	6.027	6.241	0.3
  N_215171_s_at	0.3423	6.021	4.337	0.1
  N_214477_at	0.3457	6.016	4.373	0.6
  N_214906_x_at	0.3822	5.999	4.853	0.1
  N_205357_s_at	0.4958	5.996	6.84	1
  N_213375_s_at	0.4854	5.876	6.246	1
  N_218614_at	0.4945	5.821	6.403	1
  N_219832_s_at	0.4313	5.763	5.499	1
  N_201228_s_at	0.4096	5.759	5.209	0.5
  N_203639_s_at	0.3865	5.757	4.934	0.6
  N_203638_s_at	0.3355	5.716	4.233	0.6
  N_212676_at	0.474	5.706	6.068	0.1
  N_201841_s_at	0.4267	5.704	5.431	0.1
  N_219218_at	0.4582	5.676	5.858	1
  N_214330_at	0.4128	5.662	5.253	0.1
  N_202459_s_at	0.5485	5.622	7.126	0.3
  N_213639_s_at	0.3991	5.616	5.078	0.5
  N_201387_s_at	0.5724	5.607	7.494	0.1
  N_219416_at	0.5093	5.513	6.623	0.2
  N_221878_at	0.3966	5.454	5.042	0.2
  N_218322_s_at	0.4367	5.452	5.613	0.4
  N_213013_at	0.3497	5.356	4.454	0.1
  N_59644_at	0.2941	5.328	3.703	0.1
  N_221348_at	0.4434	5.273	5.682	0.1
  N_202838_at	0.4596	5.253	5.878	0.2
  N_208016_s_at	0.3089	5.212	4.438	0.1
  N_218933_at	0.5155	5.204	6.631	0.6
  N_201467_s_at	0.2978	5.193	3.75	0.1
  N_219582_at	0.5724	5.007	7.503	0.1
  N_202771_at	0.4527	4.984	5.774	0.1
  N_221185_s_at	0.3974	4.946	5.057	0.5
  N_203185_at	0.436	4.707	5.575	0.1
  N_203958_s_at	0.3729	4.143	4.724	0.1

• Accuracy: 0.7448
• tAUC: 0.7653
• sensitivity: 0.8
• specificity: 0.7305

• bootstrap:

hm <- heatMaps(Outcome = theOutcome,data = theData[,c(theOutcome,rownames(selFrequency))],title = "Heat Map",Scale = TRUE,hCluster = "col",cexRow = 0.25,cexCol = 0.25,srtCol = 45)

vlist <- rownames(selFrequency)
vlist <- cbind(vlist,vlist)
univ <- univariateRankVariables(variableList = vlist,formula = paste(theOutcome,"~1"),Outcome = theOutcome,data = theData,type = "LOGIT",rankingTest = "zIDI",uniType = "Binary")[,c("controlMean","controlStd","caseMean","caseStd","ROCAUC","WilcoxRes.p")] 

cnames <- colnames(univ);
univ <- cbind(univ,rm[rownames(univ)])
colnames(univ) <- c(cnames,"Frequency")
univ <- univ[order(-univ[,5]),]
pander::pander(univ[1:topFeat,],caption = "Features",round = 4)

Features (continued below)

	controlMean	controlStd	caseMean	caseStd	ROCAUC
N_205898_at	0.284	1.216	-0.6338	0.9395	0.7208
N_218901_at	0.1326	0.7436	-0.4592	0.6847	0.7199
N_203339_at	0.1219	0.428	-0.1854	0.4152	0.7184
N_205357_s_at	0.857	1.9	-0.2445	1.355	0.7181
N_213375_s_at	0.1369	0.4984	-0.2583	0.4892	0.7151
N_209773_s_at	-0.2205	1.188	0.6343	0.8893	0.7084
N_202779_s_at	-0.1287	0.9434	0.574	0.864	0.7078
N_203799_at	0.1439	0.6836	-0.3691	0.6742	0.7058
N_212549_at	0.0749	0.4289	-0.2077	0.3742	0.703
N_213058_at	0.0811	0.3952	-0.178	0.2883	0.7012
N_203380_x_at	0.1024	0.475	-0.2364	0.4238	0.6986
N_204709_s_at	-0.0332	0.5072	0.3078	0.5228	0.6977
N_219933_at	-0.1363	0.4956	0.2064	0.4313	0.6976
N_212289_at	0.0464	0.347	-0.181	0.3499	0.6946
N_201195_s_at	0.0447	1.082	0.693	0.9998	0.6935
N_218589_at	0.0463	0.5052	-0.2699	0.4367	0.6914
N_201990_s_at	0.0413	0.5692	-0.3385	0.5747	0.6877
N_220005_at	0.1439	0.5786	-0.2227	0.4961	0.6866
N_210236_at	-0.0354	0.3523	0.1995	0.4259	0.6858
N_218447_at	-0.0927	0.5557	0.286	0.5243	0.684
N_213727_x_at	0.005	0.284	-0.1737	0.2539	0.6837
N_208968_s_at	-0.0591	0.386	0.1975	0.3877	0.6835
N_221539_at	-0.0463	0.7319	0.4702	0.8404	0.6828
N_212188_at	0.0609	0.7475	-0.4526	0.8051	0.6822
N_214118_x_at	0.0702	0.5137	-0.2534	0.4459	0.6798
N_200738_s_at	-0.0991	0.5641	0.2803	0.636	0.6795
N_213340_s_at	0.0563	0.4327	-0.178	0.372	0.6793
N_213034_at	0.1736	0.6655	-0.2273	0.6497	0.6791
N_222129_at	0.0251	0.3256	-0.1947	0.3297	0.6782
N_222368_at	0.0354	0.3759	-0.1788	0.2804	0.6781

	WilcoxRes.p	Frequency
N_205898_at	0	0.51
N_218901_at	0	0.425
N_203339_at	0	0.495
N_205357_s_at	0	0.515
N_213375_s_at	0	0.495
N_209773_s_at	0	0.38
N_202779_s_at	0	0.41
N_203799_at	0	0.34
N_212549_at	0	0.41
N_213058_at	0	0.275
N_203380_x_at	0	0.38
N_204709_s_at	0	0.315
N_219933_at	0	0.345
N_212289_at	0	0.365
N_201195_s_at	0	0.26
N_218589_at	0	0.35
N_201990_s_at	0	0.215
N_220005_at	0	0.495
N_210236_at	0	0.27
N_218447_at	0	0.245
N_213727_x_at	0	0.245
N_208968_s_at	0	0.345
N_221539_at	0	0.22
N_212188_at	0	0.205
N_214118_x_at	0	0.225
N_200738_s_at	0	0.215
N_213340_s_at	0	0.205
N_213034_at	0	0.205
N_222129_at	0	0.205
N_222368_at	0	0.22