i— title: “BRCA Risk of Metastasis” author: “José Tamez-Peña” date: “Sep 7, 2017” output: word_document: fig_height: 6 fig_width: 8 toc: yes —
BRCA Analysis
library(readr)
library("epiR")
library("FRESA.CAD")
library(network)
library(GGally)
library("R.matlab")
library("gplots")
#BRCAdata <- readMat("./BRCA_2002/DMMPLCN_080423.mat")
BRCAdata <- readMat("./DMMPLCN_080423.mat")
a=as.numeric(Sys.time());
set.seed(a);
error.bar <- function(x, y, upper, lower=upper, length=0.05,...){
if(length(x) != length(y) | length(y) !=length(lower) | length(lower) != length(upper))
stop("vectors must be same length")
arrows(x,y+upper, x, y-lower, angle=90, code=3, length=length, ...)
}
roc95ci <- function(x,y,...)
{
op <- par(no.readonly = TRUE)
par(pty='s');
rout <- roc(as.vector(x),y)
ci.sp.obj <- ci.sp(rout , sensitivities=seq(0, 1, .05), boot.n=100,progress= 'none')
ci.se.obj <- ci.se(rout , specificities=seq(0, 1, .05), boot.n=100,progress= 'none')
plot(rout,grid=c(0.1, 0.1),grid.col=c("gray", "gray"),print.auc=TRUE,...)
plot(ci.sp.obj, type="s", col="gray")
plot(ci.se.obj, type="s", col="light gray")
par(op)
return (rout)
}
barPlotCiError<- function(citable,metricname,thesets,themethod,main,...)
{
colnames(citable) <- c(metricname,"lower","upper")
rownames(citable) <- rep(thesets,length(themethod))
pander::pander(citable,caption=main,round = 3)
citable <- citable[order(rep(1:length(thesets),length(themethod))),]
barmatrix <- matrix(citable[,1],length(themethod),length(thesets))
colnames(barmatrix) <- thesets
rownames(barmatrix) <- themethod
pander::pander(barmatrix,caption=main,round = 3)
barp <- barplot(barmatrix,cex.names=0.7,las=2,ylim=c(0.0,1.0),main=main,ylab=metricname,beside=TRUE,legend = themethod,...)
error.bar(barp,citable[,1],citable[,3]-citable[,1],citable[,1]-citable[,2])
return(barp)
}
summaryBRCA <- function(data)
{
sumBC <- NULL
sumBC$age <- c(mean(data$Age,na.rm=TRUE),sd(data$Age,na.rm=TRUE))
sumBC$size <- c(mean(data$size,na.rm=TRUE),sd(data$size,na.rm=TRUE))
sumBC$grade <- table(data$grade)
sumBC$ER <- table(data$er)
sumBC$type <- table(data$typeBRCA)
sumBC$t.dmfs <- c(mean(data$t.dmfs,na.rm=TRUE),sd(data$t.dmfs,na.rm=TRUE))
sumBC$e.dmfs <- table(data$e.dmfs)
sumBC$t.sos <- c(mean(data$t.sos,na.rm=TRUE),sd(data$t.sos,na.rm=TRUE))
sumBC$e.sos <- table(data$e.sos)
sumBC$ln <- table(data$ln)
cat(sprintf("Age: \t %5.1f (%4.1f)\n",sumBC$age[1],sumBC$age[2]))
cat(sprintf("Size: \t %5.1f (%4.1f)\n",sumBC$size[1],sumBC$size[2]))
cat("Grade: \t ",sumBC$grade,"\n")
cat(sprintf("ER: \t %d (%d) \n",sumBC$ER[1],sumBC$ER[2]))
cat("Type: \t ",sumBC$type,"\n")
cat(sprintf("Nodes: \t %d (%d) \n",sumBC$ln[1],sumBC$ln[2]))
cat(sprintf("DM Event:\t %d (%d) \n",sumBC$e.dmfs[1],sumBC$e.dmfs[2]))
cat(sprintf("SOS Event:\t %d (%d) \n",sumBC$e.sos[1],sumBC$e.sos[2]))
return (sumBC)
}
Data Preparation
BRCAdata2 <- BRCAdata$DMMPLCN
DataExpresion <- as.data.frame(BRCAdata2[3])
subjectsIDs <- unlist(BRCAdata2[4])
genesIDs <- unlist(BRCAdata2[5])
ngenesIDs <- gsub("-","_",genesIDs,fixed = TRUE,perl=FALSE);
ngenesIDs <- gsub("/","_",ngenesIDs,fixed = TRUE,perl=FALSE);
TgenesIDs <- unlist(BRCAdata2[6])
names(TgenesIDs) <- ngenesIDs
colnames(DataExpresion) <- subjectsIDs
rownames(DataExpresion) <- paste("N",ngenesIDs,sep="_")
#FRESA.CAD works with tranposed data frames
DataExpresionV <- as.data.frame(t(DataExpresion))
otr <- as.data.frame(BRCAdata2[7])
clinical <- otr$X1.1
subjID <- as.character(unlist(clinical$Simplified.ID))
t.dmfs <- unlist(clinical$t.dmfs)/12
e.dmfs <- unlist(clinical$e.dmfs)
t.sos <- unlist(clinical$t.sos)/12
e.sos <- unlist(clinical$e.sos)
events <- unlist(clinical$Events)
ln <- unlist(clinical$LN)
er <- unlist(clinical$ER.IHC)
lumA <- unlist(clinical$LumA.HU)
lumB <- unlist(clinical$LumB.HU)
HER2 <- unlist(clinical$Her2.HU)
Basal <- unlist(clinical$Basal.HU)
Nrmal <- unlist(clinical$Normal.HU)
typeBRCA <- unlist(clinical$which.max)
size <- unlist(clinical$Size)
sig70 <- unlist(clinical$CIN70.bin)
rsig70 <- unlist(clinical$CIN70)
otr <- as.data.frame(BRCAdata2[8])
clinical2 <- otr$X1.1
ids2 <- as.character(unlist(clinical2$Simplified.ID))
age <- as.vector(clinical2$Age)
names(age) <- ids2
grade <- as.vector(clinical2$Grade)
names(grade) <- ids2
ER2 <- as.vector(clinical2$ER)
names(ER2) <- ids2
PGR <- as.vector(clinical2$PGR)
names(PGR) <- ids2
Node <- as.vector(clinical2$Node)
names(Node) <- ids2
#plot(unlist(clinical$CIN70),unlist(clinical$CIN25))
#table(unlist(clinical$CIN70.bin),unlist(clinical$Events))
#table(unlist(clinical$CIN25.bin),unlist(clinical$Events))
#table(unlist(clinical$CIN25.bin),unlist(clinical$CIN70.bin))
#Small data frame with clinical info:
clicalDF <- data.frame(t.dmfs,e.dmfs,t.sos,e.sos,events,ln,er,typeBRCA,size)
rownames(clicalDF) <- subjID
clicalDF$Age <- age[subjID]
clicalDF$grade <- grade[subjID]
clicalDF$PGR <- PGR[subjID]
smry <- summaryBRCA(clicalDF)
#> Age: 56.6 (13.7)
#> Size: 25.3 (13.6)
#> Grade: 158 358 277
#> ER: 200 (581)
#> Type: 187 259 179 241 92
#> Nodes: 527 (249)
#> DM Event: 374 (135)
#> SOS Event: 399 (111)
#FRESA.CAD does not like na.
settozero <- is.nan(as.matrix(DataExpresionV)) | is.na(as.matrix(DataExpresionV))
DataExpresionV[settozero] <- 0;
# Events are DM events in less than five years
Event <- 1*((t.dmfs<5)&(e.dmfs==1))
# Censor events greater than five years
c.t.dmfs <- (t.dmfs>5)&(e.dmfs==1)
#Set the event column
DataExpresionV$Event <- 1*Event
#The Subjects with no event information
included <- !is.na(Event)
#Lets get the subjects with overall LOGITival data
#Mark events that are less than five years
LOGITEvent <- 1*(as.vector((t.sos<5)*e.sos))
LOGITExclude <- as.vector(is.na(t.sos) | !is.na(Event))
DataExpresionLOGIT <- DataExpresionV
DataExpresionLOGIT$Event <- LOGITEvent
DataExpresionLOGIT <- DataExpresionLOGIT[!LOGITExclude,]
DataExpresionLOGIT$ct.dmfs <- 0;
DataExpresionLOGIT$ct.sos <- t.sos[!LOGITExclude]
clicalDF$SEvent <- LOGITEvent
clicalDF$DMEvent <- Event
LOGITD <- clicalDF[!LOGITExclude,]
sum(LOGITD$SEvent)
#> [1] 58
DataExpresionV <- DataExpresionV[included,]
LOGITM <- clicalDF[included,]
c.t.dmfsV <- c.t.dmfs[included]
c.t.dmfsV[is.na(c.t.dmfsV)] <- FALSE
ct.dmfsV <- t.dmfs[included]
ct.dmfsV[c.t.dmfsV] <- 5
LOGITM$ct.dmfsV <- ct.dmfsV
DataExpresionV$ct.dmfs <-ct.dmfsV
sum(DataExpresionV$Event)
#> [1] 101
#DataExpresionV$ER <- clicalDF[rownames(DataExpresionV),"er"]
#DataExpresionV$ER[is.nan(DataExpresionV$ER)] <- 0
#DataExpresionV$LN <- clicalDF[rownames(DataExpresionV),"ln"]
#DataExpresionV$LN[is.nan(DataExpresionV$LN)] <- 0
# We have four set of trials. I'll divde the sets
selectedSub <- rownames(DataExpresionV)
set1 <- 1:147
set2 <- 148:227
set3 <- 228:392
set4 <- 393:514
#selectedSub[set1]
#selectedSub[set2]
#selectedSub[set3]
#selectedSub[set4]
# This variables will store the results
ROCTable <- NULL
epiTable <- NULL
signatures <- NULL
controlDistances <- NULL
caseDistances <- NULL
modSignatures <- NULL
KNNpredict <- NULL
Summary of data sets
smry <- summaryBRCA(clicalDF) Age: 56.6 (13.7) Size: 25.3 (13.6) Grade: 158 358 277 ER: 200 (581) Type: 187 259 179 241 92 Nodes: 527 (249) DM Event: 374 (135) SOS Event: 399 (111)
smry <- summaryBRCA(LOGITM) Age: 53.9 (12.8) Size: 26.1 (13.4) Grade: 62 174 158 ER: 157 (351) Type: 109 136 78 144 47 Nodes: 358 (151) DM Event: 374 (135) SOS Event: 94 (28)
smry <- summaryBRCA(LOGITM[-set1,]) Age: 56.9 (13.3) Size: 27.3 (15.0) Grade: 38 116 93 ER: 106 (255) Type: 72 101 58 103 33 Nodes: 211 (151) DM Event: 275 (87) SOS Event: 94 (28)
smry <- summaryBRCA(LOGITM[-set2,]) Age: 53.6 (12.6) Size: 24.1 (11.4) Grade: 62 174 158 ER: 121 (307) Type: 95 117 66 117 39 Nodes: 330 (99) DM Event: 320 (109) SOS Event: 94 (28)
smry <- summaryBRCA(LOGITM[-set3,]) Age: 51.5 (12.7) Size: 27.5 (13.8) Grade: 38 101 126 ER: 130 (219) Type: 75 91 45 106 32 Nodes: 229 (120) DM Event: 250 (99) SOS Event: 94 (28)
smry <- summaryBRCA(LOGITM[-set4,]) Age: 53.6 (12.0) Size: 26.1 (13.4) Grade: 48 131 97 ER: 114 (272) Type: 85 99 65 106 37 Nodes: 304 (83) DM Event: 277 (110) SOS Event: NA (NA)
smry <- summaryBRCA(LOGITD) Age: 62.5 (14.0) Size: 22.2 (10.5) Grade: 88 176 111 ER: 31 (197) Type: 70 110 86 82 40 Nodes: 147 (76) DM Event: NA (NA) SOS Event: 305 (83)
roc.signature70 <- roc95ci(DataExpresionLOGIT$Event,rsig70[!LOGITExclude],main="70 Signature")
epi.signature70 <- epi.tests(table(sig70[!LOGITExclude]==0,!DataExpresionLOGIT$Event))
pander::pander(epi.signature70$tab,caption="70 Signature")| Outcome + | Outcome - | Total | |
|---|---|---|---|
| Test + | 43 | 151 | 194 |
| Test - | 15 | 179 | 194 |
| Total | 58 | 330 | 388 |
pander::pander(summary(epi.signature70),caption="Diangostic Summary 70 Signature")| est | lower | upper | |
|---|---|---|---|
| aprev | 0.5 | 0.4491 | 0.5509 |
| tprev | 0.1495 | 0.1155 | 0.1889 |
| se | 0.7414 | 0.6096 | 0.8474 |
| sp | 0.5424 | 0.487 | 0.5971 |
| diag.acc | 0.5722 | 0.5213 | 0.622 |
| diag.or | 3.398 | 1.816 | 6.357 |
| nnd | 3.524 | 2.249 | 10.36 |
| youden | 0.2838 | 0.09655 | 0.4445 |
| ppv | 0.2216 | 0.1653 | 0.2867 |
| npv | 0.9227 | 0.8757 | 0.9561 |
| plr | 1.62 | 1.337 | 1.963 |
| nlr | 0.4768 | 0.305 | 0.7454 |
FRESA.CAD Models
# Model Parameters
Folds <- 5
Repeats <- 5
filter <- 0.005
# Model 1
trainSet <- DataExpresionV[-set1,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
trainSet$ct.dmfs <- NULL
filename = paste("BRCASignatureLogitT1",Folds,Repeats,sprintf("%5.4f",filter),".RDATA",sep="_")
system.time(BRCASignatureLOGITT1 <- FRESA.Model(formula = Event ~ 1,trainSet, CVfolds=Folds, repeats=Repeats,filter.p.value=filter))
save(BRCASignatureLOGITT1,file=filename)
#load(filename)
# Model 2
trainSet <- DataExpresionV[-set2,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
trainSet$ct.dmfs <- NULL
filename = paste("BRCASignatureLOGITT2",Folds,Repeats,sprintf("%5.4f",filter),".RDATA",sep="_")
system.time(BRCASignatureLOGITT2 <- FRESA.Model(formula = Event ~ 1,trainSet, CVfolds=Folds, repeats=Repeats,filter.p.value=filter ))
save(BRCASignatureLOGITT2,file=filename)
#load(filename)
# Model 3
trainSet <- DataExpresionV[-set3,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
trainSet$ct.dmfs <- NULL
filename = paste("BRCASignatureLOGITT3",Folds,Repeats,sprintf("%5.4f",filter),".RDATA",sep="_")
system.time(BRCASignatureLOGITT3 <- FRESA.Model(formula = Event ~ 1,trainSet, CVfolds=Folds, repeats=Repeats,filter.p.value=filter))
save(BRCASignatureLOGITT3,file=filename)
#load(filename)
# Model 4
trainSet <- DataExpresionV[-set4,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
trainSet$ct.dmfs <- NULL
filename = paste("BRCASignatureLOGITT4",Folds,Repeats,sprintf("%5.4f",filter),".RDATA",sep="_")
system.time(BRCASignatureLOGITT4 <- FRESA.Model(formula = Event ~ 1,trainSet, CVfolds=Folds, repeats=Repeats,filter.p.value=filter))
save(BRCASignatureLOGITT4,file=filename)
#load(filename)
# Model All
trainSet <- DataExpresionV
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
trainSet$ct.dmfs <- NULL
filename = paste("BRCASignatureLOGITAll",Folds,Repeats,sprintf("%5.4f",filter),".RDATA",sep="_")
system.time(BRCASignatureLOGITALL <- FRESA.Model(formula = Event ~ 1,trainSet, CVfolds=Folds, repeats=Repeats,filter.p.value=filter))
save(BRCASignatureLOGITALL,file=filename)
#load(filename)
Univariate LOGIT Model Analysis
testSet <- DataExpresionV[set1,]
trainSet <- DataExpresionV[-set1,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
#nk is the number of neighbors for KNN
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT1$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT1$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_T1 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_T1 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_T1$binProb,main="Top Univariate Features (T1)")
epiTable$UnivariateRAWKNN_T1 <- epi.tests(table(KNNpredict$UnivariateRAW_T1$binProb<0.5,!testSet$Event))
testSet <- DataExpresionV[set2,]
trainSet <- DataExpresionV[-set2,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT2$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT2$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_T2 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_T2 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_T2$binProb,main="Top Univariate Features (T2)")
epiTable$UnivariateRAWKNN_T2 <- epi.tests(table(KNNpredict$UnivariateRAW_T2$binProb<0.5,!testSet$Event))
testSet <- DataExpresionV[set3,]
trainSet <- DataExpresionV[-set3,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT3$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT3$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_T3 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_T3 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_T3$binProb,main="Top Univariate Features (T3)")
epiTable$UnivariateRAWKNN_T3 <- epi.tests(table(KNNpredict$UnivariateRAW_T3$binProb<0.5,!testSet$Event))
testSet <- DataExpresionV[set4,]
trainSet <- DataExpresionV[-set4,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT4$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT4$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_T4 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_T4 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_T4$binProb,main="Top Univariate Features (T4)")
epiTable$UnivariateRAWKNN_T4 <- epi.tests(table(KNNpredict$UnivariateRAW_T4$binProb<0.5,!testSet$Event))Test of SOS
testSet <- DataExpresionLOGIT
trainSet <- DataExpresionV[-set1,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
#nk is the number of neighbors for KNN
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT1$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT1$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_S1 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_S1 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_S1$binProb,main="Top Univariate Features (T1)")
epiTable$UnivariateRAWKNN_S1 <- epi.tests(table(KNNpredict$UnivariateRAW_S1$binProb<0.5,!testSet$Event))
trainSet <- DataExpresionV[-set2,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT2$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT2$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_S2 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_S2 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_S2$binProb,main="Top Univariate Features (T2)")
epiTable$UnivariateRAWKNN_S2 <- epi.tests(table(KNNpredict$UnivariateRAW_S2$binProb<0.5,!testSet$Event))
trainSet <- DataExpresionV[-set3,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT3$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT3$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_S3 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_S3 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_S3$binProb,main="Top Univariate Features (T3)")
epiTable$UnivariateRAWKNN_S3 <- epi.tests(table(KNNpredict$UnivariateRAW_S3$binProb<0.5,!testSet$Event))
trainSet <- DataExpresionV[-set4,]
casesample = subset(trainSet,Event==1);
controlsample = subset(trainSet,Event==0);
minTrainSamples <- min(nrow(casesample),nrow(controlsample));
KnnTrainSet <- rbind(casesample[sample(1:nrow(casesample),minTrainSamples,replace=FALSE),],controlsample[sample(1:nrow(controlsample),minTrainSamples,replace=FALSE),])
nk=2*as.integer(sqrt(nrow(KnnTrainSet))/2+0.5)+1
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT4$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT4$univariateAnalysis$Name[qvalues<0.001])
KNNpredict$UnivariateRAW_S4 <- getKNNpredictionFromFormula(snames,KnnTrainSet,testSet,Outcome="Event",nk=nk)
ROCTable$UnivariateRAWKNN_S4 <- roc95ci(testSet$Event,KNNpredict$UnivariateRAW_S4$binProb,main="Top Univariate Features (T4)")
epiTable$UnivariateRAWKNN_S4 <- epi.tests(table(KNNpredict$UnivariateRAW_S4$binProb<0.5,!testSet$Event))The Univariate Performance Tables
auctables <- rbind(
ci(ROCTable$UnivariateRAWKNN_T1,of="auc"),
ci(ROCTable$UnivariateRAWKNN_T2,of="auc"),
ci(ROCTable$UnivariateRAWKNN_T3,of="auc"),
ci(ROCTable$UnivariateRAWKNN_T4,of="auc"),
ci(ROCTable$UnivariateRAWKNN_S1,of="auc"),
ci(ROCTable$UnivariateRAWKNN_S2,of="auc"),
ci(ROCTable$UnivariateRAWKNN_S3,of="auc"),
ci(ROCTable$UnivariateRAWKNN_S4,of="auc")
)
low <- auctables[,1]; auctables[,1] <- auctables[,2]; auctables[,2] <- low;
bplot <- barPlotCiError(auctables,"ROC AUC",c("des","min","loi","chin"),c("LOSO Test","SOS Validation"),main="Univariate Ranking AUC")
acctables <- as.matrix(rbind(
epiTable$UnivariateRAWKNN_T1$elements$diag.acc,
epiTable$UnivariateRAWKNN_T2$elements$diag.acc,
epiTable$UnivariateRAWKNN_T3$elements$diag.acc,
epiTable$UnivariateRAWKNN_T4$elements$diag.acc,
epiTable$UnivariateRAWKNN_S1$elements$diag.acc,
epiTable$UnivariateRAWKNN_S2$elements$diag.acc,
epiTable$UnivariateRAWKNN_S3$elements$diag.acc,
epiTable$UnivariateRAWKNN_S4$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("LOSO Test","SOS Validation"),main="Univariate Ranking Accuracy")
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$UnivariateRAWKNN_T1$elements$sensitivity+epiTable$UnivariateRAWKNN_T1$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_T2$elements$sensitivity+epiTable$UnivariateRAWKNN_T2$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_T3$elements$sensitivity+epiTable$UnivariateRAWKNN_T3$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_T4$elements$sensitivity+epiTable$UnivariateRAWKNN_T4$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S1$elements$sensitivity+epiTable$UnivariateRAWKNN_S1$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S2$elements$sensitivity+epiTable$UnivariateRAWKNN_S2$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S3$elements$sensitivity+epiTable$UnivariateRAWKNN_S3$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S4$elements$sensitivity+epiTable$UnivariateRAWKNN_S4$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("LOSO Test","SOS Validation"),main="Univariate Ranking Balanced Error")
FRESA.CAD Models
CV analysis
rpLasso_T1 <- plotModels.ROC(BRCASignatureLOGITT1$cvObject$LASSO.testPredictions,theCVfolds=Folds,main="CV:LASSO (T1)",cex=0.8)
rpLasso_T2 <- plotModels.ROC(BRCASignatureLOGITT2$cvObject$LASSO.testPredictions,theCVfolds=Folds,main="CV:LASSO (T2)",cex=0.8)
rpLasso_T3 <- plotModels.ROC(BRCASignatureLOGITT3$cvObject$LASSO.testPredictions,theCVfolds=Folds,main="CV:LASSO (T3)",cex=0.8)
rpLasso_T4 <- plotModels.ROC(BRCASignatureLOGITT4$cvObject$LASSO.testPredictions,theCVfolds=Folds,main="CV:LASSO (T4)",cex=0.8)
epiTable$rpLasso_T1 <- epi.tests(rpLasso_T1$predictionTable)
epiTable$rpLasso_T2 <- epi.tests(rpLasso_T2$predictionTable)
epiTable$rpLasso_T3 <- epi.tests(rpLasso_T3$predictionTable)
epiTable$rpLasso_T4 <- epi.tests(rpLasso_T4$predictionTable)
rpBagged_T1 <- plotModels.ROC(BRCASignatureLOGITT1$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Bagged",main="CV Bagged(T1)",cex=0.8)
rpBagged_T2 <- plotModels.ROC(BRCASignatureLOGITT2$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Bagged",main="CV Bagged (T2)",cex=0.8)
rpBagged_T3 <- plotModels.ROC(BRCASignatureLOGITT3$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Bagged",main="CV Bagged (T3)",cex=0.8)
rpBagged_T4 <- plotModels.ROC(BRCASignatureLOGITT4$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Bagged",main="CV Bagged (T4)",cex=0.8)
epiTable$rpBagged_T1 <- epi.tests(rpBagged_T1$predictionTable)
epiTable$rpBagged_T2 <- epi.tests(rpBagged_T2$predictionTable)
epiTable$rpBagged_T3 <- epi.tests(rpBagged_T3$predictionTable)
epiTable$rpBagged_T4 <- epi.tests(rpBagged_T4$predictionTable)
rpForward_T1 <- plotModels.ROC(BRCASignatureLOGITT1$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Forward",main="CV Forward (T1)",cex=0.8)
rpForward_T2 <- plotModels.ROC(BRCASignatureLOGITT2$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Forward",main="CV Forward (T2)",cex=0.8)
rpForward_T3 <- plotModels.ROC(BRCASignatureLOGITT3$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Forward",main="CV Forward (T3)",cex=0.8)
rpForward_T4 <- plotModels.ROC(BRCASignatureLOGITT4$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Forward",main="CV Forward (T4)",cex=0.8)
epiTable$rpForward_T1 <- epi.tests(rpForward_T1$predictionTable)
epiTable$rpForward_T2 <- epi.tests(rpForward_T2$predictionTable)
epiTable$rpForward_T3 <- epi.tests(rpForward_T3$predictionTable)
epiTable$rpForward_T4 <- epi.tests(rpForward_T4$predictionTable)
rpBackwars_T1 <- plotModels.ROC(BRCASignatureLOGITT1$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Backwards",main="CV Backwards (T1)",cex=0.8)
rpBackwars_T3 <- plotModels.ROC(BRCASignatureLOGITT3$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Backwards",main="CV Backwards (T3)",cex=0.8)
rpBackwars_T2 <- plotModels.ROC(BRCASignatureLOGITT2$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Backwards",main="CV Backwards (T2)",cex=0.8)
rpBackwars_T4 <- plotModels.ROC(BRCASignatureLOGITT4$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Backwards",main="CV Backwards (T4)",cex=0.8)
epiTable$rpBackwars_T1 <- epi.tests(rpBackwars_T1$predictionTable)
epiTable$rpBackwars_T2 <- epi.tests(rpBackwars_T2$predictionTable)
epiTable$rpBackwars_T3 <- epi.tests(rpBackwars_T3$predictionTable)
epiTable$rpBackwars_T4 <- epi.tests(rpBackwars_T4$predictionTable)
rpBSWIMS_T1 <- plotModels.ROC(BRCASignatureLOGITT1$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Prediction",main="CV B:SWIMS (T1)",cex=0.8)
rpBSWIMS_T2 <- plotModels.ROC(BRCASignatureLOGITT2$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Prediction",main="CV B:SWIMS (T2)",cex=0.8)
rpBSWIMS_T3 <- plotModels.ROC(BRCASignatureLOGITT3$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Prediction",main="CV B:SWIMS (T3)",cex=0.8)
rpBSWIMS_T4 <- plotModels.ROC(BRCASignatureLOGITT4$cvObject$Models.testPrediction,theCVfolds=Folds,predictor="Prediction",main="CV B:SWIMS (T4)",cex=0.8)
epiTable$rpBSWIMS_T1 <- epi.tests(rpBSWIMS_T1$predictionTable)
epiTable$rpBSWIMS_T2 <- epi.tests(rpBSWIMS_T2$predictionTable)
epiTable$rpBSWIMS_T3 <- epi.tests(rpBSWIMS_T3$predictionTable)
epiTable$rpBSWIMS_T4 <- epi.tests(rpBSWIMS_T4$predictionTable)
The Cross-Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$rpLasso_T1$elements$sensitivity+epiTable$rpLasso_T1$elements$specificity),
1.0-0.5*(epiTable$rpLasso_T2$elements$sensitivity+epiTable$rpLasso_T2$elements$specificity),
1.0-0.5*(epiTable$rpLasso_T3$elements$sensitivity+epiTable$rpLasso_T3$elements$specificity),
1.0-0.5*(epiTable$rpLasso_T4$elements$sensitivity+epiTable$rpLasso_T4$elements$specificity),
1.0-0.5*(epiTable$rpBagged_T1$elements$sensitivity+epiTable$rpBagged_T1$elements$specificity),
1.0-0.5*(epiTable$rpBagged_T2$elements$sensitivity+epiTable$rpBagged_T2$elements$specificity),
1.0-0.5*(epiTable$rpBagged_T3$elements$sensitivity+epiTable$rpBagged_T3$elements$specificity),
1.0-0.5*(epiTable$rpBagged_T4$elements$sensitivity+epiTable$rpBagged_T4$elements$specificity),
1.0-0.5*(epiTable$rpForward_T1$elements$sensitivity+epiTable$rpForward_T1$elements$specificity),
1.0-0.5*(epiTable$rpForward_T2$elements$sensitivity+epiTable$rpForward_T2$elements$specificity),
1.0-0.5*(epiTable$rpForward_T3$elements$sensitivity+epiTable$rpForward_T3$elements$specificity),
1.0-0.5*(epiTable$rpForward_T4$elements$sensitivity+epiTable$rpForward_T4$elements$specificity),
1.0-0.5*(epiTable$rpBackwars_T1$elements$sensitivity+epiTable$rpBackwars_T1$elements$specificity),
1.0-0.5*(epiTable$rpBackwars_T2$elements$sensitivity+epiTable$rpBackwars_T2$elements$specificity),
1.0-0.5*(epiTable$rpBackwars_T3$elements$sensitivity+epiTable$rpBackwars_T3$elements$specificity),
1.0-0.5*(epiTable$rpBackwars_T4$elements$sensitivity+epiTable$rpBackwars_T4$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T1$elements$sensitivity+epiTable$rpBSWIMS_T1$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T2$elements$sensitivity+epiTable$rpBSWIMS_T2$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T3$elements$sensitivity+epiTable$rpBSWIMS_T3$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T4$elements$sensitivity+epiTable$rpBSWIMS_T4$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS"),main="Cross Validation Balanced Error")
acctables <- as.matrix(rbind(
epiTable$rpLasso_T1$elements$diag.acc,
epiTable$rpLasso_T2$elements$diag.acc,
epiTable$rpLasso_T3$elements$diag.acc,
epiTable$rpLasso_T4$elements$diag.acc,
epiTable$rpBagged_T1$elements$diag.acc,
epiTable$rpBagged_T2$elements$diag.acc,
epiTable$rpBagged_T3$elements$diag.acc,
epiTable$rpBagged_T4$elements$diag.acc,
epiTable$rpForward_T1$elements$diag.acc,
epiTable$rpForward_T2$elements$diag.acc,
epiTable$rpForward_T3$elements$diag.acc,
epiTable$rpForward_T4$elements$diag.acc,
epiTable$rpBackwars_T1$elements$diag.acc,
epiTable$rpBackwars_T2$elements$diag.acc,
epiTable$rpBackwars_T3$elements$diag.acc,
epiTable$rpBackwars_T4$elements$diag.acc,
epiTable$rpBSWIMS_T1$elements$diag.acc,
epiTable$rpBSWIMS_T2$elements$diag.acc,
epiTable$rpBSWIMS_T3$elements$diag.acc,
epiTable$rpBSWIMS_T4$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS"),main="Cross Validation Accuracy")
FRESA.CAD Full Models
trainSet <- DataExpresionV[-set1,]
testSet <- DataExpresionV[set1,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT1$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT1$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_T1 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T1)")
epiTable$FULLLASSOLOGIT_T1 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT1$bagging$bagged.model,testSet)
ROCTable$FULLBagged_T1 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T1)")
epiTable$FULLBagged_T1 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT1$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_T1 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T1)")
epiTable$FULLForward_T1 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT1$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_T1 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T1)")
epiTable$FULLBackwards_T1 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT1$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_T1 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T1)")
epiTable$FULLBSWiMS_T1 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT1$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_T1 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T1)")
epiTable$FULLeBSWiMS_T1 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set2,]
testSet <- DataExpresionV[set2,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT2$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT2$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_T2 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T2)")
epiTable$FULLLASSOLOGIT_T2 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT2$bagging$bagged.model,testSet)
ROCTable$FULLBagged_T2 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T2)")
epiTable$FULLBagged_T2 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT2$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_T2 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T2)")
epiTable$FULLForward_T2 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT2$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_T2 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T2)")
epiTable$FULLBackwards_T2 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT2$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_T2 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T2)")
epiTable$FULLBSWiMS_T2 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT2$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_T2 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T2)")
epiTable$FULLeBSWiMS_T2 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set3,]
testSet <- DataExpresionV[set3,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT3$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT3$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_T3 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T3)")
epiTable$FULLLASSOLOGIT_T3 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT3$bagging$bagged.model,testSet)
ROCTable$FULLBagged_T3 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T3)")
epiTable$FULLBagged_T3 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT3$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_T3 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T3)")
epiTable$FULLForward_T3 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT3$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_T3 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T3)")
epiTable$FULLBackwards_T3 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT3$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_T3 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T3)")
epiTable$FULLBSWiMS_T3 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT3$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_T3 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T3)")
epiTable$FULLeBSWiMS_T3 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set4,]
testSet <- DataExpresionV[set4,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT4$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT4$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_T4 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T4)")
epiTable$FULLLASSOLOGIT_T4 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT4$bagging$bagged.model,testSet)
ROCTable$FULLBagged_T4 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T4)")
epiTable$FULLBagged_T4 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT4$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_T4 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T4)")
epiTable$FULLForward_T4 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT4$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_T4 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T4)")
epiTable$FULLBackwards_T4 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT4$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_T4 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T4)")
epiTable$FULLBSWiMS_T4 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT4$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_T4 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T4)")
epiTable$FULLeBSWiMS_T4 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))The Test-Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$FULLLASSOLOGIT_T1$elements$sensitivity+epiTable$FULLLASSOLOGIT_T1$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_T2$elements$sensitivity+epiTable$FULLLASSOLOGIT_T2$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_T3$elements$sensitivity+epiTable$FULLLASSOLOGIT_T3$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_T4$elements$sensitivity+epiTable$FULLLASSOLOGIT_T4$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_T1$elements$sensitivity+epiTable$FULLBagged_T1$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_T2$elements$sensitivity+epiTable$FULLBagged_T2$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_T3$elements$sensitivity+epiTable$FULLBagged_T3$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_T4$elements$sensitivity+epiTable$FULLBagged_T4$elements$specificity),
1.0-0.5*(epiTable$FULLForward_T1$elements$sensitivity+epiTable$FULLForward_T1$elements$specificity),
1.0-0.5*(epiTable$FULLForward_T2$elements$sensitivity+epiTable$FULLForward_T2$elements$specificity),
1.0-0.5*(epiTable$FULLForward_T3$elements$sensitivity+epiTable$FULLForward_T3$elements$specificity),
1.0-0.5*(epiTable$FULLForward_T4$elements$sensitivity+epiTable$FULLForward_T4$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_T1$elements$sensitivity+epiTable$FULLBackwards_T1$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_T2$elements$sensitivity+epiTable$FULLBackwards_T2$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_T3$elements$sensitivity+epiTable$FULLBackwards_T3$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_T4$elements$sensitivity+epiTable$FULLBackwards_T4$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_T1$elements$sensitivity+epiTable$FULLBSWiMS_T1$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_T2$elements$sensitivity+epiTable$FULLBSWiMS_T2$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_T3$elements$sensitivity+epiTable$FULLBSWiMS_T3$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_T4$elements$sensitivity+epiTable$FULLBSWiMS_T4$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_T1$elements$sensitivity+epiTable$FULLeBSWiMS_T1$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_T2$elements$sensitivity+epiTable$FULLeBSWiMS_T2$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_T3$elements$sensitivity+epiTable$FULLeBSWiMS_T3$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_T4$elements$sensitivity+epiTable$FULLeBSWiMS_T4$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="LOSO Test Balanced Error")
acctables <- as.matrix(rbind(
epiTable$FULLLASSOLOGIT_T1$elements$diag.acc,
epiTable$FULLLASSOLOGIT_T2$elements$diag.acc,
epiTable$FULLLASSOLOGIT_T3$elements$diag.acc,
epiTable$FULLLASSOLOGIT_T4$elements$diag.acc,
epiTable$FULLBagged_T1$elements$diag.acc,
epiTable$FULLBagged_T2$elements$diag.acc,
epiTable$FULLBagged_T3$elements$diag.acc,
epiTable$FULLBagged_T4$elements$diag.acc,
epiTable$FULLForward_T1$elements$diag.acc,
epiTable$FULLForward_T2$elements$diag.acc,
epiTable$FULLForward_T3$elements$diag.acc,
epiTable$FULLForward_T4$elements$diag.acc,
epiTable$FULLBackwards_T1$elements$diag.acc,
epiTable$FULLBackwards_T2$elements$diag.acc,
epiTable$FULLBackwards_T3$elements$diag.acc,
epiTable$FULLBackwards_T4$elements$diag.acc,
epiTable$FULLBSWiMS_T1$elements$diag.acc,
epiTable$FULLBSWiMS_T2$elements$diag.acc,
epiTable$FULLBSWiMS_T3$elements$diag.acc,
epiTable$FULLBSWiMS_T4$elements$diag.acc,
epiTable$FULLeBSWiMS_T1$elements$diag.acc,
epiTable$FULLeBSWiMS_T2$elements$diag.acc,
epiTable$FULLeBSWiMS_T3$elements$diag.acc,
epiTable$FULLeBSWiMS_T4$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="LOSO Test Validation Accuracy")
sentables <- as.matrix(rbind(
epiTable$FULLLASSOLOGIT_T1$elements$sensitivity,
epiTable$FULLLASSOLOGIT_T2$elements$sensitivity,
epiTable$FULLLASSOLOGIT_T3$elements$sensitivity,
epiTable$FULLLASSOLOGIT_T4$elements$sensitivity,
epiTable$FULLBagged_T1$elements$sensitivity,
epiTable$FULLBagged_T2$elements$sensitivity,
epiTable$FULLBagged_T3$elements$sensitivity,
epiTable$FULLBagged_T4$elements$sensitivity,
epiTable$FULLForward_T1$elements$sensitivity,
epiTable$FULLForward_T2$elements$sensitivity,
epiTable$FULLForward_T3$elements$sensitivity,
epiTable$FULLForward_T4$elements$sensitivity,
epiTable$FULLBackwards_T1$elements$sensitivity,
epiTable$FULLBackwards_T2$elements$sensitivity,
epiTable$FULLBackwards_T3$elements$sensitivity,
epiTable$FULLBackwards_T4$elements$sensitivity,
epiTable$FULLBSWiMS_T1$elements$sensitivity,
epiTable$FULLBSWiMS_T2$elements$sensitivity,
epiTable$FULLBSWiMS_T3$elements$sensitivity,
epiTable$FULLBSWiMS_T4$elements$sensitivity,
epiTable$FULLeBSWiMS_T1$elements$sensitivity,
epiTable$FULLeBSWiMS_T2$elements$sensitivity,
epiTable$FULLeBSWiMS_T3$elements$sensitivity,
epiTable$FULLeBSWiMS_T4$elements$sensitivity
))
bplot <- barPlotCiError(sentables,"Sensitivity",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="LOSO Test Validation Sensitivity",args.legend = list(x = "bottomright"))
Validation SOS
testSet <- DataExpresionLOGIT
trainSet <- DataExpresionV[-set1,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT1$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT1$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_S1 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T1)")
epiTable$FULLLASSOLOGIT_S1 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT1$bagging$bagged.model,testSet)
ROCTable$FULLBagged_S1 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T1)")
epiTable$FULLBagged_S1 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT1$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_S1 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T1)")
epiTable$FULLForward_S1 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT1$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_S1 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T1)")
epiTable$FULLBackwards_S1 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT1$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_S1 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T1)")
epiTable$FULLBSWiMS_S1 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT1$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_S1 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T1)")
epiTable$FULLeBSWiMS_S1 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set2,]
#removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
#trainSet <- trainSet[!removeLessthan5,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT2$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT2$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_S2 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T2)")
epiTable$FULLLASSOLOGIT_S2 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT2$bagging$bagged.model,testSet)
ROCTable$FULLBagged_S2 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T2)")
epiTable$FULLBagged_S2 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT2$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_S2 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T2)")
epiTable$FULLForward_S2 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT2$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_S2 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T2)")
epiTable$FULLBackwards_S2 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT2$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_S2 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T2)")
epiTable$FULLBSWiMS_S2 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT2$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_S2 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T2)")
epiTable$FULLeBSWiMS_S2 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set3,]
#removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
#trainSet <- trainSet[!removeLessthan5,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT3$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT3$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_S3 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T3)")
epiTable$FULLLASSOLOGIT_S3 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT3$bagging$bagged.model,testSet)
ROCTable$FULLBagged_S3 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T3)")
epiTable$FULLBagged_S3 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT3$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_S3 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T3)")
epiTable$FULLForward_S3 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT3$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_S3 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T3)")
epiTable$FULLBackwards_S3 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT3$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_S3 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T3)")
epiTable$FULLBSWiMS_S3 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT3$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_S3 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T3)")
epiTable$FULLeBSWiMS_S3 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
trainSet <- DataExpresionV[-set4,]
#removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
#trainSet <- trainSet[!removeLessthan5,]
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT4$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT4$univariateAnalysis)
tempPredict <- predict(FULLLASSOLOGIT$bagged.model,testSet)
ROCTable$FULLLASSOLOGIT_S4 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T4)")
epiTable$FULLLASSOLOGIT_S4 <- epi.tests(table(tempPredict<0,!testSet$Event))
FULLBagged <- predict(BRCASignatureLOGITT4$bagging$bagged.model,testSet)
ROCTable$FULLBagged_S4 <- roc95ci(testSet$Event,FULLBagged,main="Bagged Model (T4)")
epiTable$FULLBagged_S4 <- epi.tests(table(FULLBagged<0,!testSet$Event))
FULLForward <- predict(BRCASignatureLOGITT4$updatedforwardModel$final.model,testSet)
ROCTable$FULLForward_S4 <- roc95ci(testSet$Event,FULLForward,main="Forward Model (T4)")
epiTable$FULLForward_S4 <- epi.tests(table(FULLForward<0,!testSet$Event))
FULLBackwards <- predict(BRCASignatureLOGITT4$reducedModel$at.opt.model,testSet)
ROCTable$FULLBackwards_S4 <- roc95ci(testSet$Event,FULLBackwards,main="Backwards Model (T4)")
epiTable$FULLBackwards_S4 <- epi.tests(table(FULLBackwards<0,!testSet$Event))
FULLBSWiMS <- predict(BRCASignatureLOGITT4$BSWiMS.model,testSet)
ROCTable$FULLBSWiMS_S4 <- roc95ci(testSet$Event,FULLBSWiMS,main="BSWiMS Model (T4)")
epiTable$FULLBSWiMS_S4 <- epi.tests(table(FULLBSWiMS<0,!testSet$Event))
FULLeBSWiMS <- predict(BRCASignatureLOGITT4$eBSWiMS.model$equivalentModel,testSet)
ROCTable$FULLeBSWiMS_S4 <- roc95ci(testSet$Event,FULLeBSWiMS,main="eBSWiMS Model (T4)")
epiTable$FULLeBSWiMS_S4 <- epi.tests(table(FULLeBSWiMS<0,!testSet$Event))
The Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$FULLLASSOLOGIT_S1$elements$sensitivity+epiTable$FULLLASSOLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S2$elements$sensitivity+epiTable$FULLLASSOLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S3$elements$sensitivity+epiTable$FULLLASSOLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S4$elements$sensitivity+epiTable$FULLLASSOLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_S1$elements$sensitivity+epiTable$FULLBagged_S1$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_S2$elements$sensitivity+epiTable$FULLBagged_S2$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_S3$elements$sensitivity+epiTable$FULLBagged_S3$elements$specificity),
1.0-0.5*(epiTable$FULLBagged_S4$elements$sensitivity+epiTable$FULLBagged_S4$elements$specificity),
1.0-0.5*(epiTable$FULLForward_S1$elements$sensitivity+epiTable$FULLForward_S1$elements$specificity),
1.0-0.5*(epiTable$FULLForward_S2$elements$sensitivity+epiTable$FULLForward_S2$elements$specificity),
1.0-0.5*(epiTable$FULLForward_S3$elements$sensitivity+epiTable$FULLForward_S3$elements$specificity),
1.0-0.5*(epiTable$FULLForward_S4$elements$sensitivity+epiTable$FULLForward_S4$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_S1$elements$sensitivity+epiTable$FULLBackwards_S1$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_S2$elements$sensitivity+epiTable$FULLBackwards_S2$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_S3$elements$sensitivity+epiTable$FULLBackwards_S3$elements$specificity),
1.0-0.5*(epiTable$FULLBackwards_S4$elements$sensitivity+epiTable$FULLBackwards_S4$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S1$elements$sensitivity+epiTable$FULLBSWiMS_S1$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S2$elements$sensitivity+epiTable$FULLBSWiMS_S2$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S3$elements$sensitivity+epiTable$FULLBSWiMS_S3$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S4$elements$sensitivity+epiTable$FULLBSWiMS_S4$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_S1$elements$sensitivity+epiTable$FULLeBSWiMS_S1$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_S2$elements$sensitivity+epiTable$FULLeBSWiMS_S2$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_S3$elements$sensitivity+epiTable$FULLeBSWiMS_S3$elements$specificity),
1.0-0.5*(epiTable$FULLeBSWiMS_S4$elements$sensitivity+epiTable$FULLeBSWiMS_S4$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="SOS Validation Error",args.legend = list(x = "topright"))
acctables <- as.matrix(rbind(
epiTable$FULLLASSOLOGIT_S1$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S2$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S3$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S4$elements$diag.acc,
epiTable$FULLBagged_S1$elements$diag.acc,
epiTable$FULLBagged_S2$elements$diag.acc,
epiTable$FULLBagged_S3$elements$diag.acc,
epiTable$FULLBagged_S4$elements$diag.acc,
epiTable$FULLForward_S1$elements$diag.acc,
epiTable$FULLForward_S2$elements$diag.acc,
epiTable$FULLForward_S3$elements$diag.acc,
epiTable$FULLForward_S4$elements$diag.acc,
epiTable$FULLBackwards_S1$elements$diag.acc,
epiTable$FULLBackwards_S2$elements$diag.acc,
epiTable$FULLBackwards_S3$elements$diag.acc,
epiTable$FULLBackwards_S4$elements$diag.acc,
epiTable$FULLBSWiMS_S1$elements$diag.acc,
epiTable$FULLBSWiMS_S2$elements$diag.acc,
epiTable$FULLBSWiMS_S3$elements$diag.acc,
epiTable$FULLBSWiMS_S4$elements$diag.acc,
epiTable$FULLeBSWiMS_S1$elements$diag.acc,
epiTable$FULLeBSWiMS_S2$elements$diag.acc,
epiTable$FULLeBSWiMS_S3$elements$diag.acc,
epiTable$FULLeBSWiMS_S4$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="SOS Validation Accuracy",args.legend = list(x = "topright"))
sentables <- as.matrix(rbind(
epiTable$FULLLASSOLOGIT_S1$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S2$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S3$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S4$elements$sensitivity,
epiTable$FULLBagged_S1$elements$sensitivity,
epiTable$FULLBagged_S2$elements$sensitivity,
epiTable$FULLBagged_S3$elements$sensitivity,
epiTable$FULLBagged_S4$elements$sensitivity,
epiTable$FULLForward_S1$elements$sensitivity,
epiTable$FULLForward_S2$elements$sensitivity,
epiTable$FULLForward_S3$elements$sensitivity,
epiTable$FULLForward_S4$elements$sensitivity,
epiTable$FULLBackwards_S1$elements$sensitivity,
epiTable$FULLBackwards_S2$elements$sensitivity,
epiTable$FULLBackwards_S3$elements$sensitivity,
epiTable$FULLBackwards_S4$elements$sensitivity,
epiTable$FULLBSWiMS_S1$elements$sensitivity,
epiTable$FULLBSWiMS_S2$elements$sensitivity,
epiTable$FULLBSWiMS_S3$elements$sensitivity,
epiTable$FULLBSWiMS_S4$elements$sensitivity,
epiTable$FULLeBSWiMS_S1$elements$sensitivity,
epiTable$FULLeBSWiMS_S2$elements$sensitivity,
epiTable$FULLeBSWiMS_S3$elements$sensitivity,
epiTable$FULLeBSWiMS_S4$elements$sensitivity
))
bplot <- barPlotCiError(sentables,"Sensitivity",c("des","min","loi","chin"),c("LASSO","Bagged","Forward","Backwards","B:SWiMS","eB:SWiMS"),main="SOS Validation Sensitivity",args.legend = list(x = "bottomright"))
Cross Validation Bagged Models
testSet <- DataExpresionLOGIT
# Model 1
trainSet <- DataExpresionV[-set1,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
KNNtrainSet <- trainSet[!removeLessthan5,]
LASSOlist <- BRCASignatureLOGITT1$cvObject$LASSOVariables[-1]
BSWiWMlist <- BRCASignatureLOGITT1$cvObject$formula.list
## LASSO
### Bagged prediction
flist <- BRCASignatureLOGITT1$cvObject$LASSOVariables[-1]
CVLASSOLOGIT_S1 <-baggedModel(LASSOlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT1$univariateAnalysis)
tempPredict <- predict(CVLASSOLOGIT_S1$bagged.model,testSet)
ROCTable$CVLASSOLOGIT_S1 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T1)")
epiTable$CVLASSOLOGIT_S1 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVLASSOLOGIT_S1 <- medianPredict(LASSOlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVLASSOLOGIT_S1 <- roc95ci(testSet$Event,mpCVLASSOLOGIT_S1$medianKNNPredict,main="CV LASSO-KNN Features (T1)")
epiTable$KNNCVLASSOLOGIT_S1 <- epi.tests(table(mpCVLASSOLOGIT_S1$medianKNNPredict<0.5,!testSet$Event))
## BSWiMS
### Bagged prediction
CVBSWiMSLOGIT_S1 <-baggedModel(BSWiWMlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT1$univariateAnalysis)
tempPredict <- predict(CVBSWiMSLOGIT_S1$bagged.model,testSet)
ROCTable$CVBSWiMSLOGIT_S1 <- roc95ci(testSet$Event,tempPredict,main="CV B:SWiMS Model (T1)")
epiTable$CVBSWiMSLOGIT_S1 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVBSWiMSLOGIT_S1 <- medianPredict(BSWiWMlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVBSWiMSLOGIT_S1 <- roc95ci(testSet$Event,mpCVBSWiMSLOGIT_S1$medianKNNPredict,main="CV B:SWiMS-KNN Features (T1)")
epiTable$KNNCVBSWiMSLOGIT_S1 <- epi.tests(table(mpCVBSWiMSLOGIT_S1$medianKNNPredict<0.5,!testSet$Event))
# Model 2
trainSet <- DataExpresionV[-set2,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
KNNtrainSet <- trainSet[!removeLessthan5,]
LASSOlist <- BRCASignatureLOGITT2$cvObject$LASSOVariables[-1]
BSWiWMlist <- BRCASignatureLOGITT2$cvObject$formula.list
## LASSO
### Bagged prediction
CVLASSOLOGIT_S2 <-baggedModel(LASSOlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT2$univariateAnalysis)
tempPredict <- predict(CVLASSOLOGIT_S2$bagged.model,testSet)
ROCTable$CVLASSOLOGIT_S2 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T2)")
epiTable$CVLASSOLOGIT_S2 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVLASSOLOGIT_S2 <- medianPredict(LASSOlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVLASSOLOGIT_S2 <- roc95ci(testSet$Event,mpCVLASSOLOGIT_S2$medianKNNPredict,main="CV LASSO-KNN Features (T2)")
epiTable$KNNCVLASSOLOGIT_S2 <- epi.tests(table(mpCVLASSOLOGIT_S2$medianKNNPredict<0.5,!testSet$Event))
## BSWiMS
### Bagged prediction
CVBSWiMSLOGIT_S2 <-baggedModel(BSWiWMlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT2$univariateAnalysis)
tempPredict <- predict(CVBSWiMSLOGIT_S2$bagged.model,testSet)
ROCTable$CVBSWiMSLOGIT_S2 <- roc95ci(testSet$Event,tempPredict,main="CV B:SWiMS Model (T2)")
epiTable$CVBSWiMSLOGIT_S2 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVBSWiMSLOGIT_S2 <- medianPredict(BSWiWMlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVBSWiMSLOGIT_S2 <- roc95ci(testSet$Event,mpCVBSWiMSLOGIT_S2$medianKNNPredict,main="CV B:SWiMS-KNN Features (T2)")
epiTable$KNNCVBSWiMSLOGIT_S2 <- epi.tests(table(mpCVBSWiMSLOGIT_S2$medianKNNPredict<0.5,!testSet$Event))
# Model 3
trainSet <- DataExpresionV[-set3,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
KNNtrainSet <- trainSet[!removeLessthan5,]
LASSOlist <- BRCASignatureLOGITT3$cvObject$LASSOVariables[-1]
BSWiWMlist <- BRCASignatureLOGITT3$cvObject$formula.list
## LASSO
### Bagged prediction
CVLASSOLOGIT_S3 <-baggedModel(LASSOlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT3$univariateAnalysis)
tempPredict <- predict(CVLASSOLOGIT_S3$bagged.model,testSet)
ROCTable$CVLASSOLOGIT_S3 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T3)")
epiTable$CVLASSOLOGIT_S3 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVLASSOLOGIT_S3 <- medianPredict(LASSOlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVLASSOLOGIT_S3 <- roc95ci(testSet$Event,mpCVLASSOLOGIT_S3$medianKNNPredict,main="CV LASSO-KNN Features (T3)")
epiTable$KNNCVLASSOLOGIT_S3 <- epi.tests(table(mpCVLASSOLOGIT_S3$medianKNNPredict<0.5,!testSet$Event))
## BSWiMS
### Bagged prediction
CVBSWiMSLOGIT_S3 <-baggedModel(BSWiWMlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT3$univariateAnalysis)
tempPredict <- predict(CVBSWiMSLOGIT_S3$bagged.model,testSet)
ROCTable$CVBSWiMSLOGIT_S3 <- roc95ci(testSet$Event,tempPredict,main="CV B:SWiMS Model (T3)")
epiTable$CVBSWiMSLOGIT_S3 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVBSWiMSLOGIT_S3 <- medianPredict(BSWiWMlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVBSWiMSLOGIT_S3 <- roc95ci(testSet$Event,mpCVBSWiMSLOGIT_S3$medianKNNPredict,main="CV B:SWiMS-KNN Features (T3)")
epiTable$KNNCVBSWiMSLOGIT_S3 <- epi.tests(table(mpCVBSWiMSLOGIT_S3$medianKNNPredict<0.5,!testSet$Event))
# Model 4
trainSet <- DataExpresionV[-set4,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
KNNtrainSet <- trainSet[!removeLessthan5,]
LASSOlist <- BRCASignatureLOGITT4$cvObject$LASSOVariables[-1]
BSWiWMlist <- BRCASignatureLOGITT4$cvObject$formula.list
## LASSO
### Bagged prediction
CVLASSOLOGIT_S4 <-baggedModel(LASSOlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT4$univariateAnalysis)
tempPredict <- predict(CVLASSOLOGIT_S4$bagged.model,testSet)
ROCTable$CVLASSOLOGIT_S4 <- roc95ci(testSet$Event,tempPredict,main="Enet CV LASSO Model (T4)")
epiTable$CVLASSOLOGIT_S4 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVLASSOLOGIT_S4 <- medianPredict(LASSOlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVLASSOLOGIT_S4 <- roc95ci(testSet$Event,mpCVLASSOLOGIT_S4$medianKNNPredict,main="CV LASSO-KNN Features (T4)")
epiTable$KNNCVLASSOLOGIT_S4 <- epi.tests(table(mpCVLASSOLOGIT_S4$medianKNNPredict<0.5,!testSet$Event))
## BSWiMS
### Bagged prediction
CVBSWiMSLOGIT_S4 <-baggedModel(BSWiWMlist,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT4$univariateAnalysis)
ROCTable$CVBSWiMSLOGIT_S4 <- roc95ci(testSet$Event,tempPredict,main="CV B:SWiMS Model (T4)")
epiTable$CVBSWiMSLOGIT_S4 <- epi.tests(table(tempPredict<0,!testSet$Event))
### The ensemble KNN prediction
mpCVBSWiMSLOGIT_S4 <- medianPredict(BSWiWMlist,KNNtrainSet,testSet,predictType ="linear",type = "LOGIT",Outcome="Event")
ROCTable$KNNCVBSWiMSLOGIT_S4 <- roc95ci(testSet$Event,mpCVBSWiMSLOGIT_S4$medianKNNPredict,main="CV B:SWiMS-KNN Features (T4)")
epiTable$KNNCVBSWiMSLOGIT_S4 <- epi.tests(table(mpCVBSWiMSLOGIT_S4$medianKNNPredict<0.5,!testSet$Event))
The CVModels Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$CVLASSOLOGIT_S1$elements$sensitivity+epiTable$CVLASSOLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S2$elements$sensitivity+epiTable$CVLASSOLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S3$elements$sensitivity+epiTable$CVLASSOLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S4$elements$sensitivity+epiTable$CVLASSOLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S1$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S2$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S3$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S4$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$KNNCVLASSOLOGIT_S1$elements$sensitivity+epiTable$KNNCVLASSOLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$KNNCVLASSOLOGIT_S2$elements$sensitivity+epiTable$KNNCVLASSOLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$KNNCVLASSOLOGIT_S3$elements$sensitivity+epiTable$KNNCVLASSOLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$KNNCVLASSOLOGIT_S4$elements$sensitivity+epiTable$KNNCVLASSOLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$KNNCVBSWiMSLOGIT_S1$elements$sensitivity+epiTable$KNNCVBSWiMSLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$KNNCVBSWiMSLOGIT_S2$elements$sensitivity+epiTable$KNNCVBSWiMSLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$KNNCVBSWiMSLOGIT_S3$elements$sensitivity+epiTable$KNNCVBSWiMSLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$KNNCVBSWiMSLOGIT_S4$elements$sensitivity+epiTable$KNNCVBSWiMSLOGIT_S4$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("LASSO","B:SWiMS","KNN LASSO","KNN B:SWiMS"),main="CV Models: SOS Validation Error",args.legend = list(x = "topright"))
acctables <- as.matrix(rbind(
epiTable$CVLASSOLOGIT_S1$elements$diag.acc,
epiTable$CVLASSOLOGIT_S2$elements$diag.acc,
epiTable$CVLASSOLOGIT_S3$elements$diag.acc,
epiTable$CVLASSOLOGIT_S4$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S1$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S2$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S3$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S4$elements$diag.acc,
epiTable$KNNCVLASSOLOGIT_S1$elements$diag.acc,
epiTable$KNNCVLASSOLOGIT_S2$elements$diag.acc,
epiTable$KNNCVLASSOLOGIT_S3$elements$diag.acc,
epiTable$KNNCVLASSOLOGIT_S4$elements$diag.acc,
epiTable$KNNCVBSWiMSLOGIT_S1$elements$diag.acc,
epiTable$KNNCVBSWiMSLOGIT_S2$elements$diag.acc,
epiTable$KNNCVBSWiMSLOGIT_S3$elements$diag.acc,
epiTable$KNNCVBSWiMSLOGIT_S4$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("LASSO","B:SWiMS","KNN LASSO","KNN B:SWiMS"),main="CV Models: SOS Validation Accuracy",args.legend = list(x = "topright"))
sentables <- as.matrix(rbind(
epiTable$CVLASSOLOGIT_S1$elements$sensitivity,
epiTable$CVLASSOLOGIT_S2$elements$sensitivity,
epiTable$CVLASSOLOGIT_S3$elements$sensitivity,
epiTable$CVLASSOLOGIT_S4$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S1$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S2$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S3$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S4$elements$sensitivity,
epiTable$KNNCVLASSOLOGIT_S1$elements$sensitivity,
epiTable$KNNCVLASSOLOGIT_S2$elements$sensitivity,
epiTable$KNNCVLASSOLOGIT_S3$elements$sensitivity,
epiTable$KNNCVLASSOLOGIT_S4$elements$sensitivity,
epiTable$KNNCVBSWiMSLOGIT_S1$elements$sensitivity,
epiTable$KNNCVBSWiMSLOGIT_S2$elements$sensitivity,
epiTable$KNNCVBSWiMSLOGIT_S3$elements$sensitivity,
epiTable$KNNCVBSWiMSLOGIT_S4$elements$sensitivity
))
bplot <- barPlotCiError(sentables,"Sensitivity",c("des","min","loi","chin"),c("LASSO","B:SWiMS","KNN LASSO","KNN B:SWiMS"),main="CV Models: SOS Validation Sensitivity",args.legend = list(x = "bottomright"))
Correlation Signatures
All Signatures Validated on SOS
testSet <- DataExpresionLOGIT
# Model 1
trainSet <- DataExpresionV[-set1,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
## Univariate
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT1$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT1$univariateAnalysis$Name[qvalues<0.01])
snames <- snames[1:150]
system.time(modSignatures$UniSig_S1 <- getSignature(data=trainSet,varlist=snames,Outcome="Event",method="spearman"))
Univaraitedistance <- signatureDistance(modSignatures$UniSig_S1$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$UniSig_S1$controlTemplate,testSet,"spearman")
ROCTable$UniSig_S1 <- roc95ci(as.vector(testSet$Event),Univaraitedistance,main="Univariate LOGIT Rank Signature (T1)") 
epiTable$UniSig_S1 <- epi.tests(table(Univaraitedistance>0,!testSet$Event))
## CV LASSO
system.time(modSignatures$LassoSig_S1 <- getSignature(data=trainSet,varlist=names(CVLASSOLOGIT_S1$frequencyTable[CVLASSOLOGIT_S1$frequencyTable>1]),Outcome="Event",method="spearman"))
Lassodistance <- signatureDistance(modSignatures$LassoSig_S1$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$LassoSig_S1$controlTemplate,testSet,"spearman")
ROCTable$LassoSig_S1 <- roc95ci(as.vector(testSet$Event),Lassodistance,main="CV LASSO Signature (T1)") 
epiTable$LassoSig_S1 <- epi.tests(table(Lassodistance>0,!testSet$Event))
## CV B:SWiMS
system.time(modSignatures$BSWIMSSig_S1 <- getSignature(data=trainSet,varlist=names(CVBSWiMSLOGIT_S1$frequencyTable[CVBSWiMSLOGIT_S1$frequencyTable>1]),Outcome="Event",method="spearman"))
BSWIMSdistance <- signatureDistance(modSignatures$BSWIMSSig_S1$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$BSWIMSSig_S1$controlTemplate,testSet,"spearman")
ROCTable$BSWIMSSig_S1 <- roc95ci(as.vector(testSet$Event),BSWIMSdistance,main="CV B:SWiMS Signature (T1)") 
epiTable$BSWIMSSig_S1 <- epi.tests(table(BSWIMSdistance>0,!testSet$Event))
# Model 2
trainSet <- DataExpresionV[-set2,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
## Univariate
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT2$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT2$univariateAnalysis$Name[qvalues<0.01])
snames <- snames[1:150]
system.time(modSignatures$UniSig_S2 <- getSignature(data=trainSet,varlist=snames,Outcome="Event",method="spearman"))
Univaraitedistance <- signatureDistance(modSignatures$UniSig_S2$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$UniSig_S2$controlTemplate,testSet,"spearman")
ROCTable$UniSig_S2 <- roc95ci(as.vector(testSet$Event),Univaraitedistance,main="Univariate LOGIT Rank Signature (T2)") 
epiTable$UniSig_S2 <- epi.tests(table(Univaraitedistance>0,!testSet$Event))
## CV LASSO
system.time(modSignatures$LassoSig_S2 <- getSignature(data=trainSet,varlist=names(CVLASSOLOGIT_S2$frequencyTable[CVLASSOLOGIT_S2$frequencyTable>1]),Outcome="Event",method="spearman"))
Lassodistance <- signatureDistance(modSignatures$LassoSig_S2$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$LassoSig_S2$controlTemplate,testSet,"spearman")
ROCTable$LassoSig_S2 <- roc95ci(as.vector(testSet$Event),Lassodistance,main="CV LASSO Signature (T2)") 
epiTable$LassoSig_S2 <- epi.tests(table(Lassodistance>0,!testSet$Event))
## CV B:SWiMS
system.time(modSignatures$BSWIMSSig_S2 <- getSignature(data=trainSet,varlist=names(CVBSWiMSLOGIT_S2$frequencyTable[CVBSWiMSLOGIT_S2$frequencyTable>1]),Outcome="Event",method="spearman"))
BSWIMSdistance <- signatureDistance(modSignatures$BSWIMSSig_S2$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$BSWIMSSig_S2$controlTemplate,testSet,"spearman")
ROCTable$BSWIMSSig_S2 <- roc95ci(as.vector(testSet$Event),BSWIMSdistance,main="CV B:SWiMS Signature (T2)") 
epiTable$BSWIMSSig_S2 <- epi.tests(table(BSWIMSdistance>0,!testSet$Event))
# Model 3
trainSet <- DataExpresionV[-set3,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
## Univariate
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT3$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT3$univariateAnalysis$Name[qvalues<0.01])
snames <- snames[1:150]
system.time(modSignatures$UniSig_S3 <- getSignature(data=trainSet,varlist=snames,Outcome="Event",method="spearman"))
Univaraitedistance <- signatureDistance(modSignatures$UniSig_S3$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$UniSig_S3$controlTemplate,testSet,"spearman")
ROCTable$UniSig_S3 <- roc95ci(as.vector(testSet$Event),Univaraitedistance,main="Univariate LOGIT Rank Signature (T3)") 
epiTable$UniSig_S3 <- epi.tests(table(Univaraitedistance>0,!testSet$Event))
## CV LASSO
system.time(modSignatures$LassoSig_S3 <- getSignature(data=trainSet,varlist=names(CVLASSOLOGIT_S3$frequencyTable[CVLASSOLOGIT_S3$frequencyTable>1]),Outcome="Event",method="spearman"))
Lassodistance <- signatureDistance(modSignatures$LassoSig_S3$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$LassoSig_S3$controlTemplate,testSet,"spearman")
ROCTable$LassoSig_S3 <- roc95ci(as.vector(testSet$Event),Lassodistance,main="CV LASSO Signature (T3)") 
epiTable$LassoSig_S3 <- epi.tests(table(Lassodistance>0,!testSet$Event))
## CV B:SWiMS
system.time(modSignatures$BSWIMSSig_S3 <- getSignature(data=trainSet,varlist=names(CVBSWiMSLOGIT_S3$frequencyTable[CVBSWiMSLOGIT_S3$frequencyTable>1]),Outcome="Event",method="spearman"))
BSWIMSdistance <- signatureDistance(modSignatures$BSWIMSSig_S3$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$BSWIMSSig_S3$controlTemplate,testSet,"spearman")
ROCTable$BSWIMSSig_S3 <- roc95ci(as.vector(testSet$Event),BSWIMSdistance,main="CV B:SWiMS Signature (T3)") 
epiTable$BSWIMSSig_S3 <- epi.tests(table(BSWIMSdistance>0,!testSet$Event))
# Model 4
trainSet <- DataExpresionV[-set4,]
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
## Univariate
qvalues <- p.adjust(1-pnorm(BRCASignatureLOGITT4$univariateAnalysis$ZUni),method="BH")
snames <- as.character(BRCASignatureLOGITT4$univariateAnalysis$Name[qvalues<0.01])
snames <- snames[1:150]
system.time(modSignatures$UniSig_S4 <- getSignature(data=trainSet,varlist=snames,Outcome="Event",method="spearman"))
Univaraitedistance <- signatureDistance(modSignatures$UniSig_S4$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$UniSig_S4$controlTemplate,testSet,"spearman")
ROCTable$UniSig_S4 <- roc95ci(as.vector(testSet$Event),Univaraitedistance,main="Univariate LOGIT Rank Signature (T4)") 
epiTable$UniSig_S4 <- epi.tests(table(Univaraitedistance>0,!testSet$Event))
## CV LASSO
system.time(modSignatures$LassoSig_S4 <- getSignature(data=trainSet,varlist=names(CVLASSOLOGIT_S4$frequencyTable[CVLASSOLOGIT_S4$frequencyTable>1]),Outcome="Event",method="spearman"))
Lassodistance <- signatureDistance(modSignatures$LassoSig_S4$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$LassoSig_S4$controlTemplate,testSet,"spearman")
ROCTable$LassoSig_S4 <- roc95ci(as.vector(testSet$Event),Lassodistance,main="CV LASSO Signature (T4)") 
epiTable$LassoSig_S4 <- epi.tests(table(Lassodistance>0,!testSet$Event))
## CV B:SWiMS
system.time(modSignatures$BSWIMSSig_S4 <- getSignature(data=trainSet,varlist=names(CVBSWiMSLOGIT_S4$frequencyTable[CVBSWiMSLOGIT_S4$frequencyTable>1]),Outcome="Event",method="spearman"))
BSWIMSdistance <- signatureDistance(modSignatures$BSWIMSSig_S4$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$BSWIMSSig_S4$controlTemplate,testSet,"spearman")
ROCTable$BSWIMSSig_S4 <- roc95ci(as.vector(testSet$Event),BSWIMSdistance,main="CV B:SWiMS Signature (T4)") 
epiTable$BSWIMSSig_S4 <- epi.tests(table(BSWIMSdistance>0,!testSet$Event))
The signature of using all sets
## with All eB:SWiMS
trainSet <- DataExpresionV
removeLessthan5 <- (trainSet$Event==0) & (trainSet$ct.dmfs<5)
trainSet <- trainSet[!removeLessthan5,]
testSet <- DataExpresionLOGIT
equimatrix <- BRCASignatureLOGITALL$eBSWiMS.model$equivalentMatrix
CVBSWiMSLOGIT_ALL <-baggedModel(BRCASignatureLOGITALL$cvObject$formula.list,trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITALL$univariateAnalysis)
system.time(modSignatures$AllSets <- getSignature(data=trainSet,varlist=names(CVBSWiMSLOGIT_ALL$frequencyTable[CVBSWiMSLOGIT_ALL$frequencyTable>1]),Outcome="Event",method="spearman"))
BSWIMSdistance <- signatureDistance(modSignatures$AllSets$caseTamplate,testSet,"spearman") - signatureDistance(modSignatures$AllSets$controlTemplate,testSet,"spearman")
ROCTable$AllSets <- roc95ci(as.vector(testSet$Event),BSWIMSdistance,main="CV eB:SWiMS Signature All") 
epiTable$AllSets <- epi.tests(table(BSWIMSdistance>0,!testSet$Event))
sigvs70 <- epi.kappa(table(BSWIMSdistance>0,sig70[!LOGITExclude]==0))
sigmatrix <- equimatrix
sigmatrix <- sigmatrix[order(sigmatrix$p.value),]
sigmatrix$Extendend_Name <- NULL
Tables of final signature
pander::pander(sigmatrix,caption="Genes Performance")| Name | Locus | UniPerformance | FullPerformance | |
|---|---|---|---|---|
| 6 | N_218811_at | N_218811_at | 0.4969 | 0.7138 |
| 4 | N_222026_at | N_206287_s_at | 0.4988 | 0.7435 |
| 7 | N_222026_at | N_211596_s_at | 0.4988 | 0.715 |
| 12 | N_210006_at | N_211596_s_at | 0.5156 | 0.7462 |
| 5 | N_222026_at | N_218811_at | 0.4988 | 0.7054 |
| 8 | N_211596_s_at | N_211596_s_at | 0.5208 | 0.7138 |
| 2 | N_222026_at | N_204973_at | 0.4988 | 0.7431 |
| 15 | N_222026_at | N_202350_s_at | 0.4988 | 0.7385 |
| 11 | N_202770_s_at | N_211596_s_at | 0.5 | 0.7271 |
| 9 | N_203273_s_at | N_211596_s_at | 0.5038 | 0.7576 |
| 16 | N_202350_s_at | N_202350_s_at | 0.5 | 0.7138 |
| 10 | N_202769_at | N_211596_s_at | 0.505 | 0.7271 |
| 14 | N_203485_at | N_203485_at | 0.4985 | 0.7138 |
| 13 | N_205101_at | N_205101_at | 0.4973 | 0.7138 |
| 1 | N_204973_at | N_204973_at | 0.5034 | 0.7138 |
| 3 | N_206287_s_at | N_206287_s_at | 0.5056 | 0.7138 |
| DeltaPerformance | ImprovementFraction | Estimate | Std. Error | |
|---|---|---|---|---|
| 6 | 0.03124 | 0.27 | -3.185 | 0.6574 |
| 4 | 0.03772 | 0.258 | -1.939 | 0.4224 |
| 7 | 0.01109 | 0.3215 | -2.083 | 0.4216 |
| 12 | 0.04232 | 0.3066 | -1.41 | 0.3221 |
| 5 | 0.02287 | 0.2748 | -1.777 | 0.4018 |
| 8 | 0.009901 | 0.3358 | -1.016 | 0.1924 |
| 2 | 0.02475 | 0.2198 | -1.793 | 0.4224 |
| 15 | 0.04455 | 0.2413 | -1.943 | 0.4231 |
| 11 | 0.02322 | 0.3618 | -1.251 | 0.2432 |
| 9 | 0.05376 | 0.2632 | -3.06 | 0.615 |
| 16 | 0.0198 | 0.1924 | -0.6661 | 0.1615 |
| 10 | 0.02322 | 0.3587 | -1.442 | 0.2739 |
| 14 | 0.009901 | 0.2463 | -0.5564 | 0.1593 |
| 13 | 0.001883 | 0.235 | -1.82 | 0.4768 |
| 1 | -0.004601 | 0.1054 | -1.425 | 0.4693 |
| 3 | 0.008018 | 0.238 | -1.828 | 0.5438 |
| z value | Pr(>|z|) | p.value | |
|---|---|---|---|
| 6 | -4.845 | 1.269e-06 | 5.02e-08 |
| 4 | -4.59 | 4.441e-06 | 1.341e-07 |
| 7 | -4.94 | 7.807e-07 | 2.588e-07 |
| 12 | -4.379 | 1.195e-05 | 6.398e-07 |
| 5 | -4.422 | 9.763e-06 | 6.863e-07 |
| 8 | -5.282 | 1.275e-07 | 7.719e-07 |
| 2 | -4.245 | 2.187e-05 | 9.653e-07 |
| 15 | -4.592 | 4.387e-06 | 1.078e-06 |
| 11 | -5.142 | 2.712e-07 | 1.315e-06 |
| 9 | -4.975 | 6.512e-07 | 2.12e-06 |
| 16 | -4.124 | 3.719e-05 | 2.822e-06 |
| 10 | -5.263 | 1.414e-07 | 2.926e-06 |
| 14 | -3.494 | 0.0004765 | 1.754e-05 |
| 13 | -3.818 | 0.0001346 | 2.127e-05 |
| 1 | -3.037 | 0.002393 | 0.0006046 |
| 3 | -3.361 | 0.0007768 | 0.0007371 |
pander::pander(epiTable$AllSets$tab,"Signature Confusion")| Outcome + | Outcome - | Total | |
|---|---|---|---|
| Test + | 37 | 147 | 184 |
| Test - | 21 | 183 | 204 |
| Total | 58 | 330 | 388 |
pander::pander(summary(epiTable$AllSets),"Diangostic Summary",round=3)| est | lower | upper | |
|---|---|---|---|
| aprev | 0.474 | 0.424 | 0.525 |
| tprev | 0.149 | 0.116 | 0.189 |
| se | 0.638 | 0.501 | 0.76 |
| sp | 0.555 | 0.499 | 0.609 |
| diag.acc | 0.567 | 0.516 | 0.617 |
| diag.or | 2.193 | 1.231 | 3.909 |
| nnd | 5.195 | 2.71 | 3341 |
| youden | 0.192 | 0 | 0.369 |
| ppv | 0.201 | 0.146 | 0.266 |
| npv | 0.897 | 0.847 | 0.935 |
| plr | 1.432 | 1.14 | 1.799 |
| nlr | 0.653 | 0.458 | 0.931 |
pander::pander(sigvs70$kappa,caption="Kappa Agreement between 70 signature and Ensemble Signature")| est | se | lower | upper |
|---|---|---|---|
| 0.3196 | 0.0507 | 0.2202 | 0.419 |
The Signatures Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$UniSig_S1$elements$sensitivity+epiTable$UniSig_S1$elements$specificity),
1.0-0.5*(epiTable$UniSig_S2$elements$sensitivity+epiTable$UniSig_S2$elements$specificity),
1.0-0.5*(epiTable$UniSig_S3$elements$sensitivity+epiTable$UniSig_S3$elements$specificity),
1.0-0.5*(epiTable$UniSig_S4$elements$sensitivity+epiTable$UniSig_S4$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S1$elements$sensitivity+epiTable$LassoSig_S1$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S2$elements$sensitivity+epiTable$LassoSig_S2$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S3$elements$sensitivity+epiTable$LassoSig_S3$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S4$elements$sensitivity+epiTable$LassoSig_S4$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S1$elements$sensitivity+epiTable$BSWIMSSig_S1$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S2$elements$sensitivity+epiTable$BSWIMSSig_S2$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S3$elements$sensitivity+epiTable$BSWIMSSig_S3$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S4$elements$sensitivity+epiTable$BSWIMSSig_S4$elements$specificity),
1.0-0.5*(epiTable$AllSets$elements$sensitivity+epiTable$AllSets$elements$specificity),
1.0-0.5*(epiTable$AllSets$elements$sensitivity+epiTable$AllSets$elements$specificity),
1.0-0.5*(epiTable$AllSets$elements$sensitivity+epiTable$AllSets$elements$specificity),
1.0-0.5*(epiTable$AllSets$elements$sensitivity+epiTable$AllSets$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("Univaraite","LASSO","By Set B:SWiMS","Full B:SWiMS"),main="Signature: SOS Validation Error",args.legend = list(x = "topright"))
acctables <- as.matrix(rbind(
epiTable$UniSig_S1$elements$diag.acc,
epiTable$UniSig_S2$elements$diag.acc,
epiTable$UniSig_S3$elements$diag.acc,
epiTable$UniSig_S4$elements$diag.acc,
epiTable$LassoSig_S1$elements$diag.acc,
epiTable$LassoSig_S2$elements$diag.acc,
epiTable$LassoSig_S3$elements$diag.acc,
epiTable$LassoSig_S4$elements$diag.acc,
epiTable$BSWIMSSig_S1$elements$diag.acc,
epiTable$BSWIMSSig_S2$elements$diag.acc,
epiTable$BSWIMSSig_S3$elements$diag.acc,
epiTable$BSWIMSSig_S4$elements$diag.acc,
epiTable$AllSets$elements$diag.acc,
epiTable$AllSets$elements$diag.acc,
epiTable$AllSets$elements$diag.acc,
epiTable$AllSets$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("Univaraite","LASSO","By Set B:SWiMS","Full B:SWiMS"),main="Signature: SOS Validation Accuracy",args.legend = list(x = "topright"))
sentables <- as.matrix(rbind(
epiTable$UniSig_S1$elements$sensitivity,
epiTable$UniSig_S2$elements$sensitivity,
epiTable$UniSig_S3$elements$sensitivity,
epiTable$UniSig_S4$elements$sensitivity,
epiTable$LassoSig_S1$elements$sensitivity,
epiTable$LassoSig_S2$elements$sensitivity,
epiTable$LassoSig_S3$elements$sensitivity,
epiTable$LassoSig_S4$elements$sensitivity,
epiTable$BSWIMSSig_S1$elements$sensitivity,
epiTable$BSWIMSSig_S2$elements$sensitivity,
epiTable$BSWIMSSig_S3$elements$sensitivity,
epiTable$BSWIMSSig_S4$elements$sensitivity,
epiTable$AllSets$elements$sensitivity,
epiTable$AllSets$elements$sensitivity,
epiTable$AllSets$elements$sensitivity,
epiTable$AllSets$elements$sensitivity
))
bplot <- barPlotCiError(sentables,"Sensitivity",c("des","min","loi","chin"),c("Univaraite","LASSO","By Set B:SWiMS","Full B:SWiMS"),main="Signature: SOS Validation Sensitivity",args.legend = list(x = "bottomright"))
### All Validation Tables
errtables <- as.matrix(rbind(
1.0-0.5*(epiTable$rpBSWIMS_T1$elements$sensitivity+epiTable$rpBSWIMS_T1$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T2$elements$sensitivity+epiTable$rpBSWIMS_T2$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T3$elements$sensitivity+epiTable$rpBSWIMS_T3$elements$specificity),
1.0-0.5*(epiTable$rpBSWIMS_T4$elements$sensitivity+epiTable$rpBSWIMS_T4$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S1$elements$sensitivity+epiTable$FULLBSWiMS_S1$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S2$elements$sensitivity+epiTable$FULLBSWiMS_S2$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S3$elements$sensitivity+epiTable$FULLBSWiMS_S3$elements$specificity),
1.0-0.5*(epiTable$FULLBSWiMS_S4$elements$sensitivity+epiTable$FULLBSWiMS_S4$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S1$elements$sensitivity+epiTable$UnivariateRAWKNN_S1$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S2$elements$sensitivity+epiTable$UnivariateRAWKNN_S2$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S3$elements$sensitivity+epiTable$UnivariateRAWKNN_S3$elements$specificity),
1.0-0.5*(epiTable$UnivariateRAWKNN_S4$elements$sensitivity+epiTable$UnivariateRAWKNN_S4$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S1$elements$sensitivity+epiTable$FULLLASSOLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S2$elements$sensitivity+epiTable$FULLLASSOLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S3$elements$sensitivity+epiTable$FULLLASSOLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$FULLLASSOLOGIT_S4$elements$sensitivity+epiTable$FULLLASSOLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S1$elements$sensitivity+epiTable$CVLASSOLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S2$elements$sensitivity+epiTable$CVLASSOLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S3$elements$sensitivity+epiTable$CVLASSOLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$CVLASSOLOGIT_S4$elements$sensitivity+epiTable$CVLASSOLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S1$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S1$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S2$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S2$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S3$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S3$elements$specificity),
1.0-0.5*(epiTable$CVBSWiMSLOGIT_S4$elements$sensitivity+epiTable$CVBSWiMSLOGIT_S4$elements$specificity),
1.0-0.5*(epiTable$UniSig_S1$elements$sensitivity+epiTable$UniSig_S1$elements$specificity),
1.0-0.5*(epiTable$UniSig_S2$elements$sensitivity+epiTable$UniSig_S2$elements$specificity),
1.0-0.5*(epiTable$UniSig_S3$elements$sensitivity+epiTable$UniSig_S3$elements$specificity),
1.0-0.5*(epiTable$UniSig_S4$elements$sensitivity+epiTable$UniSig_S4$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S1$elements$sensitivity+epiTable$LassoSig_S1$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S2$elements$sensitivity+epiTable$LassoSig_S2$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S3$elements$sensitivity+epiTable$LassoSig_S3$elements$specificity),
1.0-0.5*(epiTable$LassoSig_S4$elements$sensitivity+epiTable$LassoSig_S4$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S1$elements$sensitivity+epiTable$BSWIMSSig_S1$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S2$elements$sensitivity+epiTable$BSWIMSSig_S2$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S3$elements$sensitivity+epiTable$BSWIMSSig_S3$elements$specificity),
1.0-0.5*(epiTable$BSWIMSSig_S4$elements$sensitivity+epiTable$BSWIMSSig_S4$elements$specificity),
1.0-0.5*(epi.signature70$elements$sensitivity+epi.signature70$elements$specificity),
1.0-0.5*(epi.signature70$elements$sensitivity+epi.signature70$elements$specificity),
1.0-0.5*(epi.signature70$elements$sensitivity+epi.signature70$elements$specificity),
1.0-0.5*(epi.signature70$elements$sensitivity+epi.signature70$elements$specificity)
))
bplot <- barPlotCiError(errtables,"eFPFN",c("des","min","loi","chin"),c("CV Estimation","FULL B:SWiMS","RAW Univaraite","FULL LASSO","CV LASSO","CV B:SWiMS","Univaraite Sig","LASSO Sig","B:SWiMS Sig","70 Sig"),main="SOS Validation Error",args.legend = list(x = "topright",cex=0.75))
acctables <- as.matrix(rbind(
epiTable$rpBSWIMS_T1$elements$diag.acc,
epiTable$rpBSWIMS_T2$elements$diag.acc,
epiTable$rpBSWIMS_T3$elements$diag.acc,
epiTable$rpBSWIMS_T4$elements$diag.acc,
epiTable$FULLBSWiMS_S1$elements$diag.acc,
epiTable$FULLBSWiMS_S2$elements$diag.acc,
epiTable$FULLBSWiMS_S3$elements$diag.acc,
epiTable$FULLBSWiMS_S4$elements$diag.acc,
epiTable$UnivariateRAWKNN_S1$elements$diag.acc,
epiTable$UnivariateRAWKNN_S2$elements$diag.acc,
epiTable$UnivariateRAWKNN_S3$elements$diag.acc,
epiTable$UnivariateRAWKNN_S4$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S1$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S2$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S3$elements$diag.acc,
epiTable$FULLLASSOLOGIT_S4$elements$diag.acc,
epiTable$CVLASSOLOGIT_S1$elements$diag.acc,
epiTable$CVLASSOLOGIT_S2$elements$diag.acc,
epiTable$CVLASSOLOGIT_S3$elements$diag.acc,
epiTable$CVLASSOLOGIT_S4$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S1$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S2$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S3$elements$diag.acc,
epiTable$CVBSWiMSLOGIT_S4$elements$diag.acc,
epiTable$UniSig_S1$elements$diag.acc,
epiTable$UniSig_S2$elements$diag.acc,
epiTable$UniSig_S3$elements$diag.acc,
epiTable$UniSig_S4$elements$diag.acc,
epiTable$LassoSig_S1$elements$diag.acc,
epiTable$LassoSig_S2$elements$diag.acc,
epiTable$LassoSig_S3$elements$diag.acc,
epiTable$LassoSig_S4$elements$diag.acc,
epiTable$BSWIMSSig_S1$elements$diag.acc,
epiTable$BSWIMSSig_S2$elements$diag.acc,
epiTable$BSWIMSSig_S3$elements$diag.acc,
epiTable$BSWIMSSig_S4$elements$diag.acc,
epi.signature70$elements$diag.acc,
epi.signature70$elements$diag.acc,
epi.signature70$elements$diag.acc,
epi.signature70$elements$diag.acc
))
bplot <- barPlotCiError(acctables,"Accuracy",c("des","min","loi","chin"),c("CV Estimation","FULL B:SWiMS","RAW Univaraite","FULL LASSO","CV LASSO","CV B:SWiMS","Univaraite Sig","LASSO Sig","B:SWiMS Sig","70 Sig"),main="SOS Validation Accuracy",args.legend = list(x = "bottomright",cex=0.75))
sentables <- as.matrix(rbind(
epiTable$rpBSWIMS_T1$elements$sensitivity,
epiTable$rpBSWIMS_T2$elements$sensitivity,
epiTable$rpBSWIMS_T3$elements$sensitivity,
epiTable$rpBSWIMS_T4$elements$sensitivity,
epiTable$FULLBSWiMS_S1$elements$sensitivity,
epiTable$FULLBSWiMS_S2$elements$sensitivity,
epiTable$FULLBSWiMS_S3$elements$sensitivity,
epiTable$FULLBSWiMS_S4$elements$sensitivity,
epiTable$UnivariateRAWKNN_S1$elements$sensitivity,
epiTable$UnivariateRAWKNN_S2$elements$sensitivity,
epiTable$UnivariateRAWKNN_S3$elements$sensitivity,
epiTable$UnivariateRAWKNN_S4$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S1$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S2$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S3$elements$sensitivity,
epiTable$FULLLASSOLOGIT_S4$elements$sensitivity,
epiTable$CVLASSOLOGIT_S1$elements$sensitivity,
epiTable$CVLASSOLOGIT_S2$elements$sensitivity,
epiTable$CVLASSOLOGIT_S3$elements$sensitivity,
epiTable$CVLASSOLOGIT_S4$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S1$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S2$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S3$elements$sensitivity,
epiTable$CVBSWiMSLOGIT_S4$elements$sensitivity,
epiTable$UniSig_S1$elements$sensitivity,
epiTable$UniSig_S2$elements$sensitivity,
epiTable$UniSig_S3$elements$sensitivity,
epiTable$UniSig_S4$elements$sensitivity,
epiTable$LassoSig_S1$elements$sensitivity,
epiTable$LassoSig_S2$elements$sensitivity,
epiTable$LassoSig_S3$elements$sensitivity,
epiTable$LassoSig_S4$elements$sensitivity,
epiTable$BSWIMSSig_S1$elements$sensitivity,
epiTable$BSWIMSSig_S2$elements$sensitivity,
epiTable$BSWIMSSig_S3$elements$sensitivity,
epiTable$BSWIMSSig_S4$elements$sensitivity,
epi.signature70$elements$sensitivity,
epi.signature70$elements$sensitivity,
epi.signature70$elements$sensitivity,
epi.signature70$elements$sensitivity
))
bplot <- barPlotCiError(sentables,"Sensitivity",c("des","min","loi","chin"),c("CV Estimation","FULL B:SWiMS","RAW Univaraite","FULL LASSO","CV LASSO","CV B:SWiMS","Univaraite Sig","LASSO Sig","B:SWiMS Sig","70 Sig"),main="SOS Validation Sensitivity",args.legend = list(x = "bottomright",cex=0.75))
FULLLASSOLOGIT <-baggedModel(BRCASignatureLOGITT1$cvObject$LASSOVariables[1],trainSet,type="LOGIT",Outcome="Event",timeOutcome = "ct.dmfs",frequencyThreshold=0.1,univariate=BRCASignatureLOGITT1$univariateAnalysis)Num. Models: 1 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
numberofFeatures <- matrix(c(
length(BRCASignatureLOGITT1$BSWiMS.model$coefficients),
length(BRCASignatureLOGITT2$BSWiMS.model$coefficients),
length(BRCASignatureLOGITT3$BSWiMS.model$coefficients),
length(BRCASignatureLOGITT4$BSWiMS.model$coefficients),
length(BRCASignatureLOGITT1$eBSWiMS.model$equivalentMatrix$Name),
length(BRCASignatureLOGITT2$eBSWiMS.model$equivalentMatrix$Name),
length(BRCASignatureLOGITT3$eBSWiMS.model$equivalentMatrix$Name),
length(BRCASignatureLOGITT4$eBSWiMS.model$equivalentMatrix$Name),
str_count(BRCASignatureLOGITT1$cvObject$LASSOVariables[1], "\\+"),
str_count(BRCASignatureLOGITT2$cvObject$LASSOVariables[1], "\\+"),
str_count(BRCASignatureLOGITT3$cvObject$LASSOVariables[1], "\\+"),
str_count(BRCASignatureLOGITT4$cvObject$LASSOVariables[1], "\\+"),
length(modSignatures$UniSig_S1$controlTemplate),
length(modSignatures$UniSig_S2$controlTemplate),
length(modSignatures$UniSig_S3$controlTemplate),
length(modSignatures$UniSig_S4$controlTemplate),
length(modSignatures$LassoSig_S1$controlTemplate),
length(modSignatures$LassoSig_S2$controlTemplate),
length(modSignatures$LassoSig_S3$controlTemplate),
length(modSignatures$LassoSig_S4$controlTemplate),
length(modSignatures$BSWIMSSig_S1$controlTemplate),
length(modSignatures$BSWIMSSig_S2$controlTemplate),
length(modSignatures$BSWIMSSig_S3$controlTemplate),
length(modSignatures$BSWIMSSig_S4$controlTemplate)
),4,6)
rownames(numberofFeatures) <- c("des","min","loi","chin")
colnames(numberofFeatures) <- c("B:SWiMS","eB:SWiMS","LASSO","Univariate","CV LASSO","CV B:SWiMS")
barplot(numberofFeatures,cex.names=0.7,las=2,ylim=c(0.0,150),main="Feature Size",ylab="#",beside=TRUE,legend = rownames(numberofFeatures),args.legend = list(x = "topleft"))
pander::pander(numberofFeatures,caption="Number of Features")| B:SWiMS | eB:SWiMS | LASSO | Univariate | CV LASSO | CV B:SWiMS | |
|---|---|---|---|---|---|---|
| des | 5 | 12 | 55 | 134 | 156 | 13 |
| min | 4 | 7 | 72 | 125 | 227 | 13 |
| loi | 6 | 10 | 46 | 131 | 96 | 14 |
| chin | 6 | 16 | 68 | 134 | 176 | 17 |
Venn Diagrams
par(mfrow=c(2,2))
UniLOGIT <- names(modSignatures$UniSig_S1$controlTemplate)
LASSOF <- names(modSignatures$LassoSig_S1$controlTemplate)
BSWIMSF <- names(modSignatures$BSWIMSSig_S1$controlTemplate)
eBSWiMS <- BRCASignatureLOGITT1$eBSWiMS.model$equivalentMatrix$Name
featurelist <- list(Univariate=UniLOGIT,LASSO=LASSOF,BSWIMS=BSWIMSF,eBSWIMS=eBSWiMS)
vend <- venn(featurelist)
title("des")
UniLOGIT <- names(modSignatures$UniSig_S2$controlTemplate)
LASSOF <- names(modSignatures$LassoSig_S2$controlTemplate)
BSWIMSF <- names(modSignatures$BSWIMSSig_S2$controlTemplate)
eBSWiMS <- BRCASignatureLOGITT2$eBSWiMS.model$equivalentMatrix$Name
featurelist <- list(Univariate=UniLOGIT,LASSO=LASSOF,BSWIMS=BSWIMSF,eBSWIMS=eBSWiMS)
vend <- venn(featurelist)
title("min")
UniLOGIT <- names(modSignatures$UniSig_S3$controlTemplate)
LASSOF <- names(modSignatures$LassoSig_S3$controlTemplate)
BSWIMSF <- names(modSignatures$BSWIMSSig_S3$controlTemplate)
eBSWiMS <- BRCASignatureLOGITT3$eBSWiMS.model$equivalentMatrix$Name
featurelist <- list(Univariate=UniLOGIT,LASSO=LASSOF,BSWIMS=BSWIMSF,eBSWIMS=eBSWiMS)
vend <- venn(featurelist)
title("loi")
UniLOGIT <- names(modSignatures$UniSig_S4$controlTemplate)
LASSOF <- names(modSignatures$LassoSig_S4$controlTemplate)
BSWIMSF <- names(modSignatures$BSWIMSSig_S4$controlTemplate)
eBSWiMS <- BRCASignatureLOGITT4$eBSWiMS.model$equivalentMatrix$Name
featurelist <- list(Univariate=UniLOGIT,LASSO=LASSOF,BSWIMS=BSWIMSF,eBSWIMS=eBSWiMS)
vend <- venn(featurelist)
title("chin")
par(mfrow=c(1,1))
Venn Diagrams for the four tests
par(mfrow=c(2,2))
Signature_1 <- names(modSignatures$BSWIMSSig_S1$controlTemplate)
Signature_2 <- names(modSignatures$BSWIMSSig_S2$controlTemplate)
Signature_3 <- names(modSignatures$BSWIMSSig_S3$controlTemplate)
Signature_4 <- names(modSignatures$BSWIMSSig_S4$controlTemplate)
featurelist <- list(des=Signature_1,min=Signature_2,loi=Signature_3,chin=Signature_4)
vend <- venn(featurelist)
title("B:SWiMS")
Model_1 <- BRCASignatureLOGITT1$eBSWiMS.model$equivalentMatrix$Name
Model_2 <- BRCASignatureLOGITT2$eBSWiMS.model$equivalentMatrix$Name
Model_3 <- BRCASignatureLOGITT3$eBSWiMS.model$equivalentMatrix$Name
Model_4 <- BRCASignatureLOGITT4$eBSWiMS.model$equivalentMatrix$Name
featurelist <- list(des=Model_1,min=Model_2,loi=Model_3,chin=Model_4)
vend <- venn(featurelist)
title("eB:SWiMS")
Signature_1 <- names(modSignatures$LassoSig_S1$controlTemplate)
Signature_2 <- names(modSignatures$LassoSig_S2$controlTemplate)
Signature_3 <- names(modSignatures$LassoSig_S3$controlTemplate)
Signature_4 <- names(modSignatures$LassoSig_S4$controlTemplate)
featurelist <- list(des=Signature_1,min=Signature_2,loi=Signature_3,chin=Signature_4)
vend <- venn(featurelist)
title("LASSO")
Signature_1 <- names(modSignatures$UniSig_S1$controlTemplate)
Signature_2 <- names(modSignatures$UniSig_S2$controlTemplate)
Signature_3 <- names(modSignatures$UniSig_S3$controlTemplate)
Signature_4 <- names(modSignatures$UniSig_S4$controlTemplate)
featurelist <- list(des=Signature_1,min=Signature_2,loi=Signature_3,chin=Signature_4)
vend <- venn(featurelist)
title("Univariate")
par(mfrow=c(1,1))