ARCENE 025
José Tamez-Peña
October 2, 2017
library("epiR")
library("FRESA.CAD")
library(network)
library(GGally)
library("e1071")
library("gplots")
error.bar <- function(x, y, upper, lower=upper, length=0.05,...){
if(length(x) != length(y) | length(y) !=length(lower) | length(lower) != length(upper))
stop("vectors must be same length")
arrows(x,y+upper, x, y-lower, angle=90, code=3, length=length, ...)
}
barPlotCiError<- function(citable,metricname,thesets,themethod,main,...)
{
colnames(citable) <- c(metricname,"lower","upper")
rownames(citable) <- rep(thesets,length(themethod))
pander::pander(citable,caption=main,round = 3)
citable <- citable[order(rep(1:length(thesets),length(themethod))),]
barmatrix <- matrix(citable[,1],length(themethod),length(thesets))
colnames(barmatrix) <- thesets
rownames(barmatrix) <- themethod
pander::pander(barmatrix,caption=main,round = 3)
barp <- barplot(barmatrix,cex.names=0.7,las=2,ylim=c(0.0,1.0),main=main,ylab=metricname,beside=TRUE,legend = themethod,...)
error.bar(barp,citable[,1],citable[,3]-citable[,1],citable[,1]-citable[,2])
return(barp)
}
#trainLabeled <- read.delim("./Arcene/ARCENE/trainSet.txt")
#validLabeled <- read.delim("./Arcene/ARCENE/arcene_valid.txt")
#testLabeled <- read.delim("./Arcene/ARCENE/arcene_test.txt")
trainLabeled <- read.delim("trainSet.txt")
validLabeled <- read.delim("arcene_valid.txt")
testLabeled <- read.delim("arcene_test.txt")
trainLabeled$Labels <- 1*(trainLabeled$Labels>0)
validLabeled$Labels <- 1*(validLabeled$Labels>0)
arcene.norm <- rbind(trainLabeled,validLabeled)diagTable <- NULL
citable <- NULL
errorcitable <- NULL
collectPerfomance<- function(bswimsmodel)
{
diagTable <- NULL
citable <- NULL
errorcitable <- NULL
pr <- predict(bswimsmodel$BSWiMS.model,validLabeled)
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),pr),main="B:SWiMS")
diagTable$BaggingBSWIMS <- pm$predictionTable
ci <- epi.tests(diagTable$BaggingBSWIMS)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pr <- predict(bswimsmodel$bagging$bagged.model,validLabeled)
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),pr),main="Bagged")
diagTable$BagginForward <- pm$predictionTable
ci <- epi.tests(diagTable$BagginForward)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
prm <- ensemblePredict(bswimsmodel$BSWiMS.models$formula.list,trainLabeled,validLabeled)
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prm$ensemblePredict),main="Ensemble B:SWiMS")
diagTable$EnsembleBSwims <- pm$predictionTable
ci <- epi.tests(diagTable$EnsembleBSwims)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
prm2 <- ensemblePredict(bswimsmodel$BSWiMS.models$forward.selection.list,trainLabeled,validLabeled)
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prm2$ensemblePredict),main="Ensemble Forward")
diagTable$EnsembleForward <- pm$predictionTable
ci <- epi.tests(diagTable$EnsembleForward)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
ds <- trainLabeled[,c("Labels",names(bswimsmodel$BSWiMS.models$bagging$frequencyTable))]
psvm <- svm(formula("Labels ~ ."),ds)
prsvm <- predict(psvm,validLabeled)-0.5
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prsvm),main="SVM: B:SWiMS Selection")
diagTable$SVM_BaggingBSWIMS <- pm$predictionTable
ci <- epi.tests(diagTable$SVM_BaggingBSWIMS)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
ds <- trainLabeled[,c("Labels",rownames(bswimsmodel$univariateAnalysis[1:length(bswimsmodel$BSWiMS.models$bagging$frequencyTable),]))]
psvm <- svm(formula("Labels ~ ."),ds)
prsvm <- predict(psvm,validLabeled)-0.5
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prsvm),main="SVM: Univariate Selection")
diagTable$SVM_BaggingBSWIMS <- pm$predictionTable
ci <- epi.tests(diagTable$SVM_BaggingBSWIMS)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
ds <- trainLabeled[,c("Labels",names(bswimsmodel$bagging$frequencyTable))]
psvm <- svm(formula("Labels ~ ."),ds)
prsvm <- predict(psvm,validLabeled)-0.5
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prsvm),main="SVM: Forward Selection")
diagTable$SVM_BaggingForward <- pm$predictionTable
ci <- epi.tests(diagTable$SVM_BaggingForward)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
ds <- trainLabeled[,c("Labels",rownames(bswimsmodel$univariateAnalysis[1:length(bswimsmodel$bagging$frequencyTable),]))]
psvm <- svm(formula("Labels ~ ."),ds)
prsvm <- predict(psvm,validLabeled)-0.5
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prsvm),main="SVM: Univariate Forward Selection")
diagTable$SVM_BaggingBSWIMS <- pm$predictionTable
ci <- epi.tests(diagTable$SVM_BaggingBSWIMS)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
ds <- trainLabeled[,c("Labels",as.character(subset(bswimsmodel$univariateAnalysis,ZUni>3.0)[,1]))]
psvm <- svm(formula("Labels ~ ."),ds)
prsvm <- predict(psvm,validLabeled)-0.5
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),prsvm),main="SVM: Z>3")
diagTable$SVM_3Sunivariate <- pm$predictionTable
ci <- epi.tests(diagTable$SVM_3Sunivariate)
citable <- rbind(citable,ci$elements$diag.acc)
errorcitable <- rbind(errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
result <- list(errorcitable=errorcitable,citable=citable,diagTable=diagTable)
return(result)
}
arceneCVOne <- FRESA.Model(formula = Labels ~ 1,data = trainLabeled,bswimsCycles=5,filter.p.value = 0.01)
pm<-plot(arceneCVOne$bootstrappedModel)
pander::pander(summary(arceneCVOne$BSWiMS.model))coefficients:
Table continues below Estimate lower mean upper u.Accuracy r.Accuracy V698 -0.009289 NA NA NA 0.66 0.78 V4290 -0.01089 NA NA NA 0.69 0.74 V8502 -0.006158 NA NA NA 0.7 0.72 V5473 -0.003825 NA NA NA 0.7 0.77 V1184 -0.0599 NA NA NA 0.6 0.81 V1975 -0.04117 NA NA NA 0.68 0.92 V3339 -0.07282 NA NA NA 0.65 0.94 V7272 -0.01224 NA NA NA 0.67 0.77 V3206 -0.009384 NA NA NA 0.67 0.73 V9104 -1.685 NA NA NA 0.59 0.84 V7899 -0.1328 NA NA NA 0.59 0.83 V4352 -0.01085 NA NA NA 0.66 0.82 V7513 -0.1226 NA NA NA 0.57 0.94 V9868 -0.007911 NA NA NA 0.55 0.79 V1831 -0.02816 NA NA NA 0.59 0.9 V9818 -0.004621 NA NA NA 0.56 0.81 V9319 -0.01406 NA NA NA 0.63 0.83 V5956 -0.04824 NA NA NA 0.57 0.92 V4542 -0.003579 NA NA NA 0.61 0.82 V8156 -0.003971 NA NA NA 0.58 0.83 V9275 -0.003784 NA NA NA 0.58 0.77 full.Accuracy u.AUC r.AUC full.AUC IDI NRI z.IDI z.NRI V698 0.88 0.6672 0.7841 0.8807 0.1795 0.8506 4.911 5.203 V4290 0.85 0.6964 0.7386 0.8515 0.2197 1.263 5.512 8.364 V8502 0.84 0.7054 0.7256 0.8401 0.141 1.127 4.049 6.791 V5473 0.8 0.7054 0.7703 0.7995 0.1371 0.8929 4.013 4.98 V1184 0.95 0.6015 0.8084 0.9505 0.3335 1.705 7.535 17.19 V1975 0.95 0.6972 0.9188 0.9505 0.1149 1.721 3.44 16.78 V3339 0.95 0.6826 0.9416 0.9505 0.1307 1.649 4.3 14.69 V7272 0.84 0.6932 0.7679 0.8401 0.1339 1.175 4.017 7.734 V3206 0.8 0.6932 0.7297 0.7995 0.1667 1.058 4.596 6.621 V9104 0.88 0.6339 0.8401 0.8807 0.1449 0.9091 4.468 6.157 V7899 0.85 0.6315 0.8287 0.8515 0.09401 0.7403 3.427 5.362 V4352 0.85 0.6696 0.8222 0.8515 0.1334 0.9481 3.821 5.498 V7513 0.95 0.599 0.9416 0.9505 0.08622 1.578 3.082 13.09 V9868 0.84 0.5763 0.7881 0.8401 0.08145 0.9318 3.169 5.481 V1831 0.95 0.595 0.9034 0.9505 0.1698 1.594 4.779 13.03 V9818 0.84 0.5633 0.8133 0.8401 0.09667 1.101 3.1 6.633 V9319 0.85 0.6477 0.8312 0.8515 0.11 1.065 3.61 6.426 V5956 0.95 0.5869 0.9213 0.9505 0.08596 1.766 3.424 18.96 V4542 0.88 0.6201 0.8222 0.8807 0.1082 0.6623 3.749 3.908 V8156 0.88 0.586 0.8336 0.8807 0.1078 0.7792 3.619 4.726 V9275 0.8 0.5812 0.7703 0.7995 0.126 0.9026 3.692 5.082 - Accuracy: 0.97
- tAUC: 0.9708
bootstrap:
bg <- baggedModel(arceneCVOne$BSWiMS.models$formula.list,trainLabeled,type="LOGIT",univariate=arceneCVOne$univariateAnalysis,n_bootstrap = 30)Num. Models: 5 To Test: 21 TopFreq: 1 Thrf: 0 Removed: 0 ******************************
pander::pander(summary(bg$bagged.model))coefficients:
Table continues below Estimate lower OR upper u.Accuracy r.Accuracy V698 -0.01001 0.9097 0.943 0.9776 0.6727 0.7703 V4290 -0.01038 0.8949 0.9329 0.9726 0.6963 0.7703 V8502 -0.006944 0.9193 0.9489 0.9794 0.6997 0.7697 V5473 -0.004563 0.9321 0.9559 0.9803 0.6977 0.7527 V1184 -0.7148 0.006795 0.2923 12.57 0.6203 0.8267 V1975 -0.4981 0.02692 0.4242 6.683 0.6777 0.9313 V3339 -0.9046 0.002897 0.2038 14.33 0.6603 0.93 V7272 -0.0141 0.8347 0.896 0.9618 0.674 0.7883 V3206 -0.01125 0.8294 0.8934 0.9624 0.6673 0.728 V9104 -1.443 2.486e-06 0.000133 0.007118 0.595 0.842 V7899 -0.2467 0.006599 0.1687 4.312 0.5917 0.833 V4352 -0.01028 0.8874 0.9304 0.9754 0.6597 0.82 V7513 -1.386 7.494e-05 0.09618 123.4 0.5773 0.9447 V9868 -0.01021 0.8737 0.9254 0.9801 0.5653 0.8083 V1831 -0.2826 0.1517 0.61 2.452 0.5847 0.9047 V9818 -0.004715 0.9398 0.9634 0.9876 0.5723 0.8207 V9319 -0.01359 0.8531 0.9127 0.9765 0.6213 0.837 V5956 -0.6214 0.01265 0.3548 9.954 0.5893 0.937 V4542 -0.00434 0.9549 0.9754 0.9962 0.6243 0.8257 V8156 -0.004293 0.957 0.9748 0.993 0.5823 0.8337 V9275 -0.004308 0.9355 0.9579 0.9808 0.566 0.7657 full.Accuracy u.AUC r.AUC full.AUC IDI NRI z.IDI z.NRI V698 0.8777 0.6799 0.7749 0.8801 0.1822 0.9132 4.922 5.941 V4290 0.865 0.7032 0.7699 0.8659 0.2104 1.111 5.27 7.22 V8502 0.853 0.7051 0.7762 0.8553 0.1505 1.023 4.238 6.224 V5473 0.7987 0.7023 0.7548 0.7995 0.1503 0.9133 4.227 5.336 V1184 0.9713 0.6217 0.8276 0.9713 0.3347 1.661 7.572 Inf V1975 0.9713 0.6954 0.9315 0.9713 0.1439 1.681 4.017 Inf V3339 0.9713 0.6915 0.9309 0.9713 0.1789 1.74 5.122 Inf V7272 0.853 0.6971 0.789 0.8553 0.1467 1.114 4.263 7.288 V3206 0.7987 0.691 0.7263 0.7995 0.18 1.01 4.856 6.426 V9104 0.8777 0.6348 0.842 0.8801 0.1358 0.9034 4.247 6.573 V7899 0.865 0.6295 0.8317 0.8659 0.08118 0.7089 3.091 5.289 V4352 0.865 0.6654 0.8231 0.8659 0.1451 0.9693 3.968 6.089 V7513 0.9713 0.5797 0.9456 0.9713 0.08489 1.338 3.068 14.39 V9868 0.853 0.576 0.8099 0.8553 0.08008 0.8295 3.039 5.128 V1831 0.9713 0.5847 0.906 0.9713 0.1879 1.582 5.053 Inf V9818 0.853 0.5725 0.8241 0.8553 0.09797 1.055 3.074 6.392 V9319 0.865 0.6185 0.8407 0.8659 0.1034 0.9808 3.307 6.358 V5956 0.9713 0.5893 0.9379 0.9713 0.1164 1.465 3.72 Inf V4542 0.8777 0.6315 0.8285 0.8801 0.123 0.7587 3.872 4.809 V8156 0.8777 0.58 0.8371 0.8801 0.1131 0.8094 3.602 5.223 V9275 0.7987 0.5627 0.7675 0.7995 0.1334 0.9594 3.797 5.711 - Accuracy: 0.97
- tAUC: 0.9683
bootstrap:
bv <- bootstrapValidation_Bin(model.formula=arceneCVOne$BSWiMS.model$formula,Outcome="Labels",data=trainLabeled,type="LOGIT")
pm<-plot(bv)
pc <- collectPerfomance(arceneCVOne)
citable <- rbind(citable,pc$citable)
errorcitable <- rbind(errorcitable,pc$errorcitable)
thesets <- c("BSWIMS","Forward:Bagging","BSWIMS:Ensemble","Forward:Ensemble","SVM:BSWIMS","SVM:Top(Nb)","SVM:Forward","SVM:Top(Nf)","SVM:3 Sigma")
bp <- barPlotCiError(as.matrix(citable),metricname="Accuracy",thesets=thesets,themethod=c("Validation"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(errorcitable),metricname="Balanced Error",thesets=thesets,themethod=c("Validation"),main="Balanced Error",args.legend = list(x = "topright"))
arceneCVTwo <- FRESA.Model(formula = Labels ~ 1,data = trainLabeled,bswimsCycles=10,filter.p.value = 0.01)
pc <- collectPerfomance(arceneCVTwo)
citable <- rbind(citable,pc$citable)
errorcitable <- rbind(errorcitable,pc$errorcitable)
bp <- barPlotCiError(as.matrix(citable),metricname="Accuracy",thesets=thesets,themethod=c("V5","V10"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(errorcitable),metricname="Balanced Error",thesets=thesets,themethod=c("V5","V10"),main="Balanced Error",args.legend = list(x = "topright"))
arceneCVThree <- FRESA.Model(formula = Labels ~ 1,data = trainLabeled,bswimsCycles=10,filter.p.value = 0.025)
pc <- collectPerfomance(arceneCVThree)
citable <- rbind(citable,pc$citable)
errorcitable <- rbind(errorcitable,pc$errorcitable)
bp <- barPlotCiError(as.matrix(citable),metricname="Accuracy",thesets=thesets,themethod=c("V5","V10","V10b"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(errorcitable),metricname="Balanced Error",thesets=thesets,themethod=c("V5","V10","V10b"),main="Balanced Error",args.legend = list(x = "topright"))
sig_ACCtable <- NULL
sig_errorcitable <- NULL
sizesig <- NULL
varlist <- as.character(subset(arceneCVOne$univariateAnalysis,ZUni>3.0)[,1])
#############################################################################################
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="pearson"))
#> 7 Number of features: 7 Max AUC: 0.656 AUC: 0.644 Z: 0.394 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.662 AUC: 0.662 Z: 0.501 Rdelta: 0.10000
#> 9 Number of features: 8 Max AUC: 0.662 AUC: 0.503 Z: 0.009 Rdelta: 0.08000
#> 10 Number of features: 8 Max AUC: 0.662 AUC: 0.623 Z: 0.353 Rdelta: 0.06400
#> 11 Number of features: 9 Max AUC: 0.686 AUC: 0.686 Z: 0.588 Rdelta: 0.06760
#> 12 Number of features: 9 Max AUC: 0.686 AUC: 0.585 Z: 0.118 Rdelta: 0.05408
#> 13 Number of features: 9 Max AUC: 0.686 AUC: 0.584 Z: 0.150 Rdelta: 0.04326
#> 14 Number of features: 9 Max AUC: 0.686 AUC: 0.587 Z: 0.048 Rdelta: 0.03461
#> 15 Number of features: 9 Max AUC: 0.686 AUC: 0.612 Z: 0.251 Rdelta: 0.02769
#> 16 Number of features: 9 Max AUC: 0.686 AUC: 0.585 Z: 0.136 Rdelta: 0.02215
#> 17 Number of features: 9 Max AUC: 0.686 AUC: 0.536 Z: -0.048 Rdelta: 0.01772
#> 18 Number of features: 9 Max AUC: 0.686 AUC: 0.605 Z: 0.310 Rdelta: 0.01418
#> 19 Number of features: 9 Max AUC: 0.686 AUC: 0.550 Z: 0.037 Rdelta: 0.01134
#> 20 Number of features: 9 Max AUC: 0.686 AUC: 0.503 Z: -0.109 Rdelta: 0.00907
#> 21 Number of features: 9 Max AUC: 0.686 AUC: 0.533 Z: -0.053 Rdelta: 0.00726
#> 22 Number of features: 9 Max AUC: 0.686 AUC: 0.580 Z: 0.158 Rdelta: 0.00581
#> 23 Number of features: 9 Max AUC: 0.686 AUC: 0.611 Z: 0.268 Rdelta: 0.00465
#> 24 Number of features: 9 Max AUC: 0.686 AUC: 0.607 Z: 0.167 Rdelta: 0.00372
#> 25 Number of features: 10 Max AUC: 0.695 AUC: 0.695 Z: 0.568 Rdelta: 0.01334
#> 26 Number of features: 10 Max AUC: 0.695 AUC: 0.622 Z: 0.297 Rdelta: 0.01068
#> 27 Number of features: 10 Max AUC: 0.695 AUC: 0.552 Z: 0.059 Rdelta: 0.00854
#> 28 Number of features: 10 Max AUC: 0.695 AUC: 0.553 Z: 0.024 Rdelta: 0.00683
#> 29 Number of features: 10 Max AUC: 0.695 AUC: 0.680 Z: 0.607 Rdelta: 0.00547
#> 30 Number of features: 10 Max AUC: 0.695 AUC: 0.643 Z: 0.395 Rdelta: 0.00437
#> 31 Number of features: 10 Max AUC: 0.695 AUC: 0.601 Z: 0.271 Rdelta: 0.00350
#> 32 Number of features: 10 Max AUC: 0.695 AUC: 0.561 Z: 0.126 Rdelta: 0.00280
#> 33 Number of features: 10 Max AUC: 0.695 AUC: 0.608 Z: 0.224 Rdelta: 0.00224
#> 34 Number of features: 10 Max AUC: 0.695 AUC: 0.563 Z: 0.043 Rdelta: 0.00179
#> 35 Number of features: 10 Max AUC: 0.695 AUC: 0.629 Z: 0.371 Rdelta: 0.00143
#> 36 Number of features: 10 Max AUC: 0.695 AUC: 0.565 Z: 0.107 Rdelta: 0.00115
#> 37 Number of features: 10 Max AUC: 0.695 AUC: 0.596 Z: 0.251 Rdelta: 0.00092
#> 38 Number of features: 10 Max AUC: 0.695 AUC: 0.614 Z: 0.250 Rdelta: 0.00073
#> 39 Number of features: 10 Max AUC: 0.695 AUC: 0.634 Z: 0.303 Rdelta: 0.00059
#> 40 Number of features: 10 Max AUC: 0.695 AUC: 0.589 Z: 0.168 Rdelta: 0.00047
#> 41 Number of features: 10 Max AUC: 0.695 AUC: 0.577 Z: 0.142 Rdelta: 0.00038
#> 42 Number of features: 10 Max AUC: 0.695 AUC: 0.641 Z: 0.382 Rdelta: 0.00030
#> 43 Number of features: 10 Max AUC: 0.695 AUC: 0.623 Z: 0.277 Rdelta: 0.00024
#> 44 Number of features: 10 Max AUC: 0.695 AUC: 0.552 Z: 0.063 Rdelta: 0.00019
#> 45 Number of features: 11 Max AUC: 0.697 AUC: 0.697 Z: 0.565 Rdelta: 0.01017
#> 46 Number of features: 11 Max AUC: 0.697 AUC: 0.635 Z: 0.367 Rdelta: 0.00814
#> 47 Number of features: 11 Max AUC: 0.697 AUC: 0.596 Z: 0.211 Rdelta: 0.00651
#> 48 Number of features: 12 Max AUC: 0.697 AUC: 0.695 Z: 0.667 Rdelta: 0.01586
#> 49 Number of features: 12 Max AUC: 0.697 AUC: 0.576 Z: 0.141 Rdelta: 0.01269
#> 50 Number of features: 12 Max AUC: 0.697 AUC: 0.678 Z: 0.602 Rdelta: 0.01015
#> 51 Number of features: 12 Max AUC: 0.697 AUC: 0.661 Z: 0.450 Rdelta: 0.00812
#> 52 Number of features: 13 Max AUC: 0.697 AUC: 0.694 Z: 0.616 Rdelta: 0.01731
#> 53 Number of features: 13 Max AUC: 0.697 AUC: 0.636 Z: 0.430 Rdelta: 0.01385
#> 54 Number of features: 13 Max AUC: 0.697 AUC: 0.623 Z: 0.261 Rdelta: 0.01108
#> 55 Number of features: 13 Max AUC: 0.697 AUC: 0.639 Z: 0.512 Rdelta: 0.00886
#> 56 Number of features: 13 Max AUC: 0.697 AUC: 0.645 Z: 0.432 Rdelta: 0.00709
#> 57 Number of features: 13 Max AUC: 0.697 AUC: 0.567 Z: 0.199 Rdelta: 0.00567
#> 58 Number of features: 13 Max AUC: 0.697 AUC: 0.627 Z: 0.330 Rdelta: 0.00454
#> 59 Number of features: 13 Max AUC: 0.697 AUC: 0.636 Z: 0.399 Rdelta: 0.00363
#> 60 Number of features: 13 Max AUC: 0.697 AUC: 0.648 Z: 0.460 Rdelta: 0.00290
#> 61 Number of features: 13 Max AUC: 0.697 AUC: 0.657 Z: 0.505 Rdelta: 0.00232
#> 62 Number of features: 13 Max AUC: 0.697 AUC: 0.630 Z: 0.406 Rdelta: 0.00186
#> 63 Number of features: 13 Max AUC: 0.697 AUC: 0.649 Z: 0.489 Rdelta: 0.00149
#> 64 Number of features: 13 Max AUC: 0.697 AUC: 0.643 Z: 0.434 Rdelta: 0.00119
#> 65 Number of features: 13 Max AUC: 0.697 AUC: 0.652 Z: 0.449 Rdelta: 0.00095
#> 66 Number of features: 13 Max AUC: 0.697 AUC: 0.633 Z: 0.429 Rdelta: 0.00076
#> 67 Number of features: 13 Max AUC: 0.697 AUC: 0.646 Z: 0.473 Rdelta: 0.00061
#> 68 Number of features: 13 Max AUC: 0.697 AUC: 0.635 Z: 0.444 Rdelta: 0.00049
#> 69 Number of features: 13 Max AUC: 0.697 AUC: 0.612 Z: 0.248 Rdelta: 0.00039
#> 70 Number of features: 13 Max AUC: 0.697 AUC: 0.646 Z: 0.453 Rdelta: 0.00031
#> 71 Number of features: 13 Max AUC: 0.697 AUC: 0.614 Z: 0.322 Rdelta: 0.00025
#> 72 Number of features: 13 Max AUC: 0.697 AUC: 0.629 Z: 0.451 Rdelta: 0.00020
#> 73 Number of features: 13 Max AUC: 0.697 AUC: 0.663 Z: 0.443 Rdelta: 0.00016
#> 74 Number of features: 13 Max AUC: 0.697 AUC: 0.672 Z: 0.550 Rdelta: 0.00013
#> 75 Number of features: 13 Max AUC: 0.697 AUC: 0.622 Z: 0.430 Rdelta: 0.00010
#> 76 Number of features: 13 Max AUC: 0.697 AUC: 0.622 Z: 0.408 Rdelta: 0.00008
#> user system elapsed
#> 19.5 0.0 19.5
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"pearson")+signatureDistance(signature$controlTemplate,validLabeled,"pearson")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS"))
#> 7 Number of features: 7 Max AUC: 0.685 AUC: 0.685 Z: 0.834 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.685 AUC: 0.677 Z: 0.811 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.750 AUC: 0.750 Z: 1.009 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.750 AUC: 0.746 Z: 1.014 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.750 AUC: 0.750 Z: 1.115 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.762 AUC: 0.762 Z: 0.958 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.763 AUC: 0.763 Z: 0.825 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.778 AUC: 0.778 Z: 0.196 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.778 AUC: 0.777 Z: 0.436 Rdelta: 0.10000
#> 16 Number of features: 16 Max AUC: 0.787 AUC: 0.787 Z: 0.618 Rdelta: 0.10000
#> 17 Number of features: 17 Max AUC: 0.787 AUC: 0.781 Z: 0.757 Rdelta: 0.10000
#> 18 Number of features: 17 Max AUC: 0.787 AUC: 0.767 Z: 0.522 Rdelta: 0.08000
#> 19 Number of features: 18 Max AUC: 0.790 AUC: 0.790 Z: 0.648 Rdelta: 0.08200
#> 20 Number of features: 18 Max AUC: 0.790 AUC: 0.766 Z: 0.688 Rdelta: 0.06560
#> 21 Number of features: 18 Max AUC: 0.790 AUC: 0.765 Z: 0.659 Rdelta: 0.05248
#> 22 Number of features: 18 Max AUC: 0.790 AUC: 0.761 Z: 0.324 Rdelta: 0.04198
#> 23 Number of features: 18 Max AUC: 0.790 AUC: 0.783 Z: 0.173 Rdelta: 0.03359
#> 24 Number of features: 19 Max AUC: 0.790 AUC: 0.785 Z: 0.587 Rdelta: 0.04023
#> 25 Number of features: 19 Max AUC: 0.790 AUC: 0.770 Z: 0.682 Rdelta: 0.03218
#> 26 Number of features: 19 Max AUC: 0.790 AUC: 0.773 Z: 0.625 Rdelta: 0.02575
#> 27 Number of features: 19 Max AUC: 0.790 AUC: 0.773 Z: 0.616 Rdelta: 0.02060
#> 28 Number of features: 20 Max AUC: 0.790 AUC: 0.778 Z: 0.608 Rdelta: 0.02854
#> 29 Number of features: 20 Max AUC: 0.790 AUC: 0.771 Z: 0.584 Rdelta: 0.02283
#> 30 Number of features: 21 Max AUC: 0.790 AUC: 0.779 Z: 0.646 Rdelta: 0.03055
#> 31 Number of features: 22 Max AUC: 0.790 AUC: 0.785 Z: 0.598 Rdelta: 0.03749
#> 32 Number of features: 22 Max AUC: 0.790 AUC: 0.777 Z: 0.259 Rdelta: 0.02999
#> 33 Number of features: 23 Max AUC: 0.790 AUC: 0.777 Z: 0.696 Rdelta: 0.03699
#> 34 Number of features: 24 Max AUC: 0.790 AUC: 0.788 Z: 0.380 Rdelta: 0.04329
#> 35 Number of features: 24 Max AUC: 0.790 AUC: 0.778 Z: 0.266 Rdelta: 0.03464
#> 36 Number of features: 24 Max AUC: 0.790 AUC: 0.772 Z: 0.593 Rdelta: 0.02771
#> 37 Number of features: 24 Max AUC: 0.790 AUC: 0.769 Z: 0.260 Rdelta: 0.02217
#> 38 Number of features: 25 Max AUC: 0.790 AUC: 0.783 Z: 0.615 Rdelta: 0.02995
#> 39 Number of features: 26 Max AUC: 0.790 AUC: 0.786 Z: 0.676 Rdelta: 0.03696
#> 40 Number of features: 27 Max AUC: 0.790 AUC: 0.780 Z: 0.669 Rdelta: 0.04326
#> 41 Number of features: 28 Max AUC: 0.790 AUC: 0.786 Z: 0.695 Rdelta: 0.04893
#> 42 Number of features: 29 Max AUC: 0.791 AUC: 0.791 Z: 0.187 Rdelta: 0.05404
#> 43 Number of features: 30 Max AUC: 0.791 AUC: 0.787 Z: 0.580 Rdelta: 0.05864
#> 44 Number of features: 31 Max AUC: 0.791 AUC: 0.783 Z: 0.621 Rdelta: 0.06277
#> 45 Number of features: 32 Max AUC: 0.792 AUC: 0.792 Z: 0.770 Rdelta: 0.06650
#> 46 Number of features: 32 Max AUC: 0.792 AUC: 0.773 Z: 0.555 Rdelta: 0.05320
#> 47 Number of features: 32 Max AUC: 0.792 AUC: 0.778 Z: 0.476 Rdelta: 0.04256
#> 48 Number of features: 32 Max AUC: 0.792 AUC: 0.780 Z: 0.250 Rdelta: 0.03405
#> 49 Number of features: 32 Max AUC: 0.792 AUC: 0.777 Z: 0.260 Rdelta: 0.02724
#> 50 Number of features: 32 Max AUC: 0.792 AUC: 0.779 Z: 0.470 Rdelta: 0.02179
#> 51 Number of features: 32 Max AUC: 0.792 AUC: 0.784 Z: 0.690 Rdelta: 0.01743
#> 52 Number of features: 32 Max AUC: 0.792 AUC: 0.783 Z: 0.402 Rdelta: 0.01395
#> 53 Number of features: 32 Max AUC: 0.792 AUC: 0.778 Z: 0.149 Rdelta: 0.01116
#> 54 Number of features: 33 Max AUC: 0.792 AUC: 0.791 Z: 0.618 Rdelta: 0.02004
#> 55 Number of features: 34 Max AUC: 0.792 AUC: 0.784 Z: 0.564 Rdelta: 0.02804
#> 56 Number of features: 35 Max AUC: 0.794 AUC: 0.794 Z: 0.699 Rdelta: 0.03523
#> 57 Number of features: 35 Max AUC: 0.794 AUC: 0.781 Z: 0.422 Rdelta: 0.02819
#> 58 Number of features: 36 Max AUC: 0.794 AUC: 0.790 Z: 0.650 Rdelta: 0.03537
#> 59 Number of features: 37 Max AUC: 0.794 AUC: 0.792 Z: 0.639 Rdelta: 0.04183
#> 60 Number of features: 38 Max AUC: 0.794 AUC: 0.791 Z: 0.408 Rdelta: 0.04765
#> 61 Number of features: 39 Max AUC: 0.794 AUC: 0.794 Z: 0.112 Rdelta: 0.05288
#> 62 Number of features: 40 Max AUC: 0.807 AUC: 0.807 Z: 0.323 Rdelta: 0.05759
#> 63 Number of features: 40 Max AUC: 0.807 AUC: 0.794 Z: 0.067 Rdelta: 0.04608
#> 64 Number of features: 40 Max AUC: 0.807 AUC: 0.797 Z: 0.209 Rdelta: 0.03686
#> 65 Number of features: 40 Max AUC: 0.807 AUC: 0.791 Z: 0.251 Rdelta: 0.02949
#> 66 Number of features: 41 Max AUC: 0.807 AUC: 0.801 Z: 0.680 Rdelta: 0.03654
#> 67 Number of features: 41 Max AUC: 0.807 AUC: 0.786 Z: 0.087 Rdelta: 0.02923
#> 68 Number of features: 41 Max AUC: 0.807 AUC: 0.780 Z: 0.605 Rdelta: 0.02339
#> 69 Number of features: 42 Max AUC: 0.807 AUC: 0.798 Z: 0.547 Rdelta: 0.03105
#> 70 Number of features: 43 Max AUC: 0.807 AUC: 0.803 Z: 0.728 Rdelta: 0.03794
#> 71 Number of features: 43 Max AUC: 0.807 AUC: 0.786 Z: 0.596 Rdelta: 0.03035
#> 72 Number of features: 44 Max AUC: 0.807 AUC: 0.804 Z: 0.525 Rdelta: 0.03732
#> 73 Number of features: 45 Max AUC: 0.807 AUC: 0.807 Z: 0.398 Rdelta: 0.04359
#> 74 Number of features: 45 Max AUC: 0.807 AUC: 0.789 Z: 0.449 Rdelta: 0.03487
#> 75 Number of features: 46 Max AUC: 0.807 AUC: 0.805 Z: 0.589 Rdelta: 0.04138
#> 76 Number of features: 46 Max AUC: 0.807 AUC: 0.788 Z: 0.506 Rdelta: 0.03311
#> 77 Number of features: 47 Max AUC: 0.807 AUC: 0.798 Z: 0.792 Rdelta: 0.03980
#> 78 Number of features: 47 Max AUC: 0.807 AUC: 0.788 Z: 0.534 Rdelta: 0.03184
#> 79 Number of features: 48 Max AUC: 0.807 AUC: 0.806 Z: 0.549 Rdelta: 0.03865
#> 80 Number of features: 49 Max AUC: 0.807 AUC: 0.800 Z: 0.582 Rdelta: 0.04479
#> 81 Number of features: 50 Max AUC: 0.807 AUC: 0.800 Z: 0.446 Rdelta: 0.05031
#> 82 Number of features: 51 Max AUC: 0.811 AUC: 0.811 Z: 0.433 Rdelta: 0.05528
#> 83 Number of features: 51 Max AUC: 0.811 AUC: 0.797 Z: 0.699 Rdelta: 0.04422
#> 84 Number of features: 51 Max AUC: 0.811 AUC: 0.798 Z: 0.539 Rdelta: 0.03538
#> 85 Number of features: 51 Max AUC: 0.811 AUC: 0.801 Z: 0.238 Rdelta: 0.02830
#> 86 Number of features: 51 Max AUC: 0.811 AUC: 0.796 Z: 0.431 Rdelta: 0.02264
#> 87 Number of features: 51 Max AUC: 0.811 AUC: 0.792 Z: 0.350 Rdelta: 0.01811
#> 88 Number of features: 51 Max AUC: 0.811 AUC: 0.797 Z: 0.665 Rdelta: 0.01449
#> 89 Number of features: 52 Max AUC: 0.812 AUC: 0.812 Z: 0.684 Rdelta: 0.02304
#> 90 Number of features: 53 Max AUC: 0.812 AUC: 0.805 Z: 0.760 Rdelta: 0.03074
#> 91 Number of features: 54 Max AUC: 0.812 AUC: 0.807 Z: 0.437 Rdelta: 0.03766
#> 92 Number of features: 54 Max AUC: 0.812 AUC: 0.794 Z: 0.542 Rdelta: 0.03013
#> 93 Number of features: 55 Max AUC: 0.812 AUC: 0.807 Z: 0.388 Rdelta: 0.03712
#> 94 Number of features: 55 Max AUC: 0.812 AUC: 0.792 Z: 0.430 Rdelta: 0.02969
#> 95 Number of features: 55 Max AUC: 0.812 AUC: 0.802 Z: 0.440 Rdelta: 0.02376
#> 96 Number of features: 55 Max AUC: 0.812 AUC: 0.801 Z: 0.459 Rdelta: 0.01900
#> 97 Number of features: 56 Max AUC: 0.812 AUC: 0.811 Z: 0.355 Rdelta: 0.02710
#> 98 Number of features: 57 Max AUC: 0.812 AUC: 0.810 Z: 0.419 Rdelta: 0.03439
#> 99 Number of features: 57 Max AUC: 0.812 AUC: 0.801 Z: 0.346 Rdelta: 0.02751
#> 100 Number of features: 57 Max AUC: 0.812 AUC: 0.797 Z: 0.211 Rdelta: 0.02201
#> 101 Number of features: 58 Max AUC: 0.816 AUC: 0.816 Z: 0.367 Rdelta: 0.02981
#> 102 Number of features: 58 Max AUC: 0.816 AUC: 0.799 Z: 0.279 Rdelta: 0.02385
#> 103 Number of features: 58 Max AUC: 0.816 AUC: 0.803 Z: 0.358 Rdelta: 0.01908
#> 104 Number of features: 58 Max AUC: 0.816 AUC: 0.805 Z: 0.365 Rdelta: 0.01526
#> 105 Number of features: 58 Max AUC: 0.816 AUC: 0.800 Z: 0.334 Rdelta: 0.01221
#> 106 Number of features: 58 Max AUC: 0.816 AUC: 0.801 Z: 0.343 Rdelta: 0.00977
#> 107 Number of features: 58 Max AUC: 0.816 AUC: 0.805 Z: 0.282 Rdelta: 0.00781
#> 108 Number of features: 58 Max AUC: 0.816 AUC: 0.794 Z: 0.290 Rdelta: 0.00625
#> 109 Number of features: 58 Max AUC: 0.816 AUC: 0.802 Z: 0.305 Rdelta: 0.00500
#> 110 Number of features: 58 Max AUC: 0.816 AUC: 0.805 Z: 0.380 Rdelta: 0.00400
#> 111 Number of features: 58 Max AUC: 0.816 AUC: 0.803 Z: 0.345 Rdelta: 0.00320
#> 112 Number of features: 58 Max AUC: 0.816 AUC: 0.802 Z: 0.354 Rdelta: 0.00256
#> 113 Number of features: 58 Max AUC: 0.816 AUC: 0.800 Z: 0.385 Rdelta: 0.00205
#> 114 Number of features: 58 Max AUC: 0.816 AUC: 0.802 Z: 0.361 Rdelta: 0.00164
#> 115 Number of features: 58 Max AUC: 0.816 AUC: 0.800 Z: 0.364 Rdelta: 0.00131
#> 116 Number of features: 58 Max AUC: 0.816 AUC: 0.797 Z: 0.387 Rdelta: 0.00105
#> 117 Number of features: 59 Max AUC: 0.816 AUC: 0.812 Z: 0.407 Rdelta: 0.01094
#> 118 Number of features: 59 Max AUC: 0.816 AUC: 0.801 Z: 0.377 Rdelta: 0.00876
#> 119 Number of features: 60 Max AUC: 0.816 AUC: 0.807 Z: 0.355 Rdelta: 0.01788
#> 120 Number of features: 60 Max AUC: 0.816 AUC: 0.788 Z: 0.260 Rdelta: 0.01430
#> 121 Number of features: 60 Max AUC: 0.816 AUC: 0.804 Z: 0.366 Rdelta: 0.01144
#> 122 Number of features: 60 Max AUC: 0.816 AUC: 0.790 Z: 0.381 Rdelta: 0.00915
#> 123 Number of features: 61 Max AUC: 0.827 AUC: 0.827 Z: 0.325 Rdelta: 0.01824
#> 124 Number of features: 61 Max AUC: 0.827 AUC: 0.812 Z: 0.323 Rdelta: 0.01459
#> 125 Number of features: 61 Max AUC: 0.827 AUC: 0.816 Z: 0.338 Rdelta: 0.01167
#> 126 Number of features: 61 Max AUC: 0.827 AUC: 0.817 Z: 0.297 Rdelta: 0.00934
#> 127 Number of features: 61 Max AUC: 0.827 AUC: 0.800 Z: 0.249 Rdelta: 0.00747
#> 128 Number of features: 61 Max AUC: 0.827 AUC: 0.796 Z: 0.242 Rdelta: 0.00598
#> 129 Number of features: 61 Max AUC: 0.827 AUC: 0.816 Z: 0.426 Rdelta: 0.00478
#> 130 Number of features: 61 Max AUC: 0.827 AUC: 0.812 Z: 0.279 Rdelta: 0.00382
#> 131 Number of features: 61 Max AUC: 0.827 AUC: 0.804 Z: 0.350 Rdelta: 0.00306
#> 132 Number of features: 61 Max AUC: 0.827 AUC: 0.813 Z: 0.359 Rdelta: 0.00245
#> 133 Number of features: 61 Max AUC: 0.827 AUC: 0.813 Z: 0.345 Rdelta: 0.00196
#> 134 Number of features: 61 Max AUC: 0.827 AUC: 0.813 Z: 0.314 Rdelta: 0.00157
#> 135 Number of features: 61 Max AUC: 0.827 AUC: 0.818 Z: 0.249 Rdelta: 0.00125
#> 136 Number of features: 61 Max AUC: 0.827 AUC: 0.798 Z: 0.226 Rdelta: 0.00100
#> 137 Number of features: 61 Max AUC: 0.827 AUC: 0.809 Z: 0.371 Rdelta: 0.00080
#> 138 Number of features: 61 Max AUC: 0.827 AUC: 0.805 Z: 0.346 Rdelta: 0.00064
#> 139 Number of features: 61 Max AUC: 0.827 AUC: 0.805 Z: 0.342 Rdelta: 0.00051
#> 140 Number of features: 61 Max AUC: 0.827 AUC: 0.790 Z: 0.125 Rdelta: 0.00041
#> 141 Number of features: 61 Max AUC: 0.827 AUC: 0.795 Z: 0.282 Rdelta: 0.00033
#> 142 Number of features: 61 Max AUC: 0.827 AUC: 0.812 Z: 0.370 Rdelta: 0.00026
#> 143 Number of features: 61 Max AUC: 0.827 AUC: 0.813 Z: 0.372 Rdelta: 0.00021
#> 144 Number of features: 61 Max AUC: 0.827 AUC: 0.794 Z: 0.303 Rdelta: 0.00017
#> 145 Number of features: 62 Max AUC: 0.827 AUC: 0.822 Z: 0.387 Rdelta: 0.01015
#> 146 Number of features: 62 Max AUC: 0.827 AUC: 0.805 Z: 0.379 Rdelta: 0.00812
#> 147 Number of features: 63 Max AUC: 0.827 AUC: 0.819 Z: 0.354 Rdelta: 0.01731
#> 148 Number of features: 63 Max AUC: 0.827 AUC: 0.807 Z: 0.361 Rdelta: 0.01385
#> 149 Number of features: 63 Max AUC: 0.827 AUC: 0.818 Z: 0.347 Rdelta: 0.01108
#> 150 Number of features: 64 Max AUC: 0.827 AUC: 0.825 Z: 0.268 Rdelta: 0.01997
#> 151 Number of features: 64 Max AUC: 0.827 AUC: 0.816 Z: 0.388 Rdelta: 0.01598
#> 152 Number of features: 64 Max AUC: 0.827 AUC: 0.812 Z: 0.365 Rdelta: 0.01278
#> 153 Number of features: 65 Max AUC: 0.827 AUC: 0.818 Z: 0.407 Rdelta: 0.02150
#> 154 Number of features: 66 Max AUC: 0.828 AUC: 0.828 Z: 0.479 Rdelta: 0.02935
#> 155 Number of features: 66 Max AUC: 0.828 AUC: 0.804 Z: 0.291 Rdelta: 0.02348
#> 156 Number of features: 66 Max AUC: 0.828 AUC: 0.812 Z: 0.415 Rdelta: 0.01879
#> 157 Number of features: 66 Max AUC: 0.828 AUC: 0.813 Z: 0.351 Rdelta: 0.01503
#> 158 Number of features: 66 Max AUC: 0.828 AUC: 0.810 Z: 0.408 Rdelta: 0.01202
#> 159 Number of features: 66 Max AUC: 0.828 AUC: 0.785 Z: 0.363 Rdelta: 0.00962
#> 160 Number of features: 66 Max AUC: 0.828 AUC: 0.796 Z: 0.389 Rdelta: 0.00769
#> 161 Number of features: 66 Max AUC: 0.828 AUC: 0.800 Z: 0.372 Rdelta: 0.00616
#> 162 Number of features: 66 Max AUC: 0.828 AUC: 0.808 Z: 0.274 Rdelta: 0.00492
#> 163 Number of features: 66 Max AUC: 0.828 AUC: 0.815 Z: 0.283 Rdelta: 0.00394
#> 164 Number of features: 66 Max AUC: 0.828 AUC: 0.813 Z: 0.330 Rdelta: 0.00315
#> 165 Number of features: 66 Max AUC: 0.828 AUC: 0.814 Z: 0.329 Rdelta: 0.00252
#> 166 Number of features: 66 Max AUC: 0.828 AUC: 0.802 Z: 0.267 Rdelta: 0.00202
#> 167 Number of features: 67 Max AUC: 0.828 AUC: 0.824 Z: 0.421 Rdelta: 0.01182
#> 168 Number of features: 67 Max AUC: 0.828 AUC: 0.810 Z: 0.338 Rdelta: 0.00945
#> 169 Number of features: 67 Max AUC: 0.828 AUC: 0.816 Z: 0.348 Rdelta: 0.00756
#> 170 Number of features: 67 Max AUC: 0.828 AUC: 0.815 Z: 0.319 Rdelta: 0.00605
#> 171 Number of features: 67 Max AUC: 0.828 AUC: 0.808 Z: 0.346 Rdelta: 0.00484
#> 172 Number of features: 67 Max AUC: 0.828 AUC: 0.803 Z: 0.322 Rdelta: 0.00387
#> 173 Number of features: 67 Max AUC: 0.828 AUC: 0.804 Z: 0.367 Rdelta: 0.00310
#> 174 Number of features: 67 Max AUC: 0.828 AUC: 0.813 Z: 0.382 Rdelta: 0.00248
#> 175 Number of features: 67 Max AUC: 0.828 AUC: 0.797 Z: 0.414 Rdelta: 0.00198
#> 176 Number of features: 67 Max AUC: 0.828 AUC: 0.800 Z: 0.422 Rdelta: 0.00159
#> 177 Number of features: 67 Max AUC: 0.828 AUC: 0.801 Z: 0.245 Rdelta: 0.00127
#> 178 Number of features: 67 Max AUC: 0.828 AUC: 0.802 Z: 0.334 Rdelta: 0.00101
#> 179 Number of features: 67 Max AUC: 0.828 AUC: 0.804 Z: 0.426 Rdelta: 0.00081
#> 180 Number of features: 67 Max AUC: 0.828 AUC: 0.799 Z: 0.375 Rdelta: 0.00065
#> 181 Number of features: 67 Max AUC: 0.828 AUC: 0.809 Z: 0.370 Rdelta: 0.00052
#> 182 Number of features: 67 Max AUC: 0.828 AUC: 0.810 Z: 0.367 Rdelta: 0.00042
#> 183 Number of features: 67 Max AUC: 0.828 AUC: 0.809 Z: 0.282 Rdelta: 0.00033
#> 184 Number of features: 67 Max AUC: 0.828 AUC: 0.807 Z: 0.268 Rdelta: 0.00027
#> 185 Number of features: 67 Max AUC: 0.828 AUC: 0.810 Z: 0.323 Rdelta: 0.00021
#> 186 Number of features: 67 Max AUC: 0.828 AUC: 0.814 Z: 0.376 Rdelta: 0.00017
#> 187 Number of features: 67 Max AUC: 0.828 AUC: 0.791 Z: 0.403 Rdelta: 0.00014
#> 188 Number of features: 67 Max AUC: 0.828 AUC: 0.804 Z: 0.344 Rdelta: 0.00011
#> 189 Number of features: 67 Max AUC: 0.828 AUC: 0.790 Z: 0.353 Rdelta: 0.00009
#> user system elapsed
#> 113.44 0.01 113.60
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
testDistance_cotrol <- signatureDistance(signature$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Case"))
#> 7 Number of features: 7 Max AUC: 0.736 AUC: 0.735 Z: 1.033 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.736 AUC: 0.733 Z: 1.041 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.844 AUC: 0.844 Z: 1.237 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.848 AUC: 0.848 Z: 1.220 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.849 AUC: 0.849 Z: 1.237 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.890 AUC: 0.890 Z: 0.979 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.890 AUC: 0.889 Z: 0.961 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.905 AUC: 0.905 Z: 1.025 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.905 AUC: 0.896 Z: 0.760 Rdelta: 0.10000
#> 16 Number of features: 15 Max AUC: 0.905 AUC: 0.890 Z: 0.646 Rdelta: 0.08000
#> 17 Number of features: 16 Max AUC: 0.905 AUC: 0.894 Z: 0.603 Rdelta: 0.08200
#> 18 Number of features: 16 Max AUC: 0.905 AUC: 0.889 Z: 0.319 Rdelta: 0.06560
#> 19 Number of features: 17 Max AUC: 0.905 AUC: 0.895 Z: 0.645 Rdelta: 0.06904
#> 20 Number of features: 18 Max AUC: 0.905 AUC: 0.894 Z: 0.661 Rdelta: 0.07214
#> 21 Number of features: 19 Max AUC: 0.905 AUC: 0.888 Z: 0.633 Rdelta: 0.07492
#> 22 Number of features: 20 Max AUC: 0.905 AUC: 0.883 Z: 0.686 Rdelta: 0.07743
#> 23 Number of features: 20 Max AUC: 0.905 AUC: 0.873 Z: 0.290 Rdelta: 0.06194
#> 24 Number of features: 21 Max AUC: 0.905 AUC: 0.888 Z: 0.672 Rdelta: 0.06575
#> 25 Number of features: 22 Max AUC: 0.905 AUC: 0.894 Z: 0.669 Rdelta: 0.06917
#> 26 Number of features: 23 Max AUC: 0.905 AUC: 0.884 Z: 0.671 Rdelta: 0.07226
#> 27 Number of features: 24 Max AUC: 0.905 AUC: 0.895 Z: 0.727 Rdelta: 0.07503
#> 28 Number of features: 24 Max AUC: 0.905 AUC: 0.884 Z: 0.663 Rdelta: 0.06003
#> 29 Number of features: 24 Max AUC: 0.905 AUC: 0.881 Z: 0.423 Rdelta: 0.04802
#> 30 Number of features: 25 Max AUC: 0.905 AUC: 0.891 Z: 0.753 Rdelta: 0.05322
#> 31 Number of features: 25 Max AUC: 0.905 AUC: 0.878 Z: 0.560 Rdelta: 0.04257
#> 32 Number of features: 26 Max AUC: 0.905 AUC: 0.891 Z: 0.755 Rdelta: 0.04832
#> 33 Number of features: 27 Max AUC: 0.905 AUC: 0.885 Z: 0.659 Rdelta: 0.05349
#> 34 Number of features: 28 Max AUC: 0.905 AUC: 0.890 Z: 0.739 Rdelta: 0.05814
#> 35 Number of features: 29 Max AUC: 0.905 AUC: 0.884 Z: 0.397 Rdelta: 0.06232
#> 36 Number of features: 29 Max AUC: 0.905 AUC: 0.866 Z: 0.407 Rdelta: 0.04986
#> 37 Number of features: 29 Max AUC: 0.905 AUC: 0.859 Z: 0.165 Rdelta: 0.03989
#> 38 Number of features: 29 Max AUC: 0.905 AUC: 0.882 Z: 0.390 Rdelta: 0.03191
#> 39 Number of features: 30 Max AUC: 0.905 AUC: 0.886 Z: 0.581 Rdelta: 0.03872
#> 40 Number of features: 31 Max AUC: 0.905 AUC: 0.885 Z: 0.506 Rdelta: 0.04485
#> 41 Number of features: 31 Max AUC: 0.905 AUC: 0.878 Z: 0.117 Rdelta: 0.03588
#> 42 Number of features: 31 Max AUC: 0.905 AUC: 0.870 Z: 0.511 Rdelta: 0.02870
#> 43 Number of features: 32 Max AUC: 0.905 AUC: 0.884 Z: 0.411 Rdelta: 0.03583
#> 44 Number of features: 32 Max AUC: 0.905 AUC: 0.874 Z: 0.321 Rdelta: 0.02867
#> 45 Number of features: 33 Max AUC: 0.905 AUC: 0.881 Z: 0.494 Rdelta: 0.03580
#> 46 Number of features: 34 Max AUC: 0.905 AUC: 0.882 Z: 0.432 Rdelta: 0.04222
#> 47 Number of features: 35 Max AUC: 0.905 AUC: 0.882 Z: 0.549 Rdelta: 0.04800
#> 48 Number of features: 36 Max AUC: 0.905 AUC: 0.890 Z: 0.370 Rdelta: 0.05320
#> 49 Number of features: 36 Max AUC: 0.905 AUC: 0.878 Z: 0.415 Rdelta: 0.04256
#> 50 Number of features: 37 Max AUC: 0.905 AUC: 0.882 Z: 0.403 Rdelta: 0.04830
#> 51 Number of features: 38 Max AUC: 0.905 AUC: 0.883 Z: 0.488 Rdelta: 0.05347
#> 52 Number of features: 39 Max AUC: 0.905 AUC: 0.888 Z: 0.486 Rdelta: 0.05812
#> 53 Number of features: 40 Max AUC: 0.905 AUC: 0.881 Z: 0.352 Rdelta: 0.06231
#> 54 Number of features: 40 Max AUC: 0.905 AUC: 0.876 Z: 0.298 Rdelta: 0.04985
#> 55 Number of features: 40 Max AUC: 0.905 AUC: 0.877 Z: 0.268 Rdelta: 0.03988
#> 56 Number of features: 41 Max AUC: 0.905 AUC: 0.885 Z: 0.266 Rdelta: 0.04589
#> 57 Number of features: 42 Max AUC: 0.905 AUC: 0.880 Z: 0.312 Rdelta: 0.05130
#> 58 Number of features: 42 Max AUC: 0.905 AUC: 0.876 Z: 0.379 Rdelta: 0.04104
#> 59 Number of features: 43 Max AUC: 0.905 AUC: 0.879 Z: 0.320 Rdelta: 0.04694
#> 60 Number of features: 44 Max AUC: 0.905 AUC: 0.879 Z: 0.437 Rdelta: 0.05224
#> 61 Number of features: 45 Max AUC: 0.905 AUC: 0.881 Z: 0.363 Rdelta: 0.05702
#> 62 Number of features: 46 Max AUC: 0.905 AUC: 0.882 Z: 0.443 Rdelta: 0.06132
#> 63 Number of features: 47 Max AUC: 0.905 AUC: 0.886 Z: 0.268 Rdelta: 0.06519
#> 64 Number of features: 48 Max AUC: 0.905 AUC: 0.885 Z: 0.293 Rdelta: 0.06867
#> 65 Number of features: 49 Max AUC: 0.905 AUC: 0.881 Z: 0.398 Rdelta: 0.07180
#> 66 Number of features: 50 Max AUC: 0.905 AUC: 0.884 Z: 0.377 Rdelta: 0.07462
#> 67 Number of features: 50 Max AUC: 0.905 AUC: 0.876 Z: 0.102 Rdelta: 0.05970
#> 68 Number of features: 50 Max AUC: 0.905 AUC: 0.874 Z: 0.234 Rdelta: 0.04776
#> 69 Number of features: 50 Max AUC: 0.905 AUC: 0.875 Z: 0.381 Rdelta: 0.03821
#> 70 Number of features: 51 Max AUC: 0.905 AUC: 0.879 Z: 0.454 Rdelta: 0.04439
#> 71 Number of features: 52 Max AUC: 0.905 AUC: 0.878 Z: 0.397 Rdelta: 0.04995
#> 72 Number of features: 52 Max AUC: 0.905 AUC: 0.870 Z: 0.303 Rdelta: 0.03996
#> 73 Number of features: 52 Max AUC: 0.905 AUC: 0.869 Z: 0.145 Rdelta: 0.03197
#> 74 Number of features: 53 Max AUC: 0.905 AUC: 0.880 Z: 0.260 Rdelta: 0.03877
#> 75 Number of features: 54 Max AUC: 0.905 AUC: 0.879 Z: 0.195 Rdelta: 0.04489
#> 76 Number of features: 54 Max AUC: 0.905 AUC: 0.873 Z: 0.318 Rdelta: 0.03591
#> 77 Number of features: 55 Max AUC: 0.905 AUC: 0.877 Z: 0.318 Rdelta: 0.04232
#> 78 Number of features: 56 Max AUC: 0.905 AUC: 0.875 Z: 0.345 Rdelta: 0.04809
#> 79 Number of features: 56 Max AUC: 0.905 AUC: 0.871 Z: 0.398 Rdelta: 0.03847
#> 80 Number of features: 56 Max AUC: 0.905 AUC: 0.872 Z: 0.427 Rdelta: 0.03078
#> 81 Number of features: 57 Max AUC: 0.905 AUC: 0.876 Z: 0.376 Rdelta: 0.03770
#> 82 Number of features: 57 Max AUC: 0.905 AUC: 0.870 Z: 0.269 Rdelta: 0.03016
#> 83 Number of features: 57 Max AUC: 0.905 AUC: 0.871 Z: 0.129 Rdelta: 0.02413
#> 84 Number of features: 58 Max AUC: 0.905 AUC: 0.878 Z: 0.378 Rdelta: 0.03172
#> 85 Number of features: 59 Max AUC: 0.905 AUC: 0.878 Z: 0.390 Rdelta: 0.03854
#> 86 Number of features: 60 Max AUC: 0.905 AUC: 0.873 Z: 0.349 Rdelta: 0.04469
#> 87 Number of features: 61 Max AUC: 0.905 AUC: 0.875 Z: 0.271 Rdelta: 0.05022
#> 88 Number of features: 62 Max AUC: 0.905 AUC: 0.875 Z: 0.335 Rdelta: 0.05520
#> 89 Number of features: 63 Max AUC: 0.905 AUC: 0.879 Z: 0.332 Rdelta: 0.05968
#> 90 Number of features: 64 Max AUC: 0.905 AUC: 0.883 Z: 0.432 Rdelta: 0.06371
#> 91 Number of features: 64 Max AUC: 0.905 AUC: 0.870 Z: 0.462 Rdelta: 0.05097
#> 92 Number of features: 65 Max AUC: 0.905 AUC: 0.877 Z: 0.365 Rdelta: 0.05587
#> 93 Number of features: 65 Max AUC: 0.905 AUC: 0.855 Z: 0.086 Rdelta: 0.04470
#> 94 Number of features: 65 Max AUC: 0.905 AUC: 0.865 Z: 0.383 Rdelta: 0.03576
#> 95 Number of features: 65 Max AUC: 0.905 AUC: 0.861 Z: 0.477 Rdelta: 0.02861
#> 96 Number of features: 65 Max AUC: 0.905 AUC: 0.872 Z: 0.416 Rdelta: 0.02289
#> 97 Number of features: 66 Max AUC: 0.905 AUC: 0.877 Z: 0.467 Rdelta: 0.03060
#> 98 Number of features: 66 Max AUC: 0.905 AUC: 0.871 Z: 0.379 Rdelta: 0.02448
#> 99 Number of features: 66 Max AUC: 0.905 AUC: 0.858 Z: 0.345 Rdelta: 0.01958
#> 100 Number of features: 67 Max AUC: 0.905 AUC: 0.878 Z: 0.373 Rdelta: 0.02762
#> 101 Number of features: 68 Max AUC: 0.905 AUC: 0.876 Z: 0.326 Rdelta: 0.03486
#> 102 Number of features: 69 Max AUC: 0.905 AUC: 0.876 Z: 0.268 Rdelta: 0.04138
#> 103 Number of features: 70 Max AUC: 0.905 AUC: 0.875 Z: 0.303 Rdelta: 0.04724
#> 104 Number of features: 71 Max AUC: 0.905 AUC: 0.877 Z: 0.363 Rdelta: 0.05251
#> 105 Number of features: 71 Max AUC: 0.905 AUC: 0.871 Z: 0.342 Rdelta: 0.04201
#> 106 Number of features: 72 Max AUC: 0.905 AUC: 0.878 Z: 0.296 Rdelta: 0.04781
#> 107 Number of features: 73 Max AUC: 0.905 AUC: 0.875 Z: 0.266 Rdelta: 0.05303
#> 108 Number of features: 73 Max AUC: 0.905 AUC: 0.868 Z: 0.348 Rdelta: 0.04242
#> 109 Number of features: 74 Max AUC: 0.905 AUC: 0.874 Z: 0.306 Rdelta: 0.04818
#> 110 Number of features: 75 Max AUC: 0.905 AUC: 0.875 Z: 0.345 Rdelta: 0.05336
#> 111 Number of features: 76 Max AUC: 0.905 AUC: 0.886 Z: 0.342 Rdelta: 0.05803
#> 112 Number of features: 77 Max AUC: 0.905 AUC: 0.885 Z: 0.341 Rdelta: 0.06222
#> 113 Number of features: 78 Max AUC: 0.905 AUC: 0.878 Z: 0.352 Rdelta: 0.06600
#> 114 Number of features: 78 Max AUC: 0.905 AUC: 0.875 Z: 0.327 Rdelta: 0.05280
#> 115 Number of features: 79 Max AUC: 0.905 AUC: 0.881 Z: 0.306 Rdelta: 0.05752
#> 116 Number of features: 80 Max AUC: 0.905 AUC: 0.880 Z: 0.323 Rdelta: 0.06177
#> 117 Number of features: 81 Max AUC: 0.905 AUC: 0.879 Z: 0.304 Rdelta: 0.06559
#> 118 Number of features: 82 Max AUC: 0.905 AUC: 0.879 Z: 0.354 Rdelta: 0.06903
#> 119 Number of features: 83 Max AUC: 0.905 AUC: 0.882 Z: 0.353 Rdelta: 0.07213
#> 120 Number of features: 83 Max AUC: 0.905 AUC: 0.869 Z: 0.262 Rdelta: 0.05770
#> 121 Number of features: 83 Max AUC: 0.905 AUC: 0.868 Z: 0.345 Rdelta: 0.04616
#> 122 Number of features: 83 Max AUC: 0.905 AUC: 0.862 Z: 0.191 Rdelta: 0.03693
#> 123 Number of features: 84 Max AUC: 0.905 AUC: 0.882 Z: 0.357 Rdelta: 0.04324
#> 124 Number of features: 84 Max AUC: 0.905 AUC: 0.876 Z: 0.355 Rdelta: 0.03459
#> 125 Number of features: 85 Max AUC: 0.905 AUC: 0.882 Z: 0.373 Rdelta: 0.04113
#> 126 Number of features: 86 Max AUC: 0.905 AUC: 0.877 Z: 0.248 Rdelta: 0.04702
#> 127 Number of features: 86 Max AUC: 0.905 AUC: 0.872 Z: 0.332 Rdelta: 0.03761
#> 128 Number of features: 87 Max AUC: 0.905 AUC: 0.877 Z: 0.271 Rdelta: 0.04385
#> 129 Number of features: 88 Max AUC: 0.905 AUC: 0.885 Z: 0.287 Rdelta: 0.04947
#> 130 Number of features: 89 Max AUC: 0.905 AUC: 0.883 Z: 0.350 Rdelta: 0.05452
#> 131 Number of features: 89 Max AUC: 0.905 AUC: 0.874 Z: 0.280 Rdelta: 0.04362
#> 132 Number of features: 90 Max AUC: 0.905 AUC: 0.885 Z: 0.359 Rdelta: 0.04925
#> 133 Number of features: 91 Max AUC: 0.905 AUC: 0.884 Z: 0.319 Rdelta: 0.05433
#> 134 Number of features: 92 Max AUC: 0.905 AUC: 0.887 Z: 0.336 Rdelta: 0.05890
#> 135 Number of features: 93 Max AUC: 0.905 AUC: 0.882 Z: 0.287 Rdelta: 0.06301
#> 136 Number of features: 94 Max AUC: 0.905 AUC: 0.884 Z: 0.326 Rdelta: 0.06671
#> 137 Number of features: 95 Max AUC: 0.905 AUC: 0.882 Z: 0.314 Rdelta: 0.07004
#> 138 Number of features: 95 Max AUC: 0.905 AUC: 0.873 Z: 0.281 Rdelta: 0.05603
#> 139 Number of features: 95 Max AUC: 0.905 AUC: 0.873 Z: 0.309 Rdelta: 0.04482
#> 140 Number of features: 96 Max AUC: 0.905 AUC: 0.884 Z: 0.342 Rdelta: 0.05034
#> 141 Number of features: 96 Max AUC: 0.905 AUC: 0.855 Z: 0.230 Rdelta: 0.04027
#> 142 Number of features: 96 Max AUC: 0.905 AUC: 0.876 Z: 0.172 Rdelta: 0.03222
#> 143 Number of features: 97 Max AUC: 0.905 AUC: 0.881 Z: 0.313 Rdelta: 0.03900
#> 144 Number of features: 98 Max AUC: 0.905 AUC: 0.878 Z: 0.346 Rdelta: 0.04510
#> 145 Number of features: 98 Max AUC: 0.905 AUC: 0.876 Z: 0.304 Rdelta: 0.03608
#> 146 Number of features: 98 Max AUC: 0.905 AUC: 0.872 Z: 0.351 Rdelta: 0.02886
#> 147 Number of features: 98 Max AUC: 0.905 AUC: 0.872 Z: 0.326 Rdelta: 0.02309
#> 148 Number of features: 99 Max AUC: 0.905 AUC: 0.882 Z: 0.353 Rdelta: 0.03078
#> 149 Number of features: 99 Max AUC: 0.905 AUC: 0.875 Z: 0.305 Rdelta: 0.02462
#> 150 Number of features: 99 Max AUC: 0.905 AUC: 0.876 Z: 0.333 Rdelta: 0.01970
#> 151 Number of features: 99 Max AUC: 0.905 AUC: 0.875 Z: 0.362 Rdelta: 0.01576
#> 152 Number of features: 100 Max AUC: 0.905 AUC: 0.882 Z: 0.351 Rdelta: 0.02418
#> 153 Number of features: 100 Max AUC: 0.905 AUC: 0.876 Z: 0.373 Rdelta: 0.01935
#> 154 Number of features: 101 Max AUC: 0.905 AUC: 0.881 Z: 0.385 Rdelta: 0.02741
#> 155 Number of features: 101 Max AUC: 0.905 AUC: 0.870 Z: 0.300 Rdelta: 0.02193
#> 156 Number of features: 101 Max AUC: 0.905 AUC: 0.873 Z: 0.271 Rdelta: 0.01754
#> 157 Number of features: 102 Max AUC: 0.905 AUC: 0.881 Z: 0.281 Rdelta: 0.02579
#> 158 Number of features: 102 Max AUC: 0.905 AUC: 0.874 Z: 0.335 Rdelta: 0.02063
#> 159 Number of features: 102 Max AUC: 0.905 AUC: 0.832 Z: 0.343 Rdelta: 0.01651
#> 160 Number of features: 102 Max AUC: 0.905 AUC: 0.864 Z: 0.309 Rdelta: 0.01320
#> 161 Number of features: 102 Max AUC: 0.905 AUC: 0.875 Z: 0.396 Rdelta: 0.01056
#> 162 Number of features: 102 Max AUC: 0.905 AUC: 0.867 Z: 0.297 Rdelta: 0.00845
#> 163 Number of features: 103 Max AUC: 0.905 AUC: 0.877 Z: 0.367 Rdelta: 0.01761
#> 164 Number of features: 104 Max AUC: 0.905 AUC: 0.878 Z: 0.307 Rdelta: 0.02585
#> 165 Number of features: 104 Max AUC: 0.905 AUC: 0.865 Z: 0.366 Rdelta: 0.02068
#> 166 Number of features: 104 Max AUC: 0.905 AUC: 0.875 Z: 0.312 Rdelta: 0.01654
#> 167 Number of features: 104 Max AUC: 0.905 AUC: 0.876 Z: 0.357 Rdelta: 0.01323
#> 168 Number of features: 105 Max AUC: 0.905 AUC: 0.877 Z: 0.359 Rdelta: 0.02191
#> 169 Number of features: 105 Max AUC: 0.905 AUC: 0.872 Z: 0.335 Rdelta: 0.01753
#> 170 Number of features: 105 Max AUC: 0.905 AUC: 0.870 Z: 0.322 Rdelta: 0.01402
#> 171 Number of features: 105 Max AUC: 0.905 AUC: 0.869 Z: 0.344 Rdelta: 0.01122
#> 172 Number of features: 105 Max AUC: 0.905 AUC: 0.871 Z: 0.295 Rdelta: 0.00897
#> 173 Number of features: 105 Max AUC: 0.905 AUC: 0.876 Z: 0.243 Rdelta: 0.00718
#> 174 Number of features: 106 Max AUC: 0.905 AUC: 0.877 Z: 0.322 Rdelta: 0.01646
#> 175 Number of features: 106 Max AUC: 0.905 AUC: 0.876 Z: 0.279 Rdelta: 0.01317
#> 176 Number of features: 106 Max AUC: 0.905 AUC: 0.867 Z: 0.307 Rdelta: 0.01054
#> 177 Number of features: 106 Max AUC: 0.905 AUC: 0.870 Z: 0.383 Rdelta: 0.00843
#> 178 Number of features: 106 Max AUC: 0.905 AUC: 0.870 Z: 0.355 Rdelta: 0.00674
#> 179 Number of features: 106 Max AUC: 0.905 AUC: 0.869 Z: 0.276 Rdelta: 0.00539
#> 180 Number of features: 106 Max AUC: 0.905 AUC: 0.865 Z: 0.295 Rdelta: 0.00432
#> 181 Number of features: 106 Max AUC: 0.905 AUC: 0.874 Z: 0.328 Rdelta: 0.00345
#> 182 Number of features: 107 Max AUC: 0.905 AUC: 0.880 Z: 0.278 Rdelta: 0.01311
#> 183 Number of features: 107 Max AUC: 0.905 AUC: 0.873 Z: 0.408 Rdelta: 0.01049
#> 184 Number of features: 107 Max AUC: 0.905 AUC: 0.874 Z: 0.313 Rdelta: 0.00839
#> 185 Number of features: 108 Max AUC: 0.905 AUC: 0.877 Z: 0.362 Rdelta: 0.01755
#> 186 Number of features: 109 Max AUC: 0.905 AUC: 0.878 Z: 0.343 Rdelta: 0.02579
#> 187 Number of features: 110 Max AUC: 0.905 AUC: 0.879 Z: 0.354 Rdelta: 0.03322
#> 188 Number of features: 110 Max AUC: 0.905 AUC: 0.871 Z: 0.335 Rdelta: 0.02657
#> 189 Number of features: 111 Max AUC: 0.905 AUC: 0.876 Z: 0.347 Rdelta: 0.03391
#> 190 Number of features: 111 Max AUC: 0.905 AUC: 0.870 Z: 0.290 Rdelta: 0.02713
#> 191 Number of features: 111 Max AUC: 0.905 AUC: 0.874 Z: 0.318 Rdelta: 0.02171
#> 192 Number of features: 112 Max AUC: 0.905 AUC: 0.881 Z: 0.310 Rdelta: 0.02954
#> 193 Number of features: 112 Max AUC: 0.905 AUC: 0.873 Z: 0.323 Rdelta: 0.02363
#> 194 Number of features: 112 Max AUC: 0.905 AUC: 0.872 Z: 0.289 Rdelta: 0.01890
#> 195 Number of features: 112 Max AUC: 0.905 AUC: 0.874 Z: 0.281 Rdelta: 0.01512
#> 196 Number of features: 113 Max AUC: 0.905 AUC: 0.876 Z: 0.323 Rdelta: 0.02361
#> 197 Number of features: 113 Max AUC: 0.905 AUC: 0.875 Z: 0.321 Rdelta: 0.01889
#> 198 Number of features: 113 Max AUC: 0.905 AUC: 0.860 Z: 0.386 Rdelta: 0.01511
#> 199 Number of features: 113 Max AUC: 0.905 AUC: 0.837 Z: 0.344 Rdelta: 0.01209
#> 200 Number of features: 113 Max AUC: 0.905 AUC: 0.859 Z: 0.239 Rdelta: 0.00967
#> 201 Number of features: 114 Max AUC: 0.905 AUC: 0.876 Z: 0.300 Rdelta: 0.01870
#> 202 Number of features: 114 Max AUC: 0.905 AUC: 0.872 Z: 0.336 Rdelta: 0.01496
#> 203 Number of features: 114 Max AUC: 0.905 AUC: 0.868 Z: 0.307 Rdelta: 0.01197
#> 204 Number of features: 114 Max AUC: 0.905 AUC: 0.873 Z: 0.315 Rdelta: 0.00958
#> 205 Number of features: 114 Max AUC: 0.905 AUC: 0.863 Z: 0.243 Rdelta: 0.00766
#> 206 Number of features: 114 Max AUC: 0.905 AUC: 0.868 Z: 0.303 Rdelta: 0.00613
#> 207 Number of features: 114 Max AUC: 0.905 AUC: 0.876 Z: 0.268 Rdelta: 0.00490
#> 208 Number of features: 115 Max AUC: 0.905 AUC: 0.881 Z: 0.278 Rdelta: 0.01441
#> 209 Number of features: 115 Max AUC: 0.905 AUC: 0.871 Z: 0.274 Rdelta: 0.01153
#> 210 Number of features: 115 Max AUC: 0.905 AUC: 0.854 Z: 0.402 Rdelta: 0.00922
#> 211 Number of features: 116 Max AUC: 0.905 AUC: 0.880 Z: 0.357 Rdelta: 0.01830
#> 212 Number of features: 116 Max AUC: 0.905 AUC: 0.864 Z: 0.329 Rdelta: 0.01464
#> 213 Number of features: 117 Max AUC: 0.905 AUC: 0.879 Z: 0.339 Rdelta: 0.02318
#> 214 Number of features: 117 Max AUC: 0.905 AUC: 0.875 Z: 0.314 Rdelta: 0.01854
#> 215 Number of features: 117 Max AUC: 0.905 AUC: 0.873 Z: 0.286 Rdelta: 0.01483
#> 216 Number of features: 117 Max AUC: 0.905 AUC: 0.858 Z: 0.372 Rdelta: 0.01187
#> 217 Number of features: 117 Max AUC: 0.905 AUC: 0.856 Z: 0.356 Rdelta: 0.00949
#> 218 Number of features: 117 Max AUC: 0.905 AUC: 0.873 Z: 0.332 Rdelta: 0.00759
#> 219 Number of features: 117 Max AUC: 0.905 AUC: 0.860 Z: 0.301 Rdelta: 0.00608
#> 220 Number of features: 117 Max AUC: 0.905 AUC: 0.831 Z: 0.267 Rdelta: 0.00486
#> 221 Number of features: 117 Max AUC: 0.905 AUC: 0.875 Z: 0.312 Rdelta: 0.00389
#> 222 Number of features: 117 Max AUC: 0.905 AUC: 0.868 Z: 0.313 Rdelta: 0.00311
#> 223 Number of features: 117 Max AUC: 0.905 AUC: 0.865 Z: 0.340 Rdelta: 0.00249
#> 224 Number of features: 117 Max AUC: 0.905 AUC: 0.873 Z: 0.326 Rdelta: 0.00199
#> 225 Number of features: 117 Max AUC: 0.905 AUC: 0.867 Z: 0.244 Rdelta: 0.00159
#> 226 Number of features: 117 Max AUC: 0.905 AUC: 0.875 Z: 0.304 Rdelta: 0.00127
#> 227 Number of features: 117 Max AUC: 0.905 AUC: 0.875 Z: 0.295 Rdelta: 0.00102
#> 228 Number of features: 117 Max AUC: 0.905 AUC: 0.864 Z: 0.329 Rdelta: 0.00082
#> 229 Number of features: 118 Max AUC: 0.905 AUC: 0.879 Z: 0.347 Rdelta: 0.01073
#> 230 Number of features: 118 Max AUC: 0.905 AUC: 0.871 Z: 0.337 Rdelta: 0.00859
#> 231 Number of features: 118 Max AUC: 0.905 AUC: 0.862 Z: 0.194 Rdelta: 0.00687
#> 232 Number of features: 118 Max AUC: 0.905 AUC: 0.869 Z: 0.382 Rdelta: 0.00550
#> 233 Number of features: 118 Max AUC: 0.905 AUC: 0.868 Z: 0.311 Rdelta: 0.00440
#> 234 Number of features: 118 Max AUC: 0.905 AUC: 0.867 Z: 0.333 Rdelta: 0.00352
#> 235 Number of features: 118 Max AUC: 0.905 AUC: 0.870 Z: 0.331 Rdelta: 0.00281
#> 236 Number of features: 118 Max AUC: 0.905 AUC: 0.866 Z: 0.304 Rdelta: 0.00225
#> 237 Number of features: 118 Max AUC: 0.905 AUC: 0.869 Z: 0.312 Rdelta: 0.00180
#> 238 Number of features: 118 Max AUC: 0.905 AUC: 0.873 Z: 0.317 Rdelta: 0.00144
#> 239 Number of features: 118 Max AUC: 0.905 AUC: 0.872 Z: 0.320 Rdelta: 0.00115
#> 240 Number of features: 118 Max AUC: 0.905 AUC: 0.874 Z: 0.269 Rdelta: 0.00092
#> 241 Number of features: 118 Max AUC: 0.905 AUC: 0.858 Z: 0.295 Rdelta: 0.00074
#> 242 Number of features: 118 Max AUC: 0.905 AUC: 0.869 Z: 0.314 Rdelta: 0.00059
#> 243 Number of features: 118 Max AUC: 0.905 AUC: 0.865 Z: 0.363 Rdelta: 0.00047
#> 244 Number of features: 119 Max AUC: 0.905 AUC: 0.879 Z: 0.346 Rdelta: 0.01042
#> 245 Number of features: 119 Max AUC: 0.905 AUC: 0.870 Z: 0.336 Rdelta: 0.00834
#> 246 Number of features: 119 Max AUC: 0.905 AUC: 0.873 Z: 0.361 Rdelta: 0.00667
#> 247 Number of features: 119 Max AUC: 0.905 AUC: 0.874 Z: 0.316 Rdelta: 0.00534
#> 248 Number of features: 119 Max AUC: 0.905 AUC: 0.872 Z: 0.367 Rdelta: 0.00427
#> 249 Number of features: 119 Max AUC: 0.905 AUC: 0.869 Z: 0.267 Rdelta: 0.00342
#> 250 Number of features: 119 Max AUC: 0.905 AUC: 0.864 Z: 0.366 Rdelta: 0.00273
#> 251 Number of features: 119 Max AUC: 0.905 AUC: 0.871 Z: 0.358 Rdelta: 0.00219
#> 252 Number of features: 119 Max AUC: 0.905 AUC: 0.862 Z: 0.289 Rdelta: 0.00175
#> 253 Number of features: 119 Max AUC: 0.905 AUC: 0.868 Z: 0.348 Rdelta: 0.00140
#> 254 Number of features: 119 Max AUC: 0.905 AUC: 0.864 Z: 0.348 Rdelta: 0.00112
#> 255 Number of features: 119 Max AUC: 0.905 AUC: 0.863 Z: 0.297 Rdelta: 0.00090
#> 256 Number of features: 119 Max AUC: 0.905 AUC: 0.865 Z: 0.331 Rdelta: 0.00072
#> 257 Number of features: 119 Max AUC: 0.905 AUC: 0.864 Z: 0.328 Rdelta: 0.00057
#> 258 Number of features: 119 Max AUC: 0.905 AUC: 0.873 Z: 0.333 Rdelta: 0.00046
#> 259 Number of features: 119 Max AUC: 0.905 AUC: 0.872 Z: 0.332 Rdelta: 0.00037
#> 260 Number of features: 119 Max AUC: 0.905 AUC: 0.868 Z: 0.328 Rdelta: 0.00029
#> 261 Number of features: 119 Max AUC: 0.905 AUC: 0.862 Z: 0.327 Rdelta: 0.00023
#> 262 Number of features: 119 Max AUC: 0.905 AUC: 0.870 Z: 0.326 Rdelta: 0.00019
#> 263 Number of features: 119 Max AUC: 0.905 AUC: 0.867 Z: 0.274 Rdelta: 0.00015
#> 264 Number of features: 119 Max AUC: 0.905 AUC: 0.874 Z: 0.300 Rdelta: 0.00012
#> 265 Number of features: 119 Max AUC: 0.905 AUC: 0.856 Z: 0.359 Rdelta: 0.00010
#> user system elapsed
#> 246.61 0.00 246.91
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
system.time(signatureControl <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Control"))
#> 7 Number of features: 7 Max AUC: 0.410 AUC: 0.410 Z: -0.281 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.410 AUC: 0.401 Z: -0.349 Rdelta: 0.10000
#> 9 Number of features: 8 Max AUC: 0.410 AUC: 0.381 Z: -0.321 Rdelta: 0.08000
#> 10 Number of features: 9 Max AUC: 0.410 AUC: 0.396 Z: -0.340 Rdelta: 0.08200
#> 11 Number of features: 10 Max AUC: 0.410 AUC: 0.395 Z: -0.333 Rdelta: 0.08380
#> 12 Number of features: 10 Max AUC: 0.410 AUC: 0.367 Z: -0.332 Rdelta: 0.06704
#> 13 Number of features: 10 Max AUC: 0.410 AUC: 0.361 Z: -0.363 Rdelta: 0.05363
#> 14 Number of features: 11 Max AUC: 0.410 AUC: 0.396 Z: -0.352 Rdelta: 0.05827
#> 15 Number of features: 11 Max AUC: 0.410 AUC: 0.362 Z: -0.415 Rdelta: 0.04662
#> 16 Number of features: 11 Max AUC: 0.410 AUC: 0.370 Z: -0.445 Rdelta: 0.03729
#> 17 Number of features: 11 Max AUC: 0.410 AUC: 0.360 Z: -0.435 Rdelta: 0.02983
#> 18 Number of features: 11 Max AUC: 0.410 AUC: 0.360 Z: -0.360 Rdelta: 0.02387
#> 19 Number of features: 11 Max AUC: 0.410 AUC: 0.361 Z: -0.412 Rdelta: 0.01909
#> 20 Number of features: 11 Max AUC: 0.410 AUC: 0.364 Z: -0.386 Rdelta: 0.01527
#> 21 Number of features: 11 Max AUC: 0.410 AUC: 0.354 Z: -0.361 Rdelta: 0.01222
#> 22 Number of features: 11 Max AUC: 0.410 AUC: 0.366 Z: -0.349 Rdelta: 0.00978
#> 23 Number of features: 11 Max AUC: 0.410 AUC: 0.359 Z: -0.327 Rdelta: 0.00782
#> 24 Number of features: 12 Max AUC: 0.410 AUC: 0.403 Z: -0.263 Rdelta: 0.01704
#> 25 Number of features: 12 Max AUC: 0.410 AUC: 0.387 Z: -0.352 Rdelta: 0.01363
#> 26 Number of features: 13 Max AUC: 0.410 AUC: 0.403 Z: -0.321 Rdelta: 0.02227
#> 27 Number of features: 13 Max AUC: 0.410 AUC: 0.357 Z: -0.419 Rdelta: 0.01781
#> 28 Number of features: 13 Max AUC: 0.410 AUC: 0.359 Z: -0.418 Rdelta: 0.01425
#> 29 Number of features: 14 Max AUC: 0.410 AUC: 0.399 Z: -0.358 Rdelta: 0.02283
#> 30 Number of features: 14 Max AUC: 0.410 AUC: 0.388 Z: -0.385 Rdelta: 0.01826
#> 31 Number of features: 14 Max AUC: 0.410 AUC: 0.373 Z: -0.416 Rdelta: 0.01461
#> 32 Number of features: 14 Max AUC: 0.410 AUC: 0.361 Z: -0.416 Rdelta: 0.01169
#> 33 Number of features: 14 Max AUC: 0.410 AUC: 0.367 Z: -0.358 Rdelta: 0.00935
#> 34 Number of features: 14 Max AUC: 0.410 AUC: 0.367 Z: -0.388 Rdelta: 0.00748
#> 35 Number of features: 14 Max AUC: 0.410 AUC: 0.359 Z: -0.429 Rdelta: 0.00598
#> 36 Number of features: 14 Max AUC: 0.410 AUC: 0.363 Z: -0.476 Rdelta: 0.00479
#> 37 Number of features: 14 Max AUC: 0.410 AUC: 0.367 Z: -0.462 Rdelta: 0.00383
#> 38 Number of features: 14 Max AUC: 0.410 AUC: 0.369 Z: -0.395 Rdelta: 0.00306
#> 39 Number of features: 15 Max AUC: 0.410 AUC: 0.398 Z: -0.350 Rdelta: 0.01276
#> 40 Number of features: 15 Max AUC: 0.410 AUC: 0.367 Z: -0.385 Rdelta: 0.01021
#> 41 Number of features: 15 Max AUC: 0.410 AUC: 0.353 Z: -0.368 Rdelta: 0.00816
#> 42 Number of features: 15 Max AUC: 0.410 AUC: 0.386 Z: -0.380 Rdelta: 0.00653
#> 43 Number of features: 15 Max AUC: 0.410 AUC: 0.363 Z: -0.416 Rdelta: 0.00523
#> 44 Number of features: 15 Max AUC: 0.410 AUC: 0.356 Z: -0.365 Rdelta: 0.00418
#> 45 Number of features: 15 Max AUC: 0.410 AUC: 0.392 Z: -0.401 Rdelta: 0.00334
#> 46 Number of features: 15 Max AUC: 0.410 AUC: 0.374 Z: -0.231 Rdelta: 0.00268
#> 47 Number of features: 15 Max AUC: 0.410 AUC: 0.378 Z: -0.435 Rdelta: 0.00214
#> 48 Number of features: 16 Max AUC: 0.413 AUC: 0.413 Z: -0.062 Rdelta: 0.01193
#> 49 Number of features: 16 Max AUC: 0.413 AUC: 0.370 Z: -0.339 Rdelta: 0.00954
#> 50 Number of features: 16 Max AUC: 0.413 AUC: 0.382 Z: -0.390 Rdelta: 0.00763
#> 51 Number of features: 16 Max AUC: 0.413 AUC: 0.372 Z: -0.399 Rdelta: 0.00611
#> 52 Number of features: 16 Max AUC: 0.413 AUC: 0.380 Z: -0.375 Rdelta: 0.00489
#> 53 Number of features: 16 Max AUC: 0.413 AUC: 0.360 Z: -0.393 Rdelta: 0.00391
#> 54 Number of features: 16 Max AUC: 0.413 AUC: 0.362 Z: -0.424 Rdelta: 0.00313
#> 55 Number of features: 16 Max AUC: 0.413 AUC: 0.392 Z: -0.329 Rdelta: 0.00250
#> 56 Number of features: 16 Max AUC: 0.413 AUC: 0.390 Z: -0.389 Rdelta: 0.00200
#> 57 Number of features: 16 Max AUC: 0.413 AUC: 0.367 Z: -0.299 Rdelta: 0.00160
#> 58 Number of features: 16 Max AUC: 0.413 AUC: 0.355 Z: -0.403 Rdelta: 0.00128
#> 59 Number of features: 16 Max AUC: 0.413 AUC: 0.394 Z: -0.144 Rdelta: 0.00102
#> 60 Number of features: 16 Max AUC: 0.413 AUC: 0.377 Z: -0.293 Rdelta: 0.00082
#> 61 Number of features: 16 Max AUC: 0.413 AUC: 0.397 Z: -0.326 Rdelta: 0.00066
#> 62 Number of features: 16 Max AUC: 0.413 AUC: 0.362 Z: -0.398 Rdelta: 0.00052
#> 63 Number of features: 16 Max AUC: 0.413 AUC: 0.390 Z: -0.369 Rdelta: 0.00042
#> 64 Number of features: 16 Max AUC: 0.413 AUC: 0.400 Z: -0.334 Rdelta: 0.00034
#> 65 Number of features: 16 Max AUC: 0.413 AUC: 0.376 Z: -0.373 Rdelta: 0.00027
#> 66 Number of features: 16 Max AUC: 0.413 AUC: 0.400 Z: -0.352 Rdelta: 0.00021
#> 67 Number of features: 16 Max AUC: 0.413 AUC: 0.368 Z: -0.398 Rdelta: 0.00017
#> 68 Number of features: 16 Max AUC: 0.413 AUC: 0.373 Z: -0.377 Rdelta: 0.00014
#> 69 Number of features: 16 Max AUC: 0.413 AUC: 0.372 Z: -0.380 Rdelta: 0.00011
#> 70 Number of features: 16 Max AUC: 0.413 AUC: 0.397 Z: -0.361 Rdelta: 0.00009
#> user system elapsed
#> 16.97 0.00 16.99
testDistance_control <- signatureDistance(signatureControl$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="MAN"))
#> 7 Number of features: 7 Max AUC: 0.684 AUC: 0.683 Z: 0.895 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.687 AUC: 0.687 Z: 0.918 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.741 AUC: 0.741 Z: 1.340 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.751 AUC: 0.751 Z: 1.283 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.751 AUC: 0.745 Z: 1.336 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.756 AUC: 0.756 Z: 1.485 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.756 AUC: 0.753 Z: 1.100 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.761 AUC: 0.761 Z: 1.367 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.763 AUC: 0.763 Z: 0.931 Rdelta: 0.10000
#> 16 Number of features: 16 Max AUC: 0.765 AUC: 0.765 Z: 0.539 Rdelta: 0.10000
#> 17 Number of features: 17 Max AUC: 0.765 AUC: 0.765 Z: 0.833 Rdelta: 0.10000
#> 18 Number of features: 18 Max AUC: 0.765 AUC: 0.765 Z: 0.780 Rdelta: 0.10000
#> 19 Number of features: 18 Max AUC: 0.765 AUC: 0.755 Z: 0.684 Rdelta: 0.08000
#> 20 Number of features: 18 Max AUC: 0.765 AUC: 0.755 Z: 0.535 Rdelta: 0.06400
#> 21 Number of features: 18 Max AUC: 0.765 AUC: 0.754 Z: 0.636 Rdelta: 0.05120
#> 22 Number of features: 19 Max AUC: 0.765 AUC: 0.764 Z: 0.476 Rdelta: 0.05608
#> 23 Number of features: 20 Max AUC: 0.765 AUC: 0.765 Z: 0.254 Rdelta: 0.06047
#> 24 Number of features: 21 Max AUC: 0.765 AUC: 0.765 Z: 0.239 Rdelta: 0.06442
#> 25 Number of features: 22 Max AUC: 0.767 AUC: 0.767 Z: 0.591 Rdelta: 0.06798
#> 26 Number of features: 23 Max AUC: 0.778 AUC: 0.778 Z: 0.572 Rdelta: 0.07118
#> 27 Number of features: 23 Max AUC: 0.778 AUC: 0.767 Z: 0.542 Rdelta: 0.05695
#> 28 Number of features: 24 Max AUC: 0.778 AUC: 0.777 Z: 0.223 Rdelta: 0.06125
#> 29 Number of features: 24 Max AUC: 0.778 AUC: 0.755 Z: 0.221 Rdelta: 0.04900
#> 30 Number of features: 24 Max AUC: 0.778 AUC: 0.769 Z: 0.361 Rdelta: 0.03920
#> 31 Number of features: 24 Max AUC: 0.778 AUC: 0.766 Z: 0.463 Rdelta: 0.03136
#> 32 Number of features: 24 Max AUC: 0.778 AUC: 0.767 Z: 0.291 Rdelta: 0.02509
#> 33 Number of features: 24 Max AUC: 0.778 AUC: 0.761 Z: 0.099 Rdelta: 0.02007
#> 34 Number of features: 24 Max AUC: 0.778 AUC: 0.766 Z: 0.249 Rdelta: 0.01606
#> 35 Number of features: 24 Max AUC: 0.778 AUC: 0.766 Z: 0.492 Rdelta: 0.01285
#> 36 Number of features: 24 Max AUC: 0.778 AUC: 0.760 Z: 0.292 Rdelta: 0.01028
#> 37 Number of features: 25 Max AUC: 0.778 AUC: 0.771 Z: 0.341 Rdelta: 0.01925
#> 38 Number of features: 25 Max AUC: 0.778 AUC: 0.769 Z: 0.161 Rdelta: 0.01540
#> 39 Number of features: 25 Max AUC: 0.778 AUC: 0.768 Z: 0.264 Rdelta: 0.01232
#> 40 Number of features: 25 Max AUC: 0.778 AUC: 0.762 Z: 0.284 Rdelta: 0.00986
#> 41 Number of features: 26 Max AUC: 0.778 AUC: 0.767 Z: 0.455 Rdelta: 0.01887
#> 42 Number of features: 27 Max AUC: 0.778 AUC: 0.773 Z: 0.656 Rdelta: 0.02698
#> 43 Number of features: 28 Max AUC: 0.778 AUC: 0.777 Z: 0.578 Rdelta: 0.03428
#> 44 Number of features: 28 Max AUC: 0.778 AUC: 0.769 Z: 0.285 Rdelta: 0.02743
#> 45 Number of features: 29 Max AUC: 0.778 AUC: 0.777 Z: 0.322 Rdelta: 0.03468
#> 46 Number of features: 30 Max AUC: 0.778 AUC: 0.771 Z: 0.295 Rdelta: 0.04122
#> 47 Number of features: 31 Max AUC: 0.778 AUC: 0.775 Z: 0.256 Rdelta: 0.04709
#> 48 Number of features: 31 Max AUC: 0.778 AUC: 0.764 Z: 0.279 Rdelta: 0.03768
#> 49 Number of features: 32 Max AUC: 0.780 AUC: 0.780 Z: 0.285 Rdelta: 0.04391
#> 50 Number of features: 32 Max AUC: 0.780 AUC: 0.768 Z: 0.253 Rdelta: 0.03513
#> 51 Number of features: 33 Max AUC: 0.780 AUC: 0.776 Z: 0.321 Rdelta: 0.04161
#> 52 Number of features: 34 Max AUC: 0.780 AUC: 0.774 Z: 0.255 Rdelta: 0.04745
#> 53 Number of features: 34 Max AUC: 0.780 AUC: 0.766 Z: 0.431 Rdelta: 0.03796
#> 54 Number of features: 35 Max AUC: 0.780 AUC: 0.777 Z: 0.668 Rdelta: 0.04417
#> 55 Number of features: 36 Max AUC: 0.780 AUC: 0.780 Z: 0.333 Rdelta: 0.04975
#> 56 Number of features: 37 Max AUC: 0.780 AUC: 0.777 Z: 0.484 Rdelta: 0.05477
#> 57 Number of features: 37 Max AUC: 0.780 AUC: 0.767 Z: 0.432 Rdelta: 0.04382
#> 58 Number of features: 38 Max AUC: 0.780 AUC: 0.773 Z: 0.558 Rdelta: 0.04944
#> 59 Number of features: 39 Max AUC: 0.780 AUC: 0.780 Z: 0.391 Rdelta: 0.05449
#> 60 Number of features: 39 Max AUC: 0.780 AUC: 0.766 Z: 0.489 Rdelta: 0.04360
#> 61 Number of features: 40 Max AUC: 0.782 AUC: 0.782 Z: 0.289 Rdelta: 0.04924
#> 62 Number of features: 41 Max AUC: 0.782 AUC: 0.780 Z: 0.572 Rdelta: 0.05431
#> 63 Number of features: 42 Max AUC: 0.782 AUC: 0.780 Z: 0.467 Rdelta: 0.05888
#> 64 Number of features: 42 Max AUC: 0.782 AUC: 0.773 Z: 0.299 Rdelta: 0.04710
#> 65 Number of features: 43 Max AUC: 0.782 AUC: 0.773 Z: 0.479 Rdelta: 0.05239
#> 66 Number of features: 43 Max AUC: 0.782 AUC: 0.769 Z: 0.473 Rdelta: 0.04192
#> 67 Number of features: 43 Max AUC: 0.782 AUC: 0.766 Z: 0.270 Rdelta: 0.03353
#> 68 Number of features: 43 Max AUC: 0.782 AUC: 0.765 Z: 0.444 Rdelta: 0.02683
#> 69 Number of features: 44 Max AUC: 0.782 AUC: 0.775 Z: 0.451 Rdelta: 0.03414
#> 70 Number of features: 45 Max AUC: 0.782 AUC: 0.779 Z: 0.435 Rdelta: 0.04073
#> 71 Number of features: 46 Max AUC: 0.782 AUC: 0.776 Z: 0.284 Rdelta: 0.04666
#> 72 Number of features: 46 Max AUC: 0.782 AUC: 0.770 Z: 0.374 Rdelta: 0.03732
#> 73 Number of features: 46 Max AUC: 0.782 AUC: 0.766 Z: 0.363 Rdelta: 0.02986
#> 74 Number of features: 47 Max AUC: 0.782 AUC: 0.773 Z: 0.305 Rdelta: 0.03687
#> 75 Number of features: 47 Max AUC: 0.782 AUC: 0.770 Z: 0.414 Rdelta: 0.02950
#> 76 Number of features: 48 Max AUC: 0.782 AUC: 0.775 Z: 0.289 Rdelta: 0.03655
#> 77 Number of features: 49 Max AUC: 0.782 AUC: 0.776 Z: 0.383 Rdelta: 0.04289
#> 78 Number of features: 50 Max AUC: 0.782 AUC: 0.771 Z: 0.432 Rdelta: 0.04860
#> 79 Number of features: 51 Max AUC: 0.782 AUC: 0.772 Z: 0.466 Rdelta: 0.05374
#> 80 Number of features: 51 Max AUC: 0.782 AUC: 0.764 Z: 0.552 Rdelta: 0.04300
#> 81 Number of features: 51 Max AUC: 0.782 AUC: 0.769 Z: 0.384 Rdelta: 0.03440
#> 82 Number of features: 51 Max AUC: 0.782 AUC: 0.768 Z: 0.463 Rdelta: 0.02752
#> 83 Number of features: 52 Max AUC: 0.782 AUC: 0.780 Z: 0.460 Rdelta: 0.03477
#> 84 Number of features: 52 Max AUC: 0.782 AUC: 0.768 Z: 0.411 Rdelta: 0.02781
#> 85 Number of features: 52 Max AUC: 0.782 AUC: 0.764 Z: 0.375 Rdelta: 0.02225
#> 86 Number of features: 52 Max AUC: 0.782 AUC: 0.765 Z: 0.153 Rdelta: 0.01780
#> 87 Number of features: 52 Max AUC: 0.782 AUC: 0.767 Z: 0.237 Rdelta: 0.01424
#> 88 Number of features: 53 Max AUC: 0.782 AUC: 0.775 Z: 0.259 Rdelta: 0.02282
#> 89 Number of features: 53 Max AUC: 0.782 AUC: 0.764 Z: 0.308 Rdelta: 0.01825
#> 90 Number of features: 53 Max AUC: 0.782 AUC: 0.772 Z: 0.415 Rdelta: 0.01460
#> 91 Number of features: 53 Max AUC: 0.782 AUC: 0.764 Z: 0.287 Rdelta: 0.01168
#> 92 Number of features: 53 Max AUC: 0.782 AUC: 0.757 Z: 0.356 Rdelta: 0.00935
#> 93 Number of features: 54 Max AUC: 0.782 AUC: 0.774 Z: 0.490 Rdelta: 0.01841
#> 94 Number of features: 54 Max AUC: 0.782 AUC: 0.768 Z: 0.393 Rdelta: 0.01473
#> 95 Number of features: 54 Max AUC: 0.782 AUC: 0.768 Z: 0.269 Rdelta: 0.01178
#> 96 Number of features: 54 Max AUC: 0.782 AUC: 0.756 Z: 0.493 Rdelta: 0.00943
#> 97 Number of features: 54 Max AUC: 0.782 AUC: 0.758 Z: 0.588 Rdelta: 0.00754
#> 98 Number of features: 55 Max AUC: 0.782 AUC: 0.774 Z: 0.303 Rdelta: 0.01679
#> 99 Number of features: 55 Max AUC: 0.782 AUC: 0.757 Z: 0.390 Rdelta: 0.01343
#> 100 Number of features: 56 Max AUC: 0.782 AUC: 0.777 Z: 0.399 Rdelta: 0.02209
#> 101 Number of features: 56 Max AUC: 0.782 AUC: 0.760 Z: 0.426 Rdelta: 0.01767
#> 102 Number of features: 57 Max AUC: 0.782 AUC: 0.776 Z: 0.322 Rdelta: 0.02590
#> 103 Number of features: 58 Max AUC: 0.782 AUC: 0.776 Z: 0.468 Rdelta: 0.03331
#> 104 Number of features: 58 Max AUC: 0.782 AUC: 0.771 Z: 0.402 Rdelta: 0.02665
#> 105 Number of features: 59 Max AUC: 0.782 AUC: 0.773 Z: 0.307 Rdelta: 0.03398
#> 106 Number of features: 59 Max AUC: 0.782 AUC: 0.764 Z: 0.283 Rdelta: 0.02719
#> 107 Number of features: 59 Max AUC: 0.782 AUC: 0.763 Z: 0.350 Rdelta: 0.02175
#> 108 Number of features: 59 Max AUC: 0.782 AUC: 0.762 Z: 0.350 Rdelta: 0.01740
#> 109 Number of features: 59 Max AUC: 0.782 AUC: 0.760 Z: 0.305 Rdelta: 0.01392
#> 110 Number of features: 59 Max AUC: 0.782 AUC: 0.763 Z: 0.253 Rdelta: 0.01114
#> 111 Number of features: 59 Max AUC: 0.782 AUC: 0.768 Z: 0.399 Rdelta: 0.00891
#> 112 Number of features: 59 Max AUC: 0.782 AUC: 0.771 Z: 0.307 Rdelta: 0.00713
#> 113 Number of features: 59 Max AUC: 0.782 AUC: 0.768 Z: 0.438 Rdelta: 0.00570
#> 114 Number of features: 59 Max AUC: 0.782 AUC: 0.755 Z: 0.385 Rdelta: 0.00456
#> 115 Number of features: 59 Max AUC: 0.782 AUC: 0.767 Z: 0.422 Rdelta: 0.00365
#> 116 Number of features: 59 Max AUC: 0.782 AUC: 0.760 Z: 0.383 Rdelta: 0.00292
#> 117 Number of features: 59 Max AUC: 0.782 AUC: 0.752 Z: 0.386 Rdelta: 0.00234
#> 118 Number of features: 59 Max AUC: 0.782 AUC: 0.752 Z: 0.448 Rdelta: 0.00187
#> 119 Number of features: 59 Max AUC: 0.782 AUC: 0.766 Z: 0.373 Rdelta: 0.00149
#> 120 Number of features: 59 Max AUC: 0.782 AUC: 0.765 Z: 0.315 Rdelta: 0.00120
#> 121 Number of features: 59 Max AUC: 0.782 AUC: 0.759 Z: 0.301 Rdelta: 0.00096
#> 122 Number of features: 59 Max AUC: 0.782 AUC: 0.765 Z: 0.323 Rdelta: 0.00077
#> 123 Number of features: 60 Max AUC: 0.782 AUC: 0.778 Z: 0.473 Rdelta: 0.01069
#> 124 Number of features: 60 Max AUC: 0.782 AUC: 0.766 Z: 0.331 Rdelta: 0.00855
#> 125 Number of features: 60 Max AUC: 0.782 AUC: 0.770 Z: 0.378 Rdelta: 0.00684
#> 126 Number of features: 61 Max AUC: 0.782 AUC: 0.778 Z: 0.307 Rdelta: 0.01616
#> 127 Number of features: 62 Max AUC: 0.782 AUC: 0.773 Z: 0.307 Rdelta: 0.02454
#> 128 Number of features: 63 Max AUC: 0.782 AUC: 0.779 Z: 0.273 Rdelta: 0.03209
#> 129 Number of features: 63 Max AUC: 0.782 AUC: 0.768 Z: 0.272 Rdelta: 0.02567
#> 130 Number of features: 63 Max AUC: 0.782 AUC: 0.771 Z: 0.365 Rdelta: 0.02054
#> 131 Number of features: 64 Max AUC: 0.782 AUC: 0.777 Z: 0.499 Rdelta: 0.02848
#> 132 Number of features: 64 Max AUC: 0.782 AUC: 0.762 Z: 0.414 Rdelta: 0.02279
#> 133 Number of features: 64 Max AUC: 0.782 AUC: 0.763 Z: 0.354 Rdelta: 0.01823
#> 134 Number of features: 64 Max AUC: 0.782 AUC: 0.764 Z: 0.320 Rdelta: 0.01458
#> 135 Number of features: 64 Max AUC: 0.782 AUC: 0.759 Z: 0.479 Rdelta: 0.01167
#> 136 Number of features: 64 Max AUC: 0.782 AUC: 0.760 Z: 0.223 Rdelta: 0.00933
#> 137 Number of features: 64 Max AUC: 0.782 AUC: 0.767 Z: 0.502 Rdelta: 0.00747
#> 138 Number of features: 64 Max AUC: 0.782 AUC: 0.764 Z: 0.369 Rdelta: 0.00597
#> 139 Number of features: 64 Max AUC: 0.782 AUC: 0.766 Z: 0.358 Rdelta: 0.00478
#> 140 Number of features: 64 Max AUC: 0.782 AUC: 0.757 Z: 0.318 Rdelta: 0.00382
#> 141 Number of features: 64 Max AUC: 0.782 AUC: 0.770 Z: 0.256 Rdelta: 0.00306
#> 142 Number of features: 64 Max AUC: 0.782 AUC: 0.761 Z: 0.309 Rdelta: 0.00245
#> 143 Number of features: 64 Max AUC: 0.782 AUC: 0.764 Z: 0.527 Rdelta: 0.00196
#> 144 Number of features: 64 Max AUC: 0.782 AUC: 0.766 Z: 0.429 Rdelta: 0.00157
#> 145 Number of features: 64 Max AUC: 0.782 AUC: 0.769 Z: 0.365 Rdelta: 0.00125
#> 146 Number of features: 64 Max AUC: 0.782 AUC: 0.761 Z: 0.136 Rdelta: 0.00100
#> 147 Number of features: 64 Max AUC: 0.782 AUC: 0.763 Z: 0.404 Rdelta: 0.00080
#> 148 Number of features: 64 Max AUC: 0.782 AUC: 0.763 Z: 0.413 Rdelta: 0.00064
#> 149 Number of features: 64 Max AUC: 0.782 AUC: 0.756 Z: 0.332 Rdelta: 0.00051
#> 150 Number of features: 64 Max AUC: 0.782 AUC: 0.762 Z: 0.311 Rdelta: 0.00041
#> 151 Number of features: 64 Max AUC: 0.782 AUC: 0.761 Z: 0.432 Rdelta: 0.00033
#> 152 Number of features: 64 Max AUC: 0.782 AUC: 0.767 Z: 0.386 Rdelta: 0.00026
#> 153 Number of features: 65 Max AUC: 0.782 AUC: 0.773 Z: 0.418 Rdelta: 0.01024
#> 154 Number of features: 65 Max AUC: 0.782 AUC: 0.765 Z: 0.452 Rdelta: 0.00819
#> 155 Number of features: 65 Max AUC: 0.782 AUC: 0.761 Z: 0.343 Rdelta: 0.00655
#> 156 Number of features: 65 Max AUC: 0.782 AUC: 0.770 Z: 0.361 Rdelta: 0.00524
#> 157 Number of features: 65 Max AUC: 0.782 AUC: 0.766 Z: 0.407 Rdelta: 0.00419
#> 158 Number of features: 65 Max AUC: 0.782 AUC: 0.758 Z: 0.438 Rdelta: 0.00335
#> 159 Number of features: 66 Max AUC: 0.782 AUC: 0.771 Z: 0.337 Rdelta: 0.01302
#> 160 Number of features: 66 Max AUC: 0.782 AUC: 0.750 Z: 0.476 Rdelta: 0.01042
#> 161 Number of features: 66 Max AUC: 0.782 AUC: 0.758 Z: 0.372 Rdelta: 0.00833
#> 162 Number of features: 66 Max AUC: 0.782 AUC: 0.750 Z: 0.374 Rdelta: 0.00667
#> 163 Number of features: 66 Max AUC: 0.782 AUC: 0.756 Z: 0.345 Rdelta: 0.00533
#> 164 Number of features: 66 Max AUC: 0.782 AUC: 0.762 Z: 0.443 Rdelta: 0.00427
#> 165 Number of features: 66 Max AUC: 0.782 AUC: 0.742 Z: 0.297 Rdelta: 0.00341
#> 166 Number of features: 66 Max AUC: 0.782 AUC: 0.763 Z: 0.499 Rdelta: 0.00273
#> 167 Number of features: 67 Max AUC: 0.782 AUC: 0.774 Z: 0.364 Rdelta: 0.01246
#> 168 Number of features: 67 Max AUC: 0.782 AUC: 0.757 Z: 0.474 Rdelta: 0.00997
#> 169 Number of features: 67 Max AUC: 0.782 AUC: 0.758 Z: 0.400 Rdelta: 0.00797
#> 170 Number of features: 67 Max AUC: 0.782 AUC: 0.762 Z: 0.337 Rdelta: 0.00638
#> 171 Number of features: 68 Max AUC: 0.782 AUC: 0.770 Z: 0.370 Rdelta: 0.01574
#> 172 Number of features: 68 Max AUC: 0.782 AUC: 0.763 Z: 0.309 Rdelta: 0.01259
#> 173 Number of features: 68 Max AUC: 0.782 AUC: 0.753 Z: 0.136 Rdelta: 0.01007
#> 174 Number of features: 68 Max AUC: 0.782 AUC: 0.737 Z: 0.278 Rdelta: 0.00806
#> 175 Number of features: 68 Max AUC: 0.782 AUC: 0.769 Z: 0.328 Rdelta: 0.00645
#> 176 Number of features: 68 Max AUC: 0.782 AUC: 0.766 Z: 0.242 Rdelta: 0.00516
#> 177 Number of features: 68 Max AUC: 0.782 AUC: 0.766 Z: 0.395 Rdelta: 0.00413
#> 178 Number of features: 68 Max AUC: 0.782 AUC: 0.744 Z: 0.357 Rdelta: 0.00330
#> 179 Number of features: 68 Max AUC: 0.782 AUC: 0.761 Z: 0.277 Rdelta: 0.00264
#> 180 Number of features: 68 Max AUC: 0.782 AUC: 0.764 Z: 0.341 Rdelta: 0.00211
#> 181 Number of features: 68 Max AUC: 0.782 AUC: 0.746 Z: 0.285 Rdelta: 0.00169
#> 182 Number of features: 68 Max AUC: 0.782 AUC: 0.742 Z: 0.191 Rdelta: 0.00135
#> 183 Number of features: 68 Max AUC: 0.782 AUC: 0.753 Z: 0.227 Rdelta: 0.00108
#> 184 Number of features: 68 Max AUC: 0.782 AUC: 0.763 Z: 0.316 Rdelta: 0.00087
#> 185 Number of features: 68 Max AUC: 0.782 AUC: 0.735 Z: 0.305 Rdelta: 0.00069
#> 186 Number of features: 68 Max AUC: 0.782 AUC: 0.740 Z: 0.459 Rdelta: 0.00055
#> 187 Number of features: 69 Max AUC: 0.782 AUC: 0.776 Z: 0.516 Rdelta: 0.01050
#> 188 Number of features: 69 Max AUC: 0.782 AUC: 0.751 Z: 0.398 Rdelta: 0.00840
#> 189 Number of features: 69 Max AUC: 0.782 AUC: 0.762 Z: 0.635 Rdelta: 0.00672
#> 190 Number of features: 69 Max AUC: 0.782 AUC: 0.758 Z: 0.360 Rdelta: 0.00538
#> 191 Number of features: 69 Max AUC: 0.782 AUC: 0.752 Z: 0.343 Rdelta: 0.00430
#> 192 Number of features: 69 Max AUC: 0.782 AUC: 0.753 Z: 0.319 Rdelta: 0.00344
#> 193 Number of features: 69 Max AUC: 0.782 AUC: 0.754 Z: 0.571 Rdelta: 0.00275
#> 194 Number of features: 69 Max AUC: 0.782 AUC: 0.762 Z: 0.352 Rdelta: 0.00220
#> 195 Number of features: 69 Max AUC: 0.782 AUC: 0.761 Z: 0.342 Rdelta: 0.00176
#> 196 Number of features: 69 Max AUC: 0.782 AUC: 0.754 Z: 0.393 Rdelta: 0.00141
#> 197 Number of features: 69 Max AUC: 0.782 AUC: 0.749 Z: 0.307 Rdelta: 0.00113
#> 198 Number of features: 69 Max AUC: 0.782 AUC: 0.739 Z: 0.555 Rdelta: 0.00090
#> 199 Number of features: 69 Max AUC: 0.782 AUC: 0.758 Z: 0.344 Rdelta: 0.00072
#> 200 Number of features: 69 Max AUC: 0.782 AUC: 0.752 Z: 0.422 Rdelta: 0.00058
#> 201 Number of features: 69 Max AUC: 0.782 AUC: 0.757 Z: 0.283 Rdelta: 0.00046
#> 202 Number of features: 69 Max AUC: 0.782 AUC: 0.742 Z: 0.316 Rdelta: 0.00037
#> 203 Number of features: 69 Max AUC: 0.782 AUC: 0.758 Z: 0.341 Rdelta: 0.00030
#> 204 Number of features: 69 Max AUC: 0.782 AUC: 0.755 Z: 0.414 Rdelta: 0.00024
#> 205 Number of features: 69 Max AUC: 0.782 AUC: 0.761 Z: 0.493 Rdelta: 0.00019
#> 206 Number of features: 69 Max AUC: 0.782 AUC: 0.752 Z: 0.259 Rdelta: 0.00015
#> 207 Number of features: 69 Max AUC: 0.782 AUC: 0.726 Z: 0.336 Rdelta: 0.00012
#> 208 Number of features: 69 Max AUC: 0.782 AUC: 0.745 Z: 0.427 Rdelta: 0.00010
#> user system elapsed
#> 130.24 0.00 130.35
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"MAN")+signatureDistance(signature$controlTemplate,validLabeled,"MAN")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
#############################################################################################
varlist <- names(arceneCVOne$bagging$frequencyTable)
#############################################################################################
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="pearson"))
#> 7 Number of features: 7 Max AUC: 0.519 AUC: 0.519 Z: 0.048 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.538 AUC: 0.538 Z: 0.132 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.541 AUC: 0.541 Z: 0.167 Rdelta: 0.10000
#> 10 Number of features: 9 Max AUC: 0.541 AUC: 0.500 Z: 0.010 Rdelta: 0.08000
#> 11 Number of features: 10 Max AUC: 0.547 AUC: 0.547 Z: 0.113 Rdelta: 0.08200
#> 12 Number of features: 10 Max AUC: 0.547 AUC: 0.516 Z: 0.058 Rdelta: 0.06560
#> 13 Number of features: 10 Max AUC: 0.547 AUC: 0.522 Z: 0.056 Rdelta: 0.05248
#> 14 Number of features: 10 Max AUC: 0.547 AUC: 0.533 Z: 0.066 Rdelta: 0.04198
#> 15 Number of features: 10 Max AUC: 0.547 AUC: 0.489 Z: -0.015 Rdelta: 0.03359
#> 16 Number of features: 10 Max AUC: 0.547 AUC: 0.528 Z: 0.073 Rdelta: 0.02687
#> 17 Number of features: 10 Max AUC: 0.547 AUC: 0.489 Z: 0.024 Rdelta: 0.02150
#> 18 Number of features: 11 Max AUC: 0.547 AUC: 0.546 Z: 0.102 Rdelta: 0.02935
#> 19 Number of features: 11 Max AUC: 0.547 AUC: 0.522 Z: -0.002 Rdelta: 0.02348
#> 20 Number of features: 11 Max AUC: 0.547 AUC: 0.529 Z: 0.074 Rdelta: 0.01878
#> 21 Number of features: 12 Max AUC: 0.552 AUC: 0.552 Z: 0.174 Rdelta: 0.02690
#> 22 Number of features: 12 Max AUC: 0.552 AUC: 0.517 Z: 0.079 Rdelta: 0.02152
#> 23 Number of features: 12 Max AUC: 0.552 AUC: 0.522 Z: 0.108 Rdelta: 0.01722
#> 24 Number of features: 12 Max AUC: 0.552 AUC: 0.537 Z: 0.145 Rdelta: 0.01377
#> 25 Number of features: 12 Max AUC: 0.552 AUC: 0.524 Z: 0.066 Rdelta: 0.01102
#> 26 Number of features: 12 Max AUC: 0.552 AUC: 0.540 Z: 0.148 Rdelta: 0.00882
#> 27 Number of features: 13 Max AUC: 0.552 AUC: 0.543 Z: 0.194 Rdelta: 0.01793
#> 28 Number of features: 14 Max AUC: 0.552 AUC: 0.541 Z: 0.093 Rdelta: 0.02614
#> 29 Number of features: 14 Max AUC: 0.552 AUC: 0.535 Z: 0.113 Rdelta: 0.02091
#> 30 Number of features: 14 Max AUC: 0.552 AUC: 0.533 Z: 0.056 Rdelta: 0.01673
#> 31 Number of features: 15 Max AUC: 0.611 AUC: 0.611 Z: 0.458 Rdelta: 0.02506
#> 32 Number of features: 16 Max AUC: 0.625 AUC: 0.625 Z: 0.530 Rdelta: 0.03255
#> 33 Number of features: 16 Max AUC: 0.625 AUC: 0.589 Z: 0.463 Rdelta: 0.02604
#> 34 Number of features: 16 Max AUC: 0.625 AUC: 0.610 Z: 0.481 Rdelta: 0.02083
#> 35 Number of features: 16 Max AUC: 0.625 AUC: 0.599 Z: 0.494 Rdelta: 0.01667
#> 36 Number of features: 17 Max AUC: 0.625 AUC: 0.618 Z: 0.544 Rdelta: 0.02500
#> 37 Number of features: 17 Max AUC: 0.625 AUC: 0.591 Z: 0.456 Rdelta: 0.02000
#> 38 Number of features: 18 Max AUC: 0.625 AUC: 0.625 Z: 0.627 Rdelta: 0.02800
#> 39 Number of features: 18 Max AUC: 0.625 AUC: 0.596 Z: 0.374 Rdelta: 0.02240
#> 40 Number of features: 19 Max AUC: 0.625 AUC: 0.621 Z: 0.545 Rdelta: 0.03016
#> 41 Number of features: 20 Max AUC: 0.632 AUC: 0.632 Z: 0.518 Rdelta: 0.03714
#> 42 Number of features: 21 Max AUC: 0.634 AUC: 0.634 Z: 0.508 Rdelta: 0.04343
#> 43 Number of features: 22 Max AUC: 0.759 AUC: 0.759 Z: 0.832 Rdelta: 0.04909
#> 44 Number of features: 22 Max AUC: 0.759 AUC: 0.742 Z: 0.814 Rdelta: 0.03927
#> 45 Number of features: 23 Max AUC: 0.759 AUC: 0.752 Z: 0.800 Rdelta: 0.04534
#> 46 Number of features: 23 Max AUC: 0.759 AUC: 0.681 Z: 0.694 Rdelta: 0.03627
#> 47 Number of features: 24 Max AUC: 0.759 AUC: 0.749 Z: 0.831 Rdelta: 0.04265
#> 48 Number of features: 24 Max AUC: 0.759 AUC: 0.731 Z: 0.833 Rdelta: 0.03412
#> 49 Number of features: 24 Max AUC: 0.759 AUC: 0.743 Z: 0.768 Rdelta: 0.02729
#> 50 Number of features: 25 Max AUC: 0.759 AUC: 0.755 Z: 0.832 Rdelta: 0.03456
#> 51 Number of features: 26 Max AUC: 0.759 AUC: 0.758 Z: 0.800 Rdelta: 0.04111
#> 52 Number of features: 27 Max AUC: 0.759 AUC: 0.756 Z: 0.731 Rdelta: 0.04700
#> 53 Number of features: 27 Max AUC: 0.759 AUC: 0.737 Z: 0.731 Rdelta: 0.03760
#> 54 Number of features: 28 Max AUC: 0.759 AUC: 0.758 Z: 0.821 Rdelta: 0.04384
#> 55 Number of features: 29 Max AUC: 0.759 AUC: 0.755 Z: 0.838 Rdelta: 0.04945
#> 56 Number of features: 30 Max AUC: 0.759 AUC: 0.755 Z: 0.862 Rdelta: 0.05451
#> 57 Number of features: 30 Max AUC: 0.759 AUC: 0.741 Z: 0.802 Rdelta: 0.04361
#> 58 Number of features: 30 Max AUC: 0.759 AUC: 0.687 Z: 0.646 Rdelta: 0.03489
#> 59 Number of features: 31 Max AUC: 0.759 AUC: 0.754 Z: 0.842 Rdelta: 0.04140
#> 60 Number of features: 31 Max AUC: 0.759 AUC: 0.744 Z: 0.808 Rdelta: 0.03312
#> 61 Number of features: 32 Max AUC: 0.759 AUC: 0.752 Z: 0.829 Rdelta: 0.03981
#> 62 Number of features: 32 Max AUC: 0.759 AUC: 0.723 Z: 0.836 Rdelta: 0.03184
#> 63 Number of features: 33 Max AUC: 0.764 AUC: 0.764 Z: 0.833 Rdelta: 0.03866
#> 64 Number of features: 34 Max AUC: 0.764 AUC: 0.761 Z: 0.826 Rdelta: 0.04479
#> 65 Number of features: 35 Max AUC: 0.765 AUC: 0.765 Z: 0.961 Rdelta: 0.05031
#> 66 Number of features: 35 Max AUC: 0.765 AUC: 0.756 Z: 0.911 Rdelta: 0.04025
#> 67 Number of features: 36 Max AUC: 0.765 AUC: 0.763 Z: 0.919 Rdelta: 0.04623
#> 68 Number of features: 37 Max AUC: 0.766 AUC: 0.766 Z: 0.963 Rdelta: 0.05160
#> 69 Number of features: 37 Max AUC: 0.766 AUC: 0.734 Z: 0.840 Rdelta: 0.04128
#> 70 Number of features: 37 Max AUC: 0.766 AUC: 0.734 Z: 0.877 Rdelta: 0.03303
#> 71 Number of features: 37 Max AUC: 0.766 AUC: 0.745 Z: 0.886 Rdelta: 0.02642
#> 72 Number of features: 37 Max AUC: 0.766 AUC: 0.751 Z: 0.867 Rdelta: 0.02114
#> 73 Number of features: 37 Max AUC: 0.766 AUC: 0.729 Z: 0.825 Rdelta: 0.01691
#> 74 Number of features: 38 Max AUC: 0.773 AUC: 0.773 Z: 0.934 Rdelta: 0.02522
#> 75 Number of features: 38 Max AUC: 0.773 AUC: 0.755 Z: 0.909 Rdelta: 0.02017
#> 76 Number of features: 38 Max AUC: 0.773 AUC: 0.749 Z: 0.929 Rdelta: 0.01614
#> 77 Number of features: 39 Max AUC: 0.773 AUC: 0.767 Z: 0.979 Rdelta: 0.02453
#> 78 Number of features: 39 Max AUC: 0.773 AUC: 0.748 Z: 0.952 Rdelta: 0.01962
#> 79 Number of features: 40 Max AUC: 0.778 AUC: 0.778 Z: 0.927 Rdelta: 0.02766
#> 80 Number of features: 41 Max AUC: 0.778 AUC: 0.772 Z: 0.957 Rdelta: 0.03489
#> 81 Number of features: 41 Max AUC: 0.778 AUC: 0.755 Z: 0.884 Rdelta: 0.02791
#> 82 Number of features: 41 Max AUC: 0.778 AUC: 0.768 Z: 0.944 Rdelta: 0.02233
#> 83 Number of features: 41 Max AUC: 0.778 AUC: 0.764 Z: 1.000 Rdelta: 0.01787
#> 84 Number of features: 42 Max AUC: 0.778 AUC: 0.772 Z: 0.962 Rdelta: 0.02608
#> 85 Number of features: 42 Max AUC: 0.778 AUC: 0.721 Z: 0.980 Rdelta: 0.02086
#> 86 Number of features: 42 Max AUC: 0.778 AUC: 0.732 Z: 0.870 Rdelta: 0.01669
#> 87 Number of features: 43 Max AUC: 0.778 AUC: 0.776 Z: 1.012 Rdelta: 0.02502
#> 88 Number of features: 43 Max AUC: 0.778 AUC: 0.720 Z: 0.947 Rdelta: 0.02002
#> 89 Number of features: 43 Max AUC: 0.778 AUC: 0.721 Z: 0.791 Rdelta: 0.01601
#> 90 Number of features: 43 Max AUC: 0.778 AUC: 0.747 Z: 0.935 Rdelta: 0.01281
#> 91 Number of features: 43 Max AUC: 0.778 AUC: 0.766 Z: 0.931 Rdelta: 0.01025
#> 92 Number of features: 43 Max AUC: 0.778 AUC: 0.647 Z: 0.744 Rdelta: 0.00820
#> 93 Number of features: 43 Max AUC: 0.778 AUC: 0.714 Z: 1.038 Rdelta: 0.00656
#> 94 Number of features: 43 Max AUC: 0.778 AUC: 0.765 Z: 1.016 Rdelta: 0.00525
#> 95 Number of features: 44 Max AUC: 0.778 AUC: 0.775 Z: 1.041 Rdelta: 0.01472
#> 96 Number of features: 44 Max AUC: 0.778 AUC: 0.725 Z: 0.887 Rdelta: 0.01178
#> 97 Number of features: 44 Max AUC: 0.778 AUC: 0.698 Z: 1.008 Rdelta: 0.00942
#> 98 Number of features: 45 Max AUC: 0.778 AUC: 0.773 Z: 1.000 Rdelta: 0.01848
#> 99 Number of features: 45 Max AUC: 0.778 AUC: 0.701 Z: 1.050 Rdelta: 0.01478
#> 100 Number of features: 46 Max AUC: 0.778 AUC: 0.773 Z: 1.042 Rdelta: 0.02331
#> 101 Number of features: 46 Max AUC: 0.778 AUC: 0.755 Z: 0.978 Rdelta: 0.01864
#> 102 Number of features: 47 Max AUC: 0.778 AUC: 0.773 Z: 1.004 Rdelta: 0.02678
#> 103 Number of features: 48 Max AUC: 0.778 AUC: 0.775 Z: 1.039 Rdelta: 0.03410
#> 104 Number of features: 48 Max AUC: 0.778 AUC: 0.701 Z: 0.853 Rdelta: 0.02728
#> 105 Number of features: 48 Max AUC: 0.778 AUC: 0.759 Z: 0.961 Rdelta: 0.02183
#> 106 Number of features: 48 Max AUC: 0.778 AUC: 0.765 Z: 1.031 Rdelta: 0.01746
#> 107 Number of features: 48 Max AUC: 0.778 AUC: 0.714 Z: 0.918 Rdelta: 0.01397
#> 108 Number of features: 48 Max AUC: 0.778 AUC: 0.707 Z: 1.072 Rdelta: 0.01117
#> 109 Number of features: 48 Max AUC: 0.778 AUC: 0.702 Z: 1.027 Rdelta: 0.00894
#> 110 Number of features: 48 Max AUC: 0.778 AUC: 0.753 Z: 0.982 Rdelta: 0.00715
#> 111 Number of features: 48 Max AUC: 0.778 AUC: 0.766 Z: 1.047 Rdelta: 0.00572
#> 112 Number of features: 48 Max AUC: 0.778 AUC: 0.697 Z: 1.025 Rdelta: 0.00458
#> 113 Number of features: 49 Max AUC: 0.778 AUC: 0.768 Z: 0.994 Rdelta: 0.01412
#> 114 Number of features: 49 Max AUC: 0.778 AUC: 0.765 Z: 1.020 Rdelta: 0.01130
#> 115 Number of features: 49 Max AUC: 0.778 AUC: 0.752 Z: 1.002 Rdelta: 0.00904
#> 116 Number of features: 49 Max AUC: 0.778 AUC: 0.745 Z: 0.960 Rdelta: 0.00723
#> 117 Number of features: 49 Max AUC: 0.778 AUC: 0.750 Z: 0.958 Rdelta: 0.00578
#> 118 Number of features: 49 Max AUC: 0.778 AUC: 0.678 Z: 0.858 Rdelta: 0.00463
#> 119 Number of features: 49 Max AUC: 0.778 AUC: 0.761 Z: 0.974 Rdelta: 0.00370
#> 120 Number of features: 49 Max AUC: 0.778 AUC: 0.712 Z: 0.842 Rdelta: 0.00296
#> 121 Number of features: 50 Max AUC: 0.780 AUC: 0.780 Z: 1.025 Rdelta: 0.01266
#> 122 Number of features: 50 Max AUC: 0.780 AUC: 0.754 Z: 0.955 Rdelta: 0.01013
#> 123 Number of features: 50 Max AUC: 0.780 AUC: 0.760 Z: 1.031 Rdelta: 0.00811
#> 124 Number of features: 50 Max AUC: 0.780 AUC: 0.718 Z: 1.056 Rdelta: 0.00648
#> 125 Number of features: 50 Max AUC: 0.780 AUC: 0.759 Z: 1.008 Rdelta: 0.00519
#> 126 Number of features: 50 Max AUC: 0.780 AUC: 0.711 Z: 0.969 Rdelta: 0.00415
#> 127 Number of features: 50 Max AUC: 0.780 AUC: 0.732 Z: 0.955 Rdelta: 0.00332
#> 128 Number of features: 50 Max AUC: 0.780 AUC: 0.696 Z: 0.969 Rdelta: 0.00266
#> 129 Number of features: 50 Max AUC: 0.780 AUC: 0.764 Z: 0.986 Rdelta: 0.00212
#> 130 Number of features: 51 Max AUC: 0.780 AUC: 0.780 Z: 1.055 Rdelta: 0.01191
#> 131 Number of features: 51 Max AUC: 0.780 AUC: 0.732 Z: 0.955 Rdelta: 0.00953
#> 132 Number of features: 51 Max AUC: 0.780 AUC: 0.750 Z: 0.934 Rdelta: 0.00762
#> 133 Number of features: 51 Max AUC: 0.780 AUC: 0.728 Z: 0.918 Rdelta: 0.00610
#> 134 Number of features: 51 Max AUC: 0.780 AUC: 0.709 Z: 1.043 Rdelta: 0.00488
#> 135 Number of features: 51 Max AUC: 0.780 AUC: 0.758 Z: 1.025 Rdelta: 0.00390
#> 136 Number of features: 51 Max AUC: 0.780 AUC: 0.729 Z: 0.992 Rdelta: 0.00312
#> 137 Number of features: 51 Max AUC: 0.780 AUC: 0.760 Z: 0.989 Rdelta: 0.00250
#> 138 Number of features: 51 Max AUC: 0.780 AUC: 0.656 Z: 0.742 Rdelta: 0.00200
#> 139 Number of features: 51 Max AUC: 0.780 AUC: 0.757 Z: 1.014 Rdelta: 0.00160
#> 140 Number of features: 51 Max AUC: 0.780 AUC: 0.705 Z: 1.000 Rdelta: 0.00128
#> 141 Number of features: 51 Max AUC: 0.780 AUC: 0.708 Z: 1.057 Rdelta: 0.00102
#> 142 Number of features: 51 Max AUC: 0.780 AUC: 0.743 Z: 1.027 Rdelta: 0.00082
#> 143 Number of features: 51 Max AUC: 0.780 AUC: 0.770 Z: 1.009 Rdelta: 0.00065
#> 144 Number of features: 51 Max AUC: 0.780 AUC: 0.765 Z: 0.964 Rdelta: 0.00052
#> 145 Number of features: 51 Max AUC: 0.780 AUC: 0.769 Z: 1.117 Rdelta: 0.00042
#> 146 Number of features: 51 Max AUC: 0.780 AUC: 0.762 Z: 1.038 Rdelta: 0.00034
#> 147 Number of features: 51 Max AUC: 0.780 AUC: 0.705 Z: 0.797 Rdelta: 0.00027
#> 148 Number of features: 52 Max AUC: 0.780 AUC: 0.772 Z: 1.121 Rdelta: 0.01024
#> 149 Number of features: 52 Max AUC: 0.780 AUC: 0.686 Z: 0.936 Rdelta: 0.00819
#> 150 Number of features: 52 Max AUC: 0.780 AUC: 0.733 Z: 0.906 Rdelta: 0.00655
#> 151 Number of features: 52 Max AUC: 0.780 AUC: 0.712 Z: 0.937 Rdelta: 0.00524
#> 152 Number of features: 52 Max AUC: 0.780 AUC: 0.680 Z: 0.822 Rdelta: 0.00419
#> 153 Number of features: 52 Max AUC: 0.780 AUC: 0.769 Z: 1.042 Rdelta: 0.00336
#> 154 Number of features: 52 Max AUC: 0.780 AUC: 0.647 Z: 0.728 Rdelta: 0.00268
#> 155 Number of features: 52 Max AUC: 0.780 AUC: 0.694 Z: 0.853 Rdelta: 0.00215
#> 156 Number of features: 52 Max AUC: 0.780 AUC: 0.684 Z: 0.625 Rdelta: 0.00172
#> 157 Number of features: 52 Max AUC: 0.780 AUC: 0.759 Z: 1.003 Rdelta: 0.00137
#> 158 Number of features: 52 Max AUC: 0.780 AUC: 0.717 Z: 1.069 Rdelta: 0.00110
#> 159 Number of features: 52 Max AUC: 0.780 AUC: 0.712 Z: 1.005 Rdelta: 0.00088
#> 160 Number of features: 52 Max AUC: 0.780 AUC: 0.754 Z: 0.996 Rdelta: 0.00070
#> 161 Number of features: 52 Max AUC: 0.780 AUC: 0.750 Z: 0.990 Rdelta: 0.00056
#> 162 Number of features: 52 Max AUC: 0.780 AUC: 0.748 Z: 1.007 Rdelta: 0.00045
#> 163 Number of features: 52 Max AUC: 0.780 AUC: 0.760 Z: 1.084 Rdelta: 0.00036
#> 164 Number of features: 52 Max AUC: 0.780 AUC: 0.765 Z: 1.065 Rdelta: 0.00029
#> 165 Number of features: 52 Max AUC: 0.780 AUC: 0.702 Z: 1.050 Rdelta: 0.00023
#> 166 Number of features: 52 Max AUC: 0.780 AUC: 0.752 Z: 1.045 Rdelta: 0.00018
#> 167 Number of features: 52 Max AUC: 0.780 AUC: 0.737 Z: 0.992 Rdelta: 0.00015
#> 168 Number of features: 52 Max AUC: 0.780 AUC: 0.704 Z: 1.100 Rdelta: 0.00012
#> 169 Number of features: 52 Max AUC: 0.780 AUC: 0.755 Z: 0.911 Rdelta: 0.00009
#> user system elapsed
#> 83.74 0.00 83.80
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"pearson")+signatureDistance(signature$controlTemplate,validLabeled,"pearson")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS"))
#> 7 Number of features: 7 Max AUC: 0.729 AUC: 0.729 Z: 0.995 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.733 AUC: 0.733 Z: 1.091 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.733 AUC: 0.733 Z: 1.063 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.733 AUC: 0.733 Z: 1.186 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.736 AUC: 0.736 Z: 1.229 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.736 AUC: 0.724 Z: 1.417 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.736 AUC: 0.736 Z: 1.439 Rdelta: 0.10000
#> 14 Number of features: 13 Max AUC: 0.736 AUC: 0.719 Z: 1.102 Rdelta: 0.08000
#> 15 Number of features: 14 Max AUC: 0.736 AUC: 0.736 Z: 1.278 Rdelta: 0.08200
#> 16 Number of features: 15 Max AUC: 0.736 AUC: 0.731 Z: 1.445 Rdelta: 0.08380
#> 17 Number of features: 15 Max AUC: 0.736 AUC: 0.709 Z: 1.291 Rdelta: 0.06704
#> 18 Number of features: 16 Max AUC: 0.736 AUC: 0.732 Z: 0.361 Rdelta: 0.07034
#> 19 Number of features: 16 Max AUC: 0.736 AUC: 0.730 Z: 0.216 Rdelta: 0.05627
#> 20 Number of features: 16 Max AUC: 0.736 AUC: 0.724 Z: 0.468 Rdelta: 0.04502
#> 21 Number of features: 16 Max AUC: 0.736 AUC: 0.724 Z: 0.436 Rdelta: 0.03601
#> 22 Number of features: 17 Max AUC: 0.743 AUC: 0.743 Z: 0.395 Rdelta: 0.04241
#> 23 Number of features: 17 Max AUC: 0.743 AUC: 0.726 Z: 0.409 Rdelta: 0.03393
#> 24 Number of features: 18 Max AUC: 0.746 AUC: 0.746 Z: 0.460 Rdelta: 0.04054
#> 25 Number of features: 18 Max AUC: 0.746 AUC: 0.729 Z: 0.582 Rdelta: 0.03243
#> 26 Number of features: 19 Max AUC: 0.746 AUC: 0.742 Z: 0.394 Rdelta: 0.03919
#> 27 Number of features: 19 Max AUC: 0.746 AUC: 0.722 Z: 0.516 Rdelta: 0.03135
#> 28 Number of features: 19 Max AUC: 0.746 AUC: 0.730 Z: 0.442 Rdelta: 0.02508
#> 29 Number of features: 20 Max AUC: 0.746 AUC: 0.739 Z: 0.397 Rdelta: 0.03257
#> 30 Number of features: 20 Max AUC: 0.746 AUC: 0.703 Z: 0.475 Rdelta: 0.02606
#> 31 Number of features: 20 Max AUC: 0.746 AUC: 0.714 Z: 0.379 Rdelta: 0.02085
#> 32 Number of features: 20 Max AUC: 0.746 AUC: 0.731 Z: 0.398 Rdelta: 0.01668
#> 33 Number of features: 21 Max AUC: 0.746 AUC: 0.746 Z: 0.414 Rdelta: 0.02501
#> 34 Number of features: 21 Max AUC: 0.746 AUC: 0.736 Z: 0.379 Rdelta: 0.02001
#> 35 Number of features: 21 Max AUC: 0.746 AUC: 0.728 Z: 0.449 Rdelta: 0.01601
#> 36 Number of features: 21 Max AUC: 0.746 AUC: 0.720 Z: 0.473 Rdelta: 0.01280
#> 37 Number of features: 21 Max AUC: 0.746 AUC: 0.712 Z: 0.471 Rdelta: 0.01024
#> 38 Number of features: 22 Max AUC: 0.747 AUC: 0.747 Z: 0.471 Rdelta: 0.01922
#> 39 Number of features: 23 Max AUC: 0.747 AUC: 0.742 Z: 0.446 Rdelta: 0.02730
#> 40 Number of features: 23 Max AUC: 0.747 AUC: 0.704 Z: 0.459 Rdelta: 0.02184
#> 41 Number of features: 23 Max AUC: 0.747 AUC: 0.727 Z: 0.451 Rdelta: 0.01747
#> 42 Number of features: 23 Max AUC: 0.747 AUC: 0.708 Z: 0.499 Rdelta: 0.01398
#> 43 Number of features: 24 Max AUC: 0.747 AUC: 0.741 Z: 0.522 Rdelta: 0.02258
#> 44 Number of features: 25 Max AUC: 0.759 AUC: 0.759 Z: 0.453 Rdelta: 0.03032
#> 45 Number of features: 25 Max AUC: 0.759 AUC: 0.727 Z: 0.234 Rdelta: 0.02426
#> 46 Number of features: 25 Max AUC: 0.759 AUC: 0.738 Z: 0.125 Rdelta: 0.01941
#> 47 Number of features: 26 Max AUC: 0.759 AUC: 0.757 Z: 0.542 Rdelta: 0.02746
#> 48 Number of features: 27 Max AUC: 0.779 AUC: 0.779 Z: 0.119 Rdelta: 0.03472
#> 49 Number of features: 27 Max AUC: 0.779 AUC: 0.760 Z: 0.516 Rdelta: 0.02777
#> 50 Number of features: 27 Max AUC: 0.779 AUC: 0.743 Z: 0.154 Rdelta: 0.02222
#> 51 Number of features: 27 Max AUC: 0.779 AUC: 0.765 Z: 0.286 Rdelta: 0.01778
#> 52 Number of features: 27 Max AUC: 0.779 AUC: 0.754 Z: 0.397 Rdelta: 0.01422
#> 53 Number of features: 27 Max AUC: 0.779 AUC: 0.742 Z: 0.490 Rdelta: 0.01138
#> 54 Number of features: 28 Max AUC: 0.779 AUC: 0.770 Z: 0.163 Rdelta: 0.02024
#> 55 Number of features: 28 Max AUC: 0.779 AUC: 0.738 Z: 0.318 Rdelta: 0.01619
#> 56 Number of features: 29 Max AUC: 0.779 AUC: 0.774 Z: 0.578 Rdelta: 0.02457
#> 57 Number of features: 29 Max AUC: 0.779 AUC: 0.745 Z: 0.380 Rdelta: 0.01966
#> 58 Number of features: 29 Max AUC: 0.779 AUC: 0.708 Z: 0.296 Rdelta: 0.01573
#> 59 Number of features: 30 Max AUC: 0.781 AUC: 0.781 Z: 0.528 Rdelta: 0.02415
#> 60 Number of features: 30 Max AUC: 0.781 AUC: 0.762 Z: 0.513 Rdelta: 0.01932
#> 61 Number of features: 30 Max AUC: 0.781 AUC: 0.770 Z: 0.471 Rdelta: 0.01546
#> 62 Number of features: 30 Max AUC: 0.781 AUC: 0.676 Z: 0.229 Rdelta: 0.01237
#> 63 Number of features: 30 Max AUC: 0.781 AUC: 0.755 Z: 0.404 Rdelta: 0.00989
#> 64 Number of features: 30 Max AUC: 0.781 AUC: 0.746 Z: 0.336 Rdelta: 0.00791
#> 65 Number of features: 30 Max AUC: 0.781 AUC: 0.767 Z: 0.511 Rdelta: 0.00633
#> 66 Number of features: 30 Max AUC: 0.781 AUC: 0.762 Z: 0.564 Rdelta: 0.00507
#> 67 Number of features: 30 Max AUC: 0.781 AUC: 0.742 Z: 0.193 Rdelta: 0.00405
#> 68 Number of features: 30 Max AUC: 0.781 AUC: 0.729 Z: 0.276 Rdelta: 0.00324
#> 69 Number of features: 30 Max AUC: 0.781 AUC: 0.748 Z: 0.500 Rdelta: 0.00259
#> 70 Number of features: 30 Max AUC: 0.781 AUC: 0.747 Z: 0.288 Rdelta: 0.00207
#> 71 Number of features: 30 Max AUC: 0.781 AUC: 0.722 Z: 0.209 Rdelta: 0.00166
#> 72 Number of features: 30 Max AUC: 0.781 AUC: 0.744 Z: 0.463 Rdelta: 0.00133
#> 73 Number of features: 30 Max AUC: 0.781 AUC: 0.748 Z: 0.450 Rdelta: 0.00106
#> 74 Number of features: 30 Max AUC: 0.781 AUC: 0.748 Z: 0.351 Rdelta: 0.00085
#> 75 Number of features: 30 Max AUC: 0.781 AUC: 0.765 Z: 0.477 Rdelta: 0.00068
#> 76 Number of features: 30 Max AUC: 0.781 AUC: 0.723 Z: 0.544 Rdelta: 0.00054
#> 77 Number of features: 30 Max AUC: 0.781 AUC: 0.754 Z: 0.380 Rdelta: 0.00044
#> 78 Number of features: 30 Max AUC: 0.781 AUC: 0.768 Z: 0.439 Rdelta: 0.00035
#> 79 Number of features: 30 Max AUC: 0.781 AUC: 0.758 Z: 0.199 Rdelta: 0.00028
#> 80 Number of features: 31 Max AUC: 0.781 AUC: 0.775 Z: 0.305 Rdelta: 0.01025
#> 81 Number of features: 31 Max AUC: 0.781 AUC: 0.758 Z: 0.285 Rdelta: 0.00820
#> 82 Number of features: 32 Max AUC: 0.785 AUC: 0.785 Z: 0.400 Rdelta: 0.01738
#> 83 Number of features: 32 Max AUC: 0.785 AUC: 0.745 Z: 0.274 Rdelta: 0.01390
#> 84 Number of features: 32 Max AUC: 0.785 AUC: 0.775 Z: 0.411 Rdelta: 0.01112
#> 85 Number of features: 32 Max AUC: 0.785 AUC: 0.733 Z: 0.419 Rdelta: 0.00890
#> 86 Number of features: 32 Max AUC: 0.785 AUC: 0.748 Z: 0.524 Rdelta: 0.00712
#> 87 Number of features: 32 Max AUC: 0.785 AUC: 0.745 Z: 0.309 Rdelta: 0.00570
#> 88 Number of features: 32 Max AUC: 0.785 AUC: 0.731 Z: 0.349 Rdelta: 0.00456
#> 89 Number of features: 32 Max AUC: 0.785 AUC: 0.748 Z: 0.336 Rdelta: 0.00364
#> 90 Number of features: 32 Max AUC: 0.785 AUC: 0.758 Z: 0.372 Rdelta: 0.00292
#> 91 Number of features: 32 Max AUC: 0.785 AUC: 0.738 Z: 0.408 Rdelta: 0.00233
#> 92 Number of features: 32 Max AUC: 0.785 AUC: 0.747 Z: 0.434 Rdelta: 0.00187
#> 93 Number of features: 32 Max AUC: 0.785 AUC: 0.750 Z: 0.481 Rdelta: 0.00149
#> 94 Number of features: 32 Max AUC: 0.785 AUC: 0.773 Z: 0.387 Rdelta: 0.00119
#> 95 Number of features: 32 Max AUC: 0.785 AUC: 0.759 Z: 0.331 Rdelta: 0.00096
#> 96 Number of features: 32 Max AUC: 0.785 AUC: 0.690 Z: 0.186 Rdelta: 0.00076
#> 97 Number of features: 32 Max AUC: 0.785 AUC: 0.768 Z: 0.376 Rdelta: 0.00061
#> 98 Number of features: 32 Max AUC: 0.785 AUC: 0.754 Z: 0.380 Rdelta: 0.00049
#> 99 Number of features: 32 Max AUC: 0.785 AUC: 0.775 Z: 0.395 Rdelta: 0.00039
#> 100 Number of features: 32 Max AUC: 0.785 AUC: 0.751 Z: 0.479 Rdelta: 0.00031
#> 101 Number of features: 32 Max AUC: 0.785 AUC: 0.753 Z: 0.464 Rdelta: 0.00025
#> 102 Number of features: 32 Max AUC: 0.785 AUC: 0.756 Z: 0.465 Rdelta: 0.00020
#> 103 Number of features: 32 Max AUC: 0.785 AUC: 0.729 Z: 0.330 Rdelta: 0.00016
#> 104 Number of features: 32 Max AUC: 0.785 AUC: 0.748 Z: 0.380 Rdelta: 0.00013
#> 105 Number of features: 32 Max AUC: 0.785 AUC: 0.775 Z: 0.450 Rdelta: 0.00010
#> 106 Number of features: 32 Max AUC: 0.785 AUC: 0.762 Z: 0.231 Rdelta: 0.00008
#> user system elapsed
#> 38.93 0.00 38.96
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
testDistance_cotrol <- signatureDistance(signature$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Case"))
#> 7 Number of features: 7 Max AUC: 0.922 AUC: 0.917 Z: 1.057 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.922 AUC: 0.908 Z: 0.797 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.932 AUC: 0.932 Z: 1.210 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.932 AUC: 0.931 Z: 1.115 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.932 AUC: 0.929 Z: 1.291 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.942 AUC: 0.942 Z: 1.472 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.942 AUC: 0.941 Z: 1.641 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.949 AUC: 0.949 Z: 1.446 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.949 AUC: 0.940 Z: 1.304 Rdelta: 0.10000
#> 16 Number of features: 16 Max AUC: 0.949 AUC: 0.948 Z: 1.683 Rdelta: 0.10000
#> 17 Number of features: 17 Max AUC: 0.949 AUC: 0.946 Z: 1.030 Rdelta: 0.10000
#> 18 Number of features: 18 Max AUC: 0.961 AUC: 0.961 Z: 0.466 Rdelta: 0.10000
#> 19 Number of features: 19 Max AUC: 0.961 AUC: 0.954 Z: 0.269 Rdelta: 0.10000
#> 20 Number of features: 20 Max AUC: 0.961 AUC: 0.956 Z: 0.349 Rdelta: 0.10000
#> 21 Number of features: 20 Max AUC: 0.961 AUC: 0.943 Z: 0.371 Rdelta: 0.08000
#> 22 Number of features: 21 Max AUC: 0.963 AUC: 0.963 Z: 0.287 Rdelta: 0.08200
#> 23 Number of features: 22 Max AUC: 0.963 AUC: 0.952 Z: 0.330 Rdelta: 0.08380
#> 24 Number of features: 22 Max AUC: 0.963 AUC: 0.948 Z: 0.350 Rdelta: 0.06704
#> 25 Number of features: 23 Max AUC: 0.963 AUC: 0.953 Z: 0.341 Rdelta: 0.07034
#> 26 Number of features: 24 Max AUC: 0.963 AUC: 0.959 Z: 0.334 Rdelta: 0.07330
#> 27 Number of features: 25 Max AUC: 0.963 AUC: 0.949 Z: 0.398 Rdelta: 0.07597
#> 28 Number of features: 25 Max AUC: 0.963 AUC: 0.942 Z: 0.349 Rdelta: 0.06078
#> 29 Number of features: 25 Max AUC: 0.963 AUC: 0.941 Z: 0.367 Rdelta: 0.04862
#> 30 Number of features: 25 Max AUC: 0.963 AUC: 0.944 Z: 0.411 Rdelta: 0.03890
#> 31 Number of features: 25 Max AUC: 0.963 AUC: 0.940 Z: 0.395 Rdelta: 0.03112
#> 32 Number of features: 25 Max AUC: 0.963 AUC: 0.934 Z: 0.327 Rdelta: 0.02489
#> 33 Number of features: 25 Max AUC: 0.963 AUC: 0.934 Z: 0.401 Rdelta: 0.01992
#> 34 Number of features: 25 Max AUC: 0.963 AUC: 0.936 Z: 0.368 Rdelta: 0.01593
#> 35 Number of features: 25 Max AUC: 0.963 AUC: 0.936 Z: 0.340 Rdelta: 0.01275
#> 36 Number of features: 25 Max AUC: 0.963 AUC: 0.940 Z: 0.408 Rdelta: 0.01020
#> 37 Number of features: 25 Max AUC: 0.963 AUC: 0.931 Z: 0.313 Rdelta: 0.00816
#> 38 Number of features: 25 Max AUC: 0.963 AUC: 0.933 Z: 0.275 Rdelta: 0.00653
#> 39 Number of features: 25 Max AUC: 0.963 AUC: 0.935 Z: 0.346 Rdelta: 0.00522
#> 40 Number of features: 25 Max AUC: 0.963 AUC: 0.936 Z: 0.304 Rdelta: 0.00418
#> 41 Number of features: 25 Max AUC: 0.963 AUC: 0.942 Z: 0.474 Rdelta: 0.00334
#> 42 Number of features: 25 Max AUC: 0.963 AUC: 0.943 Z: 0.393 Rdelta: 0.00267
#> 43 Number of features: 25 Max AUC: 0.963 AUC: 0.932 Z: 0.352 Rdelta: 0.00214
#> 44 Number of features: 25 Max AUC: 0.963 AUC: 0.874 Z: 0.218 Rdelta: 0.00171
#> 45 Number of features: 26 Max AUC: 0.963 AUC: 0.948 Z: 0.387 Rdelta: 0.01154
#> 46 Number of features: 26 Max AUC: 0.963 AUC: 0.890 Z: 0.372 Rdelta: 0.00923
#> 47 Number of features: 26 Max AUC: 0.963 AUC: 0.942 Z: 0.349 Rdelta: 0.00739
#> 48 Number of features: 26 Max AUC: 0.963 AUC: 0.921 Z: 0.405 Rdelta: 0.00591
#> 49 Number of features: 26 Max AUC: 0.963 AUC: 0.941 Z: 0.388 Rdelta: 0.00473
#> 50 Number of features: 26 Max AUC: 0.963 AUC: 0.934 Z: 0.345 Rdelta: 0.00378
#> 51 Number of features: 26 Max AUC: 0.963 AUC: 0.933 Z: 0.451 Rdelta: 0.00303
#> 52 Number of features: 27 Max AUC: 0.963 AUC: 0.945 Z: 0.364 Rdelta: 0.01272
#> 53 Number of features: 27 Max AUC: 0.963 AUC: 0.942 Z: 0.443 Rdelta: 0.01018
#> 54 Number of features: 27 Max AUC: 0.963 AUC: 0.936 Z: 0.398 Rdelta: 0.00814
#> 55 Number of features: 27 Max AUC: 0.963 AUC: 0.937 Z: 0.383 Rdelta: 0.00651
#> 56 Number of features: 27 Max AUC: 0.963 AUC: 0.934 Z: 0.358 Rdelta: 0.00521
#> 57 Number of features: 28 Max AUC: 0.963 AUC: 0.948 Z: 0.394 Rdelta: 0.01469
#> 58 Number of features: 28 Max AUC: 0.963 AUC: 0.850 Z: 0.374 Rdelta: 0.01175
#> 59 Number of features: 28 Max AUC: 0.963 AUC: 0.939 Z: 0.336 Rdelta: 0.00940
#> 60 Number of features: 28 Max AUC: 0.963 AUC: 0.940 Z: 0.392 Rdelta: 0.00752
#> 61 Number of features: 28 Max AUC: 0.963 AUC: 0.927 Z: 0.347 Rdelta: 0.00602
#> 62 Number of features: 28 Max AUC: 0.963 AUC: 0.858 Z: 0.375 Rdelta: 0.00481
#> 63 Number of features: 28 Max AUC: 0.963 AUC: 0.933 Z: 0.363 Rdelta: 0.00385
#> 64 Number of features: 28 Max AUC: 0.963 AUC: 0.933 Z: 0.395 Rdelta: 0.00308
#> 65 Number of features: 29 Max AUC: 0.963 AUC: 0.943 Z: 0.395 Rdelta: 0.01277
#> 66 Number of features: 30 Max AUC: 0.963 AUC: 0.942 Z: 0.414 Rdelta: 0.02150
#> 67 Number of features: 31 Max AUC: 0.963 AUC: 0.944 Z: 0.355 Rdelta: 0.02935
#> 68 Number of features: 31 Max AUC: 0.963 AUC: 0.934 Z: 0.361 Rdelta: 0.02348
#> 69 Number of features: 31 Max AUC: 0.963 AUC: 0.925 Z: 0.332 Rdelta: 0.01878
#> 70 Number of features: 31 Max AUC: 0.963 AUC: 0.937 Z: 0.403 Rdelta: 0.01503
#> 71 Number of features: 31 Max AUC: 0.963 AUC: 0.934 Z: 0.437 Rdelta: 0.01202
#> 72 Number of features: 32 Max AUC: 0.963 AUC: 0.945 Z: 0.429 Rdelta: 0.02082
#> 73 Number of features: 32 Max AUC: 0.963 AUC: 0.930 Z: 0.420 Rdelta: 0.01665
#> 74 Number of features: 32 Max AUC: 0.963 AUC: 0.937 Z: 0.382 Rdelta: 0.01332
#> 75 Number of features: 32 Max AUC: 0.963 AUC: 0.932 Z: 0.475 Rdelta: 0.01066
#> 76 Number of features: 33 Max AUC: 0.963 AUC: 0.940 Z: 0.376 Rdelta: 0.01959
#> 77 Number of features: 33 Max AUC: 0.963 AUC: 0.926 Z: 0.302 Rdelta: 0.01567
#> 78 Number of features: 34 Max AUC: 0.963 AUC: 0.942 Z: 0.350 Rdelta: 0.02411
#> 79 Number of features: 34 Max AUC: 0.963 AUC: 0.930 Z: 0.336 Rdelta: 0.01929
#> 80 Number of features: 34 Max AUC: 0.963 AUC: 0.930 Z: 0.362 Rdelta: 0.01543
#> 81 Number of features: 34 Max AUC: 0.963 AUC: 0.936 Z: 0.406 Rdelta: 0.01234
#> 82 Number of features: 35 Max AUC: 0.963 AUC: 0.945 Z: 0.408 Rdelta: 0.02111
#> 83 Number of features: 35 Max AUC: 0.963 AUC: 0.934 Z: 0.321 Rdelta: 0.01689
#> 84 Number of features: 35 Max AUC: 0.963 AUC: 0.926 Z: 0.348 Rdelta: 0.01351
#> 85 Number of features: 36 Max AUC: 0.963 AUC: 0.937 Z: 0.386 Rdelta: 0.02216
#> 86 Number of features: 36 Max AUC: 0.963 AUC: 0.917 Z: 0.536 Rdelta: 0.01773
#> 87 Number of features: 37 Max AUC: 0.963 AUC: 0.943 Z: 0.353 Rdelta: 0.02595
#> 88 Number of features: 37 Max AUC: 0.963 AUC: 0.888 Z: 0.406 Rdelta: 0.02076
#> 89 Number of features: 37 Max AUC: 0.963 AUC: 0.932 Z: 0.400 Rdelta: 0.01661
#> 90 Number of features: 38 Max AUC: 0.963 AUC: 0.940 Z: 0.404 Rdelta: 0.02495
#> 91 Number of features: 38 Max AUC: 0.963 AUC: 0.930 Z: 0.396 Rdelta: 0.01996
#> 92 Number of features: 39 Max AUC: 0.963 AUC: 0.936 Z: 0.391 Rdelta: 0.02796
#> 93 Number of features: 39 Max AUC: 0.963 AUC: 0.928 Z: 0.342 Rdelta: 0.02237
#> 94 Number of features: 40 Max AUC: 0.963 AUC: 0.939 Z: 0.314 Rdelta: 0.03013
#> 95 Number of features: 40 Max AUC: 0.963 AUC: 0.931 Z: 0.360 Rdelta: 0.02411
#> 96 Number of features: 40 Max AUC: 0.963 AUC: 0.888 Z: 0.337 Rdelta: 0.01929
#> 97 Number of features: 41 Max AUC: 0.963 AUC: 0.939 Z: 0.326 Rdelta: 0.02736
#> 98 Number of features: 42 Max AUC: 0.963 AUC: 0.936 Z: 0.317 Rdelta: 0.03462
#> 99 Number of features: 43 Max AUC: 0.963 AUC: 0.937 Z: 0.420 Rdelta: 0.04116
#> 100 Number of features: 43 Max AUC: 0.963 AUC: 0.928 Z: 0.338 Rdelta: 0.03293
#> 101 Number of features: 43 Max AUC: 0.963 AUC: 0.927 Z: 0.303 Rdelta: 0.02634
#> 102 Number of features: 43 Max AUC: 0.963 AUC: 0.930 Z: 0.339 Rdelta: 0.02107
#> 103 Number of features: 43 Max AUC: 0.963 AUC: 0.923 Z: 0.361 Rdelta: 0.01686
#> 104 Number of features: 44 Max AUC: 0.963 AUC: 0.936 Z: 0.363 Rdelta: 0.02517
#> 105 Number of features: 44 Max AUC: 0.963 AUC: 0.922 Z: 0.412 Rdelta: 0.02014
#> 106 Number of features: 45 Max AUC: 0.963 AUC: 0.933 Z: 0.310 Rdelta: 0.02812
#> 107 Number of features: 46 Max AUC: 0.963 AUC: 0.936 Z: 0.304 Rdelta: 0.03531
#> 108 Number of features: 47 Max AUC: 0.963 AUC: 0.934 Z: 0.328 Rdelta: 0.04178
#> 109 Number of features: 48 Max AUC: 0.963 AUC: 0.937 Z: 0.339 Rdelta: 0.04760
#> 110 Number of features: 49 Max AUC: 0.963 AUC: 0.933 Z: 0.318 Rdelta: 0.05284
#> 111 Number of features: 50 Max AUC: 0.963 AUC: 0.941 Z: 0.322 Rdelta: 0.05756
#> 112 Number of features: 51 Max AUC: 0.963 AUC: 0.936 Z: 0.364 Rdelta: 0.06180
#> 113 Number of features: 51 Max AUC: 0.963 AUC: 0.924 Z: 0.213 Rdelta: 0.04944
#> 114 Number of features: 51 Max AUC: 0.963 AUC: 0.930 Z: 0.327 Rdelta: 0.03955
#> 115 Number of features: 52 Max AUC: 0.963 AUC: 0.936 Z: 0.318 Rdelta: 0.04560
#> 116 Number of features: 53 Max AUC: 0.963 AUC: 0.937 Z: 0.360 Rdelta: 0.05104
#> 117 Number of features: 54 Max AUC: 0.963 AUC: 0.933 Z: 0.342 Rdelta: 0.05593
#> 118 Number of features: 55 Max AUC: 0.963 AUC: 0.935 Z: 0.342 Rdelta: 0.06034
#> 119 Number of features: 56 Max AUC: 0.963 AUC: 0.936 Z: 0.348 Rdelta: 0.06431
#> 120 Number of features: 56 Max AUC: 0.963 AUC: 0.903 Z: 0.360 Rdelta: 0.05145
#> 121 Number of features: 56 Max AUC: 0.963 AUC: 0.923 Z: 0.441 Rdelta: 0.04116
#> 122 Number of features: 56 Max AUC: 0.963 AUC: 0.924 Z: 0.362 Rdelta: 0.03293
#> 123 Number of features: 57 Max AUC: 0.963 AUC: 0.931 Z: 0.313 Rdelta: 0.03963
#> 124 Number of features: 57 Max AUC: 0.963 AUC: 0.892 Z: 0.396 Rdelta: 0.03171
#> 125 Number of features: 57 Max AUC: 0.963 AUC: 0.919 Z: 0.287 Rdelta: 0.02536
#> 126 Number of features: 57 Max AUC: 0.963 AUC: 0.849 Z: 0.323 Rdelta: 0.02029
#> 127 Number of features: 57 Max AUC: 0.963 AUC: 0.923 Z: 0.355 Rdelta: 0.01623
#> 128 Number of features: 57 Max AUC: 0.963 AUC: 0.891 Z: 0.347 Rdelta: 0.01299
#> 129 Number of features: 58 Max AUC: 0.963 AUC: 0.930 Z: 0.354 Rdelta: 0.02169
#> 130 Number of features: 59 Max AUC: 0.963 AUC: 0.939 Z: 0.352 Rdelta: 0.02952
#> 131 Number of features: 59 Max AUC: 0.963 AUC: 0.869 Z: 0.410 Rdelta: 0.02362
#> 132 Number of features: 59 Max AUC: 0.963 AUC: 0.920 Z: 0.377 Rdelta: 0.01889
#> 133 Number of features: 59 Max AUC: 0.963 AUC: 0.921 Z: 0.407 Rdelta: 0.01511
#> 134 Number of features: 59 Max AUC: 0.963 AUC: 0.921 Z: 0.364 Rdelta: 0.01209
#> 135 Number of features: 60 Max AUC: 0.963 AUC: 0.929 Z: 0.349 Rdelta: 0.02088
#> 136 Number of features: 60 Max AUC: 0.963 AUC: 0.924 Z: 0.341 Rdelta: 0.01671
#> 137 Number of features: 60 Max AUC: 0.963 AUC: 0.923 Z: 0.307 Rdelta: 0.01336
#> 138 Number of features: 60 Max AUC: 0.963 AUC: 0.927 Z: 0.307 Rdelta: 0.01069
#> 139 Number of features: 60 Max AUC: 0.963 AUC: 0.924 Z: 0.312 Rdelta: 0.00855
#> 140 Number of features: 60 Max AUC: 0.963 AUC: 0.885 Z: 0.310 Rdelta: 0.00684
#> 141 Number of features: 61 Max AUC: 0.963 AUC: 0.932 Z: 0.327 Rdelta: 0.01616
#> 142 Number of features: 61 Max AUC: 0.963 AUC: 0.924 Z: 0.342 Rdelta: 0.01293
#> 143 Number of features: 62 Max AUC: 0.963 AUC: 0.933 Z: 0.429 Rdelta: 0.02163
#> 144 Number of features: 62 Max AUC: 0.963 AUC: 0.922 Z: 0.344 Rdelta: 0.01731
#> 145 Number of features: 62 Max AUC: 0.963 AUC: 0.924 Z: 0.418 Rdelta: 0.01385
#> 146 Number of features: 63 Max AUC: 0.963 AUC: 0.930 Z: 0.501 Rdelta: 0.02246
#> 147 Number of features: 63 Max AUC: 0.963 AUC: 0.857 Z: 0.378 Rdelta: 0.01797
#> 148 Number of features: 63 Max AUC: 0.963 AUC: 0.923 Z: 0.391 Rdelta: 0.01438
#> 149 Number of features: 63 Max AUC: 0.963 AUC: 0.915 Z: 0.417 Rdelta: 0.01150
#> 150 Number of features: 63 Max AUC: 0.963 AUC: 0.925 Z: 0.408 Rdelta: 0.00920
#> 151 Number of features: 63 Max AUC: 0.963 AUC: 0.922 Z: 0.360 Rdelta: 0.00736
#> 152 Number of features: 64 Max AUC: 0.963 AUC: 0.928 Z: 0.416 Rdelta: 0.01662
#> 153 Number of features: 64 Max AUC: 0.963 AUC: 0.916 Z: 0.438 Rdelta: 0.01330
#> 154 Number of features: 64 Max AUC: 0.963 AUC: 0.916 Z: 0.382 Rdelta: 0.01064
#> 155 Number of features: 64 Max AUC: 0.963 AUC: 0.862 Z: 0.374 Rdelta: 0.00851
#> 156 Number of features: 64 Max AUC: 0.963 AUC: 0.920 Z: 0.422 Rdelta: 0.00681
#> 157 Number of features: 65 Max AUC: 0.963 AUC: 0.929 Z: 0.370 Rdelta: 0.01613
#> 158 Number of features: 65 Max AUC: 0.963 AUC: 0.926 Z: 0.362 Rdelta: 0.01290
#> 159 Number of features: 65 Max AUC: 0.963 AUC: 0.925 Z: 0.396 Rdelta: 0.01032
#> 160 Number of features: 66 Max AUC: 0.963 AUC: 0.929 Z: 0.442 Rdelta: 0.01929
#> 161 Number of features: 66 Max AUC: 0.963 AUC: 0.921 Z: 0.427 Rdelta: 0.01543
#> 162 Number of features: 66 Max AUC: 0.963 AUC: 0.925 Z: 0.393 Rdelta: 0.01235
#> 163 Number of features: 67 Max AUC: 0.963 AUC: 0.934 Z: 0.448 Rdelta: 0.02111
#> 164 Number of features: 67 Max AUC: 0.963 AUC: 0.924 Z: 0.420 Rdelta: 0.01689
#> 165 Number of features: 68 Max AUC: 0.963 AUC: 0.929 Z: 0.418 Rdelta: 0.02520
#> 166 Number of features: 68 Max AUC: 0.963 AUC: 0.924 Z: 0.432 Rdelta: 0.02016
#> 167 Number of features: 69 Max AUC: 0.963 AUC: 0.933 Z: 0.447 Rdelta: 0.02814
#> 168 Number of features: 69 Max AUC: 0.963 AUC: 0.924 Z: 0.341 Rdelta: 0.02252
#> 169 Number of features: 69 Max AUC: 0.963 AUC: 0.921 Z: 0.370 Rdelta: 0.01801
#> 170 Number of features: 69 Max AUC: 0.963 AUC: 0.922 Z: 0.338 Rdelta: 0.01441
#> 171 Number of features: 70 Max AUC: 0.963 AUC: 0.928 Z: 0.479 Rdelta: 0.02297
#> 172 Number of features: 71 Max AUC: 0.963 AUC: 0.929 Z: 0.396 Rdelta: 0.03067
#> 173 Number of features: 72 Max AUC: 0.963 AUC: 0.926 Z: 0.448 Rdelta: 0.03760
#> 174 Number of features: 72 Max AUC: 0.963 AUC: 0.925 Z: 0.408 Rdelta: 0.03008
#> 175 Number of features: 72 Max AUC: 0.963 AUC: 0.923 Z: 0.451 Rdelta: 0.02407
#> 176 Number of features: 72 Max AUC: 0.963 AUC: 0.921 Z: 0.485 Rdelta: 0.01925
#> 177 Number of features: 72 Max AUC: 0.963 AUC: 0.922 Z: 0.441 Rdelta: 0.01540
#> 178 Number of features: 72 Max AUC: 0.963 AUC: 0.920 Z: 0.379 Rdelta: 0.01232
#> 179 Number of features: 72 Max AUC: 0.963 AUC: 0.919 Z: 0.368 Rdelta: 0.00986
#> 180 Number of features: 72 Max AUC: 0.963 AUC: 0.923 Z: 0.426 Rdelta: 0.00789
#> 181 Number of features: 73 Max AUC: 0.963 AUC: 0.929 Z: 0.368 Rdelta: 0.01710
#> 182 Number of features: 74 Max AUC: 0.963 AUC: 0.931 Z: 0.459 Rdelta: 0.02539
#> 183 Number of features: 75 Max AUC: 0.963 AUC: 0.930 Z: 0.414 Rdelta: 0.03285
#> 184 Number of features: 76 Max AUC: 0.963 AUC: 0.927 Z: 0.363 Rdelta: 0.03956
#> 185 Number of features: 77 Max AUC: 0.963 AUC: 0.929 Z: 0.432 Rdelta: 0.04561
#> 186 Number of features: 77 Max AUC: 0.963 AUC: 0.894 Z: 0.348 Rdelta: 0.03649
#> 187 Number of features: 77 Max AUC: 0.963 AUC: 0.910 Z: 0.325 Rdelta: 0.02919
#> 188 Number of features: 77 Max AUC: 0.963 AUC: 0.917 Z: 0.336 Rdelta: 0.02335
#> 189 Number of features: 77 Max AUC: 0.963 AUC: 0.917 Z: 0.332 Rdelta: 0.01868
#> 190 Number of features: 77 Max AUC: 0.963 AUC: 0.924 Z: 0.443 Rdelta: 0.01494
#> 191 Number of features: 78 Max AUC: 0.963 AUC: 0.928 Z: 0.473 Rdelta: 0.02345
#> 192 Number of features: 78 Max AUC: 0.963 AUC: 0.922 Z: 0.365 Rdelta: 0.01876
#> 193 Number of features: 78 Max AUC: 0.963 AUC: 0.915 Z: 0.356 Rdelta: 0.01501
#> 194 Number of features: 78 Max AUC: 0.963 AUC: 0.860 Z: 0.444 Rdelta: 0.01201
#> 195 Number of features: 78 Max AUC: 0.963 AUC: 0.916 Z: 0.401 Rdelta: 0.00961
#> 196 Number of features: 78 Max AUC: 0.963 AUC: 0.920 Z: 0.362 Rdelta: 0.00768
#> 197 Number of features: 78 Max AUC: 0.963 AUC: 0.912 Z: 0.408 Rdelta: 0.00615
#> 198 Number of features: 79 Max AUC: 0.963 AUC: 0.927 Z: 0.414 Rdelta: 0.01553
#> 199 Number of features: 79 Max AUC: 0.963 AUC: 0.924 Z: 0.453 Rdelta: 0.01243
#> 200 Number of features: 79 Max AUC: 0.963 AUC: 0.905 Z: 0.259 Rdelta: 0.00994
#> 201 Number of features: 79 Max AUC: 0.963 AUC: 0.909 Z: 0.356 Rdelta: 0.00795
#> 202 Number of features: 79 Max AUC: 0.963 AUC: 0.919 Z: 0.427 Rdelta: 0.00636
#> 203 Number of features: 79 Max AUC: 0.963 AUC: 0.904 Z: 0.292 Rdelta: 0.00509
#> 204 Number of features: 79 Max AUC: 0.963 AUC: 0.924 Z: 0.386 Rdelta: 0.00407
#> 205 Number of features: 79 Max AUC: 0.963 AUC: 0.915 Z: 0.459 Rdelta: 0.00326
#> 206 Number of features: 79 Max AUC: 0.963 AUC: 0.919 Z: 0.400 Rdelta: 0.00261
#> 207 Number of features: 79 Max AUC: 0.963 AUC: 0.902 Z: 0.396 Rdelta: 0.00208
#> 208 Number of features: 80 Max AUC: 0.963 AUC: 0.925 Z: 0.459 Rdelta: 0.01188
#> 209 Number of features: 80 Max AUC: 0.963 AUC: 0.916 Z: 0.373 Rdelta: 0.00950
#> 210 Number of features: 80 Max AUC: 0.963 AUC: 0.911 Z: 0.391 Rdelta: 0.00760
#> 211 Number of features: 80 Max AUC: 0.963 AUC: 0.910 Z: 0.386 Rdelta: 0.00608
#> 212 Number of features: 80 Max AUC: 0.963 AUC: 0.917 Z: 0.336 Rdelta: 0.00486
#> 213 Number of features: 80 Max AUC: 0.963 AUC: 0.909 Z: 0.354 Rdelta: 0.00389
#> 214 Number of features: 80 Max AUC: 0.963 AUC: 0.924 Z: 0.370 Rdelta: 0.00311
#> 215 Number of features: 80 Max AUC: 0.963 AUC: 0.915 Z: 0.373 Rdelta: 0.00249
#> 216 Number of features: 80 Max AUC: 0.963 AUC: 0.916 Z: 0.367 Rdelta: 0.00199
#> 217 Number of features: 80 Max AUC: 0.963 AUC: 0.919 Z: 0.396 Rdelta: 0.00159
#> 218 Number of features: 80 Max AUC: 0.963 AUC: 0.912 Z: 0.436 Rdelta: 0.00128
#> 219 Number of features: 80 Max AUC: 0.963 AUC: 0.916 Z: 0.409 Rdelta: 0.00102
#> 220 Number of features: 80 Max AUC: 0.963 AUC: 0.891 Z: 0.037 Rdelta: 0.00082
#> 221 Number of features: 80 Max AUC: 0.963 AUC: 0.914 Z: 0.395 Rdelta: 0.00065
#> 222 Number of features: 80 Max AUC: 0.963 AUC: 0.921 Z: 0.406 Rdelta: 0.00052
#> 223 Number of features: 80 Max AUC: 0.963 AUC: 0.918 Z: 0.391 Rdelta: 0.00042
#> 224 Number of features: 80 Max AUC: 0.963 AUC: 0.914 Z: 0.427 Rdelta: 0.00033
#> 225 Number of features: 80 Max AUC: 0.963 AUC: 0.918 Z: 0.341 Rdelta: 0.00027
#> 226 Number of features: 80 Max AUC: 0.963 AUC: 0.920 Z: 0.436 Rdelta: 0.00021
#> 227 Number of features: 80 Max AUC: 0.963 AUC: 0.915 Z: 0.387 Rdelta: 0.00017
#> 228 Number of features: 80 Max AUC: 0.963 AUC: 0.912 Z: 0.393 Rdelta: 0.00014
#> 229 Number of features: 80 Max AUC: 0.963 AUC: 0.918 Z: 0.358 Rdelta: 0.00011
#> 230 Number of features: 80 Max AUC: 0.963 AUC: 0.923 Z: 0.423 Rdelta: 0.00009
#> user system elapsed
#> 144.41 0.00 144.54
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
system.time(signatureControl <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Control"))
#> 7 Number of features: 7 Max AUC: 0.301 AUC: 0.296 Z: -0.619 Rdelta: 0.10000
#> 8 Number of features: 7 Max AUC: 0.301 AUC: 0.282 Z: -0.705 Rdelta: 0.08000
#> 9 Number of features: 7 Max AUC: 0.301 AUC: 0.285 Z: -0.699 Rdelta: 0.06400
#> 10 Number of features: 7 Max AUC: 0.301 AUC: 0.280 Z: -0.675 Rdelta: 0.05120
#> 11 Number of features: 7 Max AUC: 0.301 AUC: 0.281 Z: -0.755 Rdelta: 0.04096
#> 12 Number of features: 7 Max AUC: 0.301 AUC: 0.253 Z: -0.818 Rdelta: 0.03277
#> 13 Number of features: 8 Max AUC: 0.301 AUC: 0.290 Z: -0.488 Rdelta: 0.03949
#> 14 Number of features: 9 Max AUC: 0.301 AUC: 0.294 Z: -0.665 Rdelta: 0.04554
#> 15 Number of features: 9 Max AUC: 0.301 AUC: 0.267 Z: -0.716 Rdelta: 0.03643
#> 16 Number of features: 9 Max AUC: 0.301 AUC: 0.281 Z: -0.747 Rdelta: 0.02915
#> 17 Number of features: 9 Max AUC: 0.301 AUC: 0.282 Z: -0.739 Rdelta: 0.02332
#> 18 Number of features: 9 Max AUC: 0.301 AUC: 0.271 Z: -0.653 Rdelta: 0.01865
#> 19 Number of features: 9 Max AUC: 0.301 AUC: 0.276 Z: -0.700 Rdelta: 0.01492
#> 20 Number of features: 9 Max AUC: 0.301 AUC: 0.240 Z: -0.752 Rdelta: 0.01194
#> 21 Number of features: 9 Max AUC: 0.301 AUC: 0.281 Z: -0.691 Rdelta: 0.00955
#> 22 Number of features: 9 Max AUC: 0.301 AUC: 0.271 Z: -0.741 Rdelta: 0.00764
#> 23 Number of features: 9 Max AUC: 0.301 AUC: 0.268 Z: -0.738 Rdelta: 0.00611
#> 24 Number of features: 9 Max AUC: 0.301 AUC: 0.250 Z: -0.744 Rdelta: 0.00489
#> 25 Number of features: 9 Max AUC: 0.301 AUC: 0.259 Z: -0.773 Rdelta: 0.00391
#> 26 Number of features: 9 Max AUC: 0.301 AUC: 0.261 Z: -0.705 Rdelta: 0.00313
#> 27 Number of features: 9 Max AUC: 0.301 AUC: 0.265 Z: -0.805 Rdelta: 0.00250
#> 28 Number of features: 9 Max AUC: 0.301 AUC: 0.269 Z: -0.670 Rdelta: 0.00200
#> 29 Number of features: 10 Max AUC: 0.301 AUC: 0.292 Z: -0.724 Rdelta: 0.01180
#> 30 Number of features: 10 Max AUC: 0.301 AUC: 0.244 Z: -0.689 Rdelta: 0.00944
#> 31 Number of features: 10 Max AUC: 0.301 AUC: 0.264 Z: -0.703 Rdelta: 0.00755
#> 32 Number of features: 11 Max AUC: 0.301 AUC: 0.285 Z: -0.634 Rdelta: 0.01680
#> 33 Number of features: 12 Max AUC: 0.301 AUC: 0.289 Z: -0.709 Rdelta: 0.02512
#> 34 Number of features: 13 Max AUC: 0.301 AUC: 0.290 Z: -0.658 Rdelta: 0.03261
#> 35 Number of features: 13 Max AUC: 0.301 AUC: 0.263 Z: -0.736 Rdelta: 0.02609
#> 36 Number of features: 13 Max AUC: 0.301 AUC: 0.274 Z: -0.635 Rdelta: 0.02087
#> 37 Number of features: 13 Max AUC: 0.301 AUC: 0.274 Z: -0.642 Rdelta: 0.01669
#> 38 Number of features: 13 Max AUC: 0.301 AUC: 0.274 Z: -0.717 Rdelta: 0.01336
#> 39 Number of features: 13 Max AUC: 0.301 AUC: 0.266 Z: -0.727 Rdelta: 0.01068
#> 40 Number of features: 13 Max AUC: 0.301 AUC: 0.241 Z: -0.742 Rdelta: 0.00855
#> 41 Number of features: 13 Max AUC: 0.301 AUC: 0.268 Z: -0.653 Rdelta: 0.00684
#> 42 Number of features: 13 Max AUC: 0.301 AUC: 0.243 Z: -0.793 Rdelta: 0.00547
#> 43 Number of features: 14 Max AUC: 0.413 AUC: 0.413 Z: -0.376 Rdelta: 0.01492
#> 44 Number of features: 15 Max AUC: 0.489 AUC: 0.489 Z: -0.045 Rdelta: 0.02343
#> 45 Number of features: 16 Max AUC: 0.493 AUC: 0.493 Z: 0.002 Rdelta: 0.03109
#> 46 Number of features: 16 Max AUC: 0.493 AUC: 0.485 Z: -0.048 Rdelta: 0.02487
#> 47 Number of features: 16 Max AUC: 0.493 AUC: 0.476 Z: -0.030 Rdelta: 0.01990
#> 48 Number of features: 17 Max AUC: 0.535 AUC: 0.535 Z: 0.081 Rdelta: 0.02791
#> 49 Number of features: 17 Max AUC: 0.535 AUC: 0.520 Z: 0.050 Rdelta: 0.02233
#> 50 Number of features: 17 Max AUC: 0.535 AUC: 0.519 Z: 0.006 Rdelta: 0.01786
#> 51 Number of features: 18 Max AUC: 0.536 AUC: 0.536 Z: 0.047 Rdelta: 0.02607
#> 52 Number of features: 19 Max AUC: 0.575 AUC: 0.575 Z: 0.145 Rdelta: 0.03347
#> 53 Number of features: 19 Max AUC: 0.575 AUC: 0.528 Z: 0.008 Rdelta: 0.02677
#> 54 Number of features: 19 Max AUC: 0.575 AUC: 0.537 Z: -0.006 Rdelta: 0.02142
#> 55 Number of features: 19 Max AUC: 0.575 AUC: 0.497 Z: -0.045 Rdelta: 0.01714
#> 56 Number of features: 19 Max AUC: 0.575 AUC: 0.555 Z: 0.138 Rdelta: 0.01371
#> 57 Number of features: 19 Max AUC: 0.575 AUC: 0.557 Z: 0.178 Rdelta: 0.01097
#> 58 Number of features: 19 Max AUC: 0.575 AUC: 0.491 Z: 0.031 Rdelta: 0.00877
#> 59 Number of features: 19 Max AUC: 0.575 AUC: 0.536 Z: 0.026 Rdelta: 0.00702
#> 60 Number of features: 19 Max AUC: 0.575 AUC: 0.514 Z: -0.003 Rdelta: 0.00561
#> 61 Number of features: 19 Max AUC: 0.575 AUC: 0.543 Z: 0.015 Rdelta: 0.00449
#> 62 Number of features: 19 Max AUC: 0.575 AUC: 0.547 Z: 0.017 Rdelta: 0.00359
#> 63 Number of features: 19 Max AUC: 0.575 AUC: 0.551 Z: 0.007 Rdelta: 0.00287
#> 64 Number of features: 19 Max AUC: 0.575 AUC: 0.565 Z: 0.133 Rdelta: 0.00230
#> 65 Number of features: 19 Max AUC: 0.575 AUC: 0.541 Z: 0.076 Rdelta: 0.00184
#> 66 Number of features: 19 Max AUC: 0.575 AUC: 0.557 Z: 0.097 Rdelta: 0.00147
#> 67 Number of features: 19 Max AUC: 0.575 AUC: 0.537 Z: 0.033 Rdelta: 0.00118
#> 68 Number of features: 19 Max AUC: 0.575 AUC: 0.532 Z: 0.028 Rdelta: 0.00094
#> 69 Number of features: 19 Max AUC: 0.575 AUC: 0.562 Z: 0.115 Rdelta: 0.00075
#> 70 Number of features: 19 Max AUC: 0.575 AUC: 0.536 Z: 0.061 Rdelta: 0.00060
#> 71 Number of features: 19 Max AUC: 0.575 AUC: 0.541 Z: 0.074 Rdelta: 0.00048
#> 72 Number of features: 19 Max AUC: 0.575 AUC: 0.550 Z: 0.150 Rdelta: 0.00039
#> 73 Number of features: 19 Max AUC: 0.575 AUC: 0.565 Z: 0.112 Rdelta: 0.00031
#> 74 Number of features: 19 Max AUC: 0.575 AUC: 0.553 Z: 0.026 Rdelta: 0.00025
#> 75 Number of features: 19 Max AUC: 0.575 AUC: 0.537 Z: 0.034 Rdelta: 0.00020
#> 76 Number of features: 19 Max AUC: 0.575 AUC: 0.541 Z: 0.036 Rdelta: 0.00016
#> 77 Number of features: 20 Max AUC: 0.575 AUC: 0.570 Z: 0.103 Rdelta: 0.01014
#> 78 Number of features: 20 Max AUC: 0.575 AUC: 0.544 Z: 0.074 Rdelta: 0.00811
#> 79 Number of features: 20 Max AUC: 0.575 AUC: 0.530 Z: 0.002 Rdelta: 0.00649
#> 80 Number of features: 20 Max AUC: 0.575 AUC: 0.533 Z: 0.006 Rdelta: 0.00519
#> 81 Number of features: 20 Max AUC: 0.575 AUC: 0.504 Z: 0.030 Rdelta: 0.00415
#> 82 Number of features: 20 Max AUC: 0.575 AUC: 0.549 Z: 0.086 Rdelta: 0.00332
#> 83 Number of features: 20 Max AUC: 0.575 AUC: 0.541 Z: 0.087 Rdelta: 0.00266
#> 84 Number of features: 20 Max AUC: 0.575 AUC: 0.526 Z: 0.036 Rdelta: 0.00213
#> 85 Number of features: 21 Max AUC: 0.575 AUC: 0.570 Z: 0.008 Rdelta: 0.01191
#> 86 Number of features: 21 Max AUC: 0.575 AUC: 0.558 Z: 0.007 Rdelta: 0.00953
#> 87 Number of features: 21 Max AUC: 0.575 AUC: 0.555 Z: 0.068 Rdelta: 0.00763
#> 88 Number of features: 22 Max AUC: 0.575 AUC: 0.566 Z: 0.170 Rdelta: 0.01686
#> 89 Number of features: 23 Max AUC: 0.575 AUC: 0.570 Z: 0.032 Rdelta: 0.02518
#> 90 Number of features: 23 Max AUC: 0.575 AUC: 0.517 Z: 0.008 Rdelta: 0.02014
#> 91 Number of features: 23 Max AUC: 0.575 AUC: 0.535 Z: 0.009 Rdelta: 0.01611
#> 92 Number of features: 23 Max AUC: 0.575 AUC: 0.550 Z: 0.104 Rdelta: 0.01289
#> 93 Number of features: 23 Max AUC: 0.575 AUC: 0.536 Z: 0.051 Rdelta: 0.01031
#> 94 Number of features: 23 Max AUC: 0.575 AUC: 0.546 Z: 0.119 Rdelta: 0.00825
#> 95 Number of features: 23 Max AUC: 0.575 AUC: 0.527 Z: -0.003 Rdelta: 0.00660
#> 96 Number of features: 24 Max AUC: 0.601 AUC: 0.601 Z: 0.085 Rdelta: 0.01594
#> 97 Number of features: 25 Max AUC: 0.601 AUC: 0.596 Z: 0.167 Rdelta: 0.02435
#> 98 Number of features: 25 Max AUC: 0.601 AUC: 0.553 Z: 0.024 Rdelta: 0.01948
#> 99 Number of features: 25 Max AUC: 0.601 AUC: 0.592 Z: 0.009 Rdelta: 0.01558
#> 100 Number of features: 25 Max AUC: 0.601 AUC: 0.555 Z: 0.111 Rdelta: 0.01247
#> 101 Number of features: 25 Max AUC: 0.601 AUC: 0.582 Z: 0.145 Rdelta: 0.00997
#> 102 Number of features: 25 Max AUC: 0.601 AUC: 0.581 Z: 0.040 Rdelta: 0.00798
#> 103 Number of features: 25 Max AUC: 0.601 AUC: 0.588 Z: 0.265 Rdelta: 0.00638
#> 104 Number of features: 26 Max AUC: 0.608 AUC: 0.608 Z: 0.196 Rdelta: 0.01574
#> 105 Number of features: 26 Max AUC: 0.608 AUC: 0.602 Z: 0.148 Rdelta: 0.01260
#> 106 Number of features: 26 Max AUC: 0.608 AUC: 0.556 Z: 0.040 Rdelta: 0.01008
#> 107 Number of features: 26 Max AUC: 0.608 AUC: 0.572 Z: 0.085 Rdelta: 0.00806
#> 108 Number of features: 26 Max AUC: 0.608 AUC: 0.572 Z: 0.130 Rdelta: 0.00645
#> 109 Number of features: 26 Max AUC: 0.608 AUC: 0.566 Z: 0.083 Rdelta: 0.00516
#> 110 Number of features: 26 Max AUC: 0.608 AUC: 0.574 Z: 0.095 Rdelta: 0.00413
#> 111 Number of features: 26 Max AUC: 0.608 AUC: 0.590 Z: 0.119 Rdelta: 0.00330
#> 112 Number of features: 26 Max AUC: 0.608 AUC: 0.580 Z: 0.094 Rdelta: 0.00264
#> 113 Number of features: 26 Max AUC: 0.608 AUC: 0.582 Z: 0.114 Rdelta: 0.00211
#> 114 Number of features: 26 Max AUC: 0.608 AUC: 0.578 Z: 0.032 Rdelta: 0.00169
#> 115 Number of features: 26 Max AUC: 0.608 AUC: 0.600 Z: 0.069 Rdelta: 0.00135
#> 116 Number of features: 26 Max AUC: 0.608 AUC: 0.543 Z: 0.023 Rdelta: 0.00108
#> 117 Number of features: 26 Max AUC: 0.608 AUC: 0.576 Z: 0.226 Rdelta: 0.00087
#> 118 Number of features: 26 Max AUC: 0.608 AUC: 0.564 Z: 0.060 Rdelta: 0.00069
#> 119 Number of features: 26 Max AUC: 0.608 AUC: 0.572 Z: 0.099 Rdelta: 0.00055
#> 120 Number of features: 27 Max AUC: 0.641 AUC: 0.641 Z: 0.221 Rdelta: 0.01050
#> 121 Number of features: 27 Max AUC: 0.641 AUC: 0.615 Z: 0.048 Rdelta: 0.00840
#> 122 Number of features: 28 Max AUC: 0.647 AUC: 0.647 Z: 0.101 Rdelta: 0.01756
#> 123 Number of features: 29 Max AUC: 0.647 AUC: 0.642 Z: 0.197 Rdelta: 0.02580
#> 124 Number of features: 29 Max AUC: 0.647 AUC: 0.622 Z: 0.045 Rdelta: 0.02064
#> 125 Number of features: 29 Max AUC: 0.647 AUC: 0.625 Z: 0.180 Rdelta: 0.01651
#> 126 Number of features: 29 Max AUC: 0.647 AUC: 0.605 Z: 0.003 Rdelta: 0.01321
#> 127 Number of features: 30 Max AUC: 0.647 AUC: 0.641 Z: 0.075 Rdelta: 0.02189
#> 128 Number of features: 30 Max AUC: 0.647 AUC: 0.635 Z: 0.010 Rdelta: 0.01751
#> 129 Number of features: 31 Max AUC: 0.647 AUC: 0.647 Z: 0.310 Rdelta: 0.02576
#> 130 Number of features: 31 Max AUC: 0.647 AUC: 0.613 Z: 0.011 Rdelta: 0.02061
#> 131 Number of features: 31 Max AUC: 0.647 AUC: 0.607 Z: 0.209 Rdelta: 0.01649
#> 132 Number of features: 31 Max AUC: 0.647 AUC: 0.626 Z: 0.080 Rdelta: 0.01319
#> 133 Number of features: 31 Max AUC: 0.647 AUC: 0.610 Z: 0.035 Rdelta: 0.01055
#> 134 Number of features: 31 Max AUC: 0.647 AUC: 0.614 Z: 0.040 Rdelta: 0.00844
#> 135 Number of features: 31 Max AUC: 0.647 AUC: 0.612 Z: 0.197 Rdelta: 0.00675
#> 136 Number of features: 31 Max AUC: 0.647 AUC: 0.589 Z: 0.076 Rdelta: 0.00540
#> 137 Number of features: 31 Max AUC: 0.647 AUC: 0.593 Z: 0.092 Rdelta: 0.00432
#> 138 Number of features: 31 Max AUC: 0.647 AUC: 0.604 Z: 0.066 Rdelta: 0.00346
#> 139 Number of features: 31 Max AUC: 0.647 AUC: 0.622 Z: 0.237 Rdelta: 0.00277
#> 140 Number of features: 31 Max AUC: 0.647 AUC: 0.640 Z: 0.170 Rdelta: 0.00221
#> 141 Number of features: 31 Max AUC: 0.647 AUC: 0.634 Z: 0.248 Rdelta: 0.00177
#> 142 Number of features: 31 Max AUC: 0.647 AUC: 0.624 Z: 0.227 Rdelta: 0.00142
#> 143 Number of features: 31 Max AUC: 0.647 AUC: 0.591 Z: 0.049 Rdelta: 0.00113
#> 144 Number of features: 31 Max AUC: 0.647 AUC: 0.607 Z: 0.015 Rdelta: 0.00091
#> 145 Number of features: 31 Max AUC: 0.647 AUC: 0.552 Z: -0.044 Rdelta: 0.00073
#> 146 Number of features: 31 Max AUC: 0.647 AUC: 0.630 Z: 0.286 Rdelta: 0.00058
#> 147 Number of features: 31 Max AUC: 0.647 AUC: 0.630 Z: 0.064 Rdelta: 0.00046
#> 148 Number of features: 31 Max AUC: 0.647 AUC: 0.614 Z: 0.183 Rdelta: 0.00037
#> 149 Number of features: 31 Max AUC: 0.647 AUC: 0.628 Z: 0.149 Rdelta: 0.00030
#> 150 Number of features: 31 Max AUC: 0.647 AUC: 0.608 Z: 0.144 Rdelta: 0.00024
#> 151 Number of features: 31 Max AUC: 0.647 AUC: 0.632 Z: 0.176 Rdelta: 0.00019
#> 152 Number of features: 31 Max AUC: 0.647 AUC: 0.614 Z: 0.099 Rdelta: 0.00015
#> 153 Number of features: 31 Max AUC: 0.647 AUC: 0.597 Z: 0.058 Rdelta: 0.00012
#> 154 Number of features: 31 Max AUC: 0.647 AUC: 0.595 Z: 0.180 Rdelta: 0.00010
#> user system elapsed
#> 50.24 0.00 50.29
testDistance_control <- signatureDistance(signatureControl$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="MAN"))
#> 7 Number of features: 7 Max AUC: 0.776 AUC: 0.766 Z: 0.948 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.776 AUC: 0.767 Z: 1.474 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.810 AUC: 0.810 Z: 1.689 Rdelta: 0.10000
#> 10 Number of features: 9 Max AUC: 0.810 AUC: 0.792 Z: 1.419 Rdelta: 0.08000
#> 11 Number of features: 10 Max AUC: 0.830 AUC: 0.830 Z: 1.769 Rdelta: 0.08200
#> 12 Number of features: 11 Max AUC: 0.837 AUC: 0.837 Z: 1.792 Rdelta: 0.08380
#> 13 Number of features: 11 Max AUC: 0.837 AUC: 0.817 Z: 1.805 Rdelta: 0.06704
#> 14 Number of features: 12 Max AUC: 0.837 AUC: 0.828 Z: 1.795 Rdelta: 0.07034
#> 15 Number of features: 13 Max AUC: 0.837 AUC: 0.820 Z: 1.931 Rdelta: 0.07330
#> 16 Number of features: 14 Max AUC: 0.837 AUC: 0.822 Z: 1.974 Rdelta: 0.07597
#> 17 Number of features: 15 Max AUC: 0.837 AUC: 0.827 Z: 1.078 Rdelta: 0.07837
#> 18 Number of features: 16 Max AUC: 0.838 AUC: 0.838 Z: 0.689 Rdelta: 0.08054
#> 19 Number of features: 16 Max AUC: 0.838 AUC: 0.827 Z: 0.325 Rdelta: 0.06443
#> 20 Number of features: 17 Max AUC: 0.839 AUC: 0.839 Z: 0.647 Rdelta: 0.06799
#> 21 Number of features: 18 Max AUC: 0.839 AUC: 0.838 Z: 0.658 Rdelta: 0.07119
#> 22 Number of features: 19 Max AUC: 0.839 AUC: 0.832 Z: 0.720 Rdelta: 0.07407
#> 23 Number of features: 20 Max AUC: 0.850 AUC: 0.850 Z: 0.752 Rdelta: 0.07666
#> 24 Number of features: 21 Max AUC: 0.864 AUC: 0.864 Z: 0.510 Rdelta: 0.07900
#> 25 Number of features: 21 Max AUC: 0.864 AUC: 0.845 Z: 0.625 Rdelta: 0.06320
#> 26 Number of features: 21 Max AUC: 0.864 AUC: 0.845 Z: 0.532 Rdelta: 0.05056
#> 27 Number of features: 21 Max AUC: 0.864 AUC: 0.840 Z: 0.631 Rdelta: 0.04045
#> 28 Number of features: 22 Max AUC: 0.864 AUC: 0.854 Z: 0.493 Rdelta: 0.04640
#> 29 Number of features: 22 Max AUC: 0.864 AUC: 0.849 Z: 0.478 Rdelta: 0.03712
#> 30 Number of features: 22 Max AUC: 0.864 AUC: 0.838 Z: 0.570 Rdelta: 0.02970
#> 31 Number of features: 22 Max AUC: 0.864 AUC: 0.848 Z: 0.574 Rdelta: 0.02376
#> 32 Number of features: 23 Max AUC: 0.864 AUC: 0.858 Z: 0.412 Rdelta: 0.03138
#> 33 Number of features: 23 Max AUC: 0.864 AUC: 0.836 Z: 0.543 Rdelta: 0.02511
#> 34 Number of features: 23 Max AUC: 0.864 AUC: 0.836 Z: 0.411 Rdelta: 0.02008
#> 35 Number of features: 23 Max AUC: 0.864 AUC: 0.830 Z: 0.551 Rdelta: 0.01607
#> 36 Number of features: 24 Max AUC: 0.864 AUC: 0.857 Z: 0.447 Rdelta: 0.02446
#> 37 Number of features: 24 Max AUC: 0.864 AUC: 0.823 Z: 0.342 Rdelta: 0.01957
#> 38 Number of features: 25 Max AUC: 0.864 AUC: 0.855 Z: 0.476 Rdelta: 0.02761
#> 39 Number of features: 25 Max AUC: 0.864 AUC: 0.841 Z: 0.483 Rdelta: 0.02209
#> 40 Number of features: 26 Max AUC: 0.864 AUC: 0.861 Z: 0.462 Rdelta: 0.02988
#> 41 Number of features: 27 Max AUC: 0.864 AUC: 0.864 Z: 0.409 Rdelta: 0.03689
#> 42 Number of features: 27 Max AUC: 0.864 AUC: 0.851 Z: 0.141 Rdelta: 0.02951
#> 43 Number of features: 27 Max AUC: 0.864 AUC: 0.843 Z: 0.529 Rdelta: 0.02361
#> 44 Number of features: 27 Max AUC: 0.864 AUC: 0.847 Z: 0.520 Rdelta: 0.01889
#> 45 Number of features: 27 Max AUC: 0.864 AUC: 0.851 Z: 0.533 Rdelta: 0.01511
#> 46 Number of features: 27 Max AUC: 0.864 AUC: 0.838 Z: 0.543 Rdelta: 0.01209
#> 47 Number of features: 28 Max AUC: 0.864 AUC: 0.858 Z: 0.529 Rdelta: 0.02088
#> 48 Number of features: 29 Max AUC: 0.864 AUC: 0.863 Z: 0.129 Rdelta: 0.02879
#> 49 Number of features: 30 Max AUC: 0.864 AUC: 0.853 Z: 0.544 Rdelta: 0.03591
#> 50 Number of features: 31 Max AUC: 0.864 AUC: 0.857 Z: 0.701 Rdelta: 0.04232
#> 51 Number of features: 31 Max AUC: 0.864 AUC: 0.841 Z: 0.467 Rdelta: 0.03386
#> 52 Number of features: 32 Max AUC: 0.864 AUC: 0.861 Z: 0.576 Rdelta: 0.04047
#> 53 Number of features: 33 Max AUC: 0.864 AUC: 0.852 Z: 0.617 Rdelta: 0.04642
#> 54 Number of features: 33 Max AUC: 0.864 AUC: 0.848 Z: 0.367 Rdelta: 0.03714
#> 55 Number of features: 34 Max AUC: 0.864 AUC: 0.859 Z: 0.518 Rdelta: 0.04343
#> 56 Number of features: 35 Max AUC: 0.864 AUC: 0.862 Z: 0.489 Rdelta: 0.04908
#> 57 Number of features: 36 Max AUC: 0.864 AUC: 0.863 Z: 0.585 Rdelta: 0.05417
#> 58 Number of features: 36 Max AUC: 0.864 AUC: 0.812 Z: 0.662 Rdelta: 0.04334
#> 59 Number of features: 36 Max AUC: 0.864 AUC: 0.854 Z: 0.539 Rdelta: 0.03467
#> 60 Number of features: 37 Max AUC: 0.864 AUC: 0.864 Z: 0.539 Rdelta: 0.04120
#> 61 Number of features: 38 Max AUC: 0.864 AUC: 0.862 Z: 0.510 Rdelta: 0.04708
#> 62 Number of features: 38 Max AUC: 0.864 AUC: 0.814 Z: 0.546 Rdelta: 0.03767
#> 63 Number of features: 39 Max AUC: 0.864 AUC: 0.861 Z: 0.565 Rdelta: 0.04390
#> 64 Number of features: 40 Max AUC: 0.864 AUC: 0.862 Z: 0.563 Rdelta: 0.04951
#> 65 Number of features: 41 Max AUC: 0.864 AUC: 0.856 Z: 0.488 Rdelta: 0.05456
#> 66 Number of features: 42 Max AUC: 0.864 AUC: 0.854 Z: 0.584 Rdelta: 0.05910
#> 67 Number of features: 42 Max AUC: 0.864 AUC: 0.851 Z: 0.568 Rdelta: 0.04728
#> 68 Number of features: 42 Max AUC: 0.864 AUC: 0.843 Z: 0.540 Rdelta: 0.03783
#> 69 Number of features: 42 Max AUC: 0.864 AUC: 0.840 Z: 0.368 Rdelta: 0.03026
#> 70 Number of features: 43 Max AUC: 0.864 AUC: 0.859 Z: 0.592 Rdelta: 0.03723
#> 71 Number of features: 44 Max AUC: 0.864 AUC: 0.857 Z: 0.537 Rdelta: 0.04351
#> 72 Number of features: 44 Max AUC: 0.864 AUC: 0.847 Z: 0.551 Rdelta: 0.03481
#> 73 Number of features: 45 Max AUC: 0.864 AUC: 0.851 Z: 0.556 Rdelta: 0.04133
#> 74 Number of features: 45 Max AUC: 0.864 AUC: 0.844 Z: 0.551 Rdelta: 0.03306
#> 75 Number of features: 46 Max AUC: 0.864 AUC: 0.861 Z: 0.634 Rdelta: 0.03976
#> 76 Number of features: 46 Max AUC: 0.864 AUC: 0.838 Z: 0.482 Rdelta: 0.03181
#> 77 Number of features: 46 Max AUC: 0.864 AUC: 0.820 Z: 0.442 Rdelta: 0.02544
#> 78 Number of features: 46 Max AUC: 0.864 AUC: 0.844 Z: 0.538 Rdelta: 0.02036
#> 79 Number of features: 47 Max AUC: 0.864 AUC: 0.857 Z: 0.562 Rdelta: 0.02832
#> 80 Number of features: 47 Max AUC: 0.864 AUC: 0.848 Z: 0.409 Rdelta: 0.02266
#> 81 Number of features: 47 Max AUC: 0.864 AUC: 0.848 Z: 0.579 Rdelta: 0.01812
#> 82 Number of features: 47 Max AUC: 0.864 AUC: 0.846 Z: 0.536 Rdelta: 0.01450
#> 83 Number of features: 48 Max AUC: 0.864 AUC: 0.860 Z: 0.497 Rdelta: 0.02305
#> 84 Number of features: 48 Max AUC: 0.864 AUC: 0.843 Z: 0.235 Rdelta: 0.01844
#> 85 Number of features: 48 Max AUC: 0.864 AUC: 0.843 Z: 0.581 Rdelta: 0.01475
#> 86 Number of features: 49 Max AUC: 0.864 AUC: 0.854 Z: 0.471 Rdelta: 0.02328
#> 87 Number of features: 49 Max AUC: 0.864 AUC: 0.835 Z: 0.450 Rdelta: 0.01862
#> 88 Number of features: 49 Max AUC: 0.864 AUC: 0.809 Z: 0.542 Rdelta: 0.01490
#> 89 Number of features: 49 Max AUC: 0.864 AUC: 0.824 Z: 0.495 Rdelta: 0.01192
#> 90 Number of features: 49 Max AUC: 0.864 AUC: 0.851 Z: 0.607 Rdelta: 0.00953
#> 91 Number of features: 50 Max AUC: 0.864 AUC: 0.860 Z: 0.504 Rdelta: 0.01858
#> 92 Number of features: 50 Max AUC: 0.864 AUC: 0.850 Z: 0.544 Rdelta: 0.01486
#> 93 Number of features: 50 Max AUC: 0.864 AUC: 0.841 Z: 0.553 Rdelta: 0.01189
#> 94 Number of features: 50 Max AUC: 0.864 AUC: 0.833 Z: 0.529 Rdelta: 0.00951
#> 95 Number of features: 50 Max AUC: 0.864 AUC: 0.835 Z: 0.571 Rdelta: 0.00761
#> 96 Number of features: 50 Max AUC: 0.864 AUC: 0.818 Z: 0.550 Rdelta: 0.00609
#> 97 Number of features: 50 Max AUC: 0.864 AUC: 0.843 Z: 0.540 Rdelta: 0.00487
#> 98 Number of features: 50 Max AUC: 0.864 AUC: 0.845 Z: 0.571 Rdelta: 0.00390
#> 99 Number of features: 50 Max AUC: 0.864 AUC: 0.824 Z: 0.502 Rdelta: 0.00312
#> 100 Number of features: 50 Max AUC: 0.864 AUC: 0.820 Z: 0.480 Rdelta: 0.00249
#> 101 Number of features: 51 Max AUC: 0.864 AUC: 0.856 Z: 0.539 Rdelta: 0.01224
#> 102 Number of features: 51 Max AUC: 0.864 AUC: 0.847 Z: 0.493 Rdelta: 0.00980
#> 103 Number of features: 52 Max AUC: 0.864 AUC: 0.857 Z: 0.509 Rdelta: 0.01882
#> 104 Number of features: 52 Max AUC: 0.864 AUC: 0.842 Z: 0.505 Rdelta: 0.01505
#> 105 Number of features: 52 Max AUC: 0.864 AUC: 0.804 Z: 0.550 Rdelta: 0.01204
#> 106 Number of features: 52 Max AUC: 0.864 AUC: 0.825 Z: 0.448 Rdelta: 0.00963
#> 107 Number of features: 52 Max AUC: 0.864 AUC: 0.847 Z: 0.507 Rdelta: 0.00771
#> 108 Number of features: 52 Max AUC: 0.864 AUC: 0.833 Z: 0.509 Rdelta: 0.00617
#> 109 Number of features: 52 Max AUC: 0.864 AUC: 0.850 Z: 0.476 Rdelta: 0.00493
#> 110 Number of features: 52 Max AUC: 0.864 AUC: 0.834 Z: 0.565 Rdelta: 0.00395
#> 111 Number of features: 52 Max AUC: 0.864 AUC: 0.847 Z: 0.532 Rdelta: 0.00316
#> 112 Number of features: 52 Max AUC: 0.864 AUC: 0.836 Z: 0.217 Rdelta: 0.00253
#> 113 Number of features: 52 Max AUC: 0.864 AUC: 0.840 Z: 0.602 Rdelta: 0.00202
#> 114 Number of features: 53 Max AUC: 0.864 AUC: 0.851 Z: 0.509 Rdelta: 0.01182
#> 115 Number of features: 53 Max AUC: 0.864 AUC: 0.839 Z: 0.576 Rdelta: 0.00945
#> 116 Number of features: 53 Max AUC: 0.864 AUC: 0.843 Z: 0.516 Rdelta: 0.00756
#> 117 Number of features: 53 Max AUC: 0.864 AUC: 0.836 Z: 0.506 Rdelta: 0.00605
#> 118 Number of features: 53 Max AUC: 0.864 AUC: 0.841 Z: 0.289 Rdelta: 0.00484
#> 119 Number of features: 53 Max AUC: 0.864 AUC: 0.842 Z: 0.489 Rdelta: 0.00387
#> 120 Number of features: 53 Max AUC: 0.864 AUC: 0.825 Z: 0.538 Rdelta: 0.00310
#> 121 Number of features: 53 Max AUC: 0.864 AUC: 0.822 Z: 0.276 Rdelta: 0.00248
#> 122 Number of features: 53 Max AUC: 0.864 AUC: 0.849 Z: 0.578 Rdelta: 0.00198
#> 123 Number of features: 53 Max AUC: 0.864 AUC: 0.834 Z: 0.224 Rdelta: 0.00159
#> 124 Number of features: 54 Max AUC: 0.864 AUC: 0.860 Z: 0.621 Rdelta: 0.01143
#> 125 Number of features: 54 Max AUC: 0.864 AUC: 0.841 Z: 0.316 Rdelta: 0.00914
#> 126 Number of features: 54 Max AUC: 0.864 AUC: 0.823 Z: 0.521 Rdelta: 0.00731
#> 127 Number of features: 54 Max AUC: 0.864 AUC: 0.825 Z: 0.580 Rdelta: 0.00585
#> 128 Number of features: 54 Max AUC: 0.864 AUC: 0.819 Z: 0.502 Rdelta: 0.00468
#> 129 Number of features: 55 Max AUC: 0.864 AUC: 0.856 Z: 0.599 Rdelta: 0.01421
#> 130 Number of features: 55 Max AUC: 0.864 AUC: 0.840 Z: 0.438 Rdelta: 0.01137
#> 131 Number of features: 55 Max AUC: 0.864 AUC: 0.818 Z: 0.385 Rdelta: 0.00910
#> 132 Number of features: 55 Max AUC: 0.864 AUC: 0.833 Z: 0.481 Rdelta: 0.00728
#> 133 Number of features: 55 Max AUC: 0.864 AUC: 0.834 Z: 0.361 Rdelta: 0.00582
#> 134 Number of features: 55 Max AUC: 0.864 AUC: 0.841 Z: 0.482 Rdelta: 0.00466
#> 135 Number of features: 55 Max AUC: 0.864 AUC: 0.838 Z: 0.574 Rdelta: 0.00373
#> 136 Number of features: 55 Max AUC: 0.864 AUC: 0.834 Z: 0.513 Rdelta: 0.00298
#> 137 Number of features: 55 Max AUC: 0.864 AUC: 0.797 Z: 0.498 Rdelta: 0.00238
#> 138 Number of features: 55 Max AUC: 0.864 AUC: 0.818 Z: 0.464 Rdelta: 0.00191
#> 139 Number of features: 55 Max AUC: 0.864 AUC: 0.834 Z: 0.479 Rdelta: 0.00153
#> 140 Number of features: 55 Max AUC: 0.864 AUC: 0.791 Z: 0.466 Rdelta: 0.00122
#> 141 Number of features: 55 Max AUC: 0.864 AUC: 0.824 Z: 0.587 Rdelta: 0.00098
#> 142 Number of features: 55 Max AUC: 0.864 AUC: 0.838 Z: 0.476 Rdelta: 0.00078
#> 143 Number of features: 55 Max AUC: 0.864 AUC: 0.840 Z: 0.236 Rdelta: 0.00063
#> 144 Number of features: 55 Max AUC: 0.864 AUC: 0.834 Z: 0.339 Rdelta: 0.00050
#> 145 Number of features: 56 Max AUC: 0.864 AUC: 0.862 Z: 0.542 Rdelta: 0.01045
#> 146 Number of features: 56 Max AUC: 0.864 AUC: 0.844 Z: 0.559 Rdelta: 0.00836
#> 147 Number of features: 56 Max AUC: 0.864 AUC: 0.807 Z: 0.568 Rdelta: 0.00669
#> 148 Number of features: 56 Max AUC: 0.864 AUC: 0.812 Z: 0.576 Rdelta: 0.00535
#> 149 Number of features: 56 Max AUC: 0.864 AUC: 0.822 Z: 0.643 Rdelta: 0.00428
#> 150 Number of features: 56 Max AUC: 0.864 AUC: 0.819 Z: 0.463 Rdelta: 0.00342
#> 151 Number of features: 57 Max AUC: 0.864 AUC: 0.852 Z: 0.637 Rdelta: 0.01308
#> 152 Number of features: 57 Max AUC: 0.864 AUC: 0.806 Z: 0.499 Rdelta: 0.01047
#> 153 Number of features: 57 Max AUC: 0.864 AUC: 0.841 Z: 0.360 Rdelta: 0.00837
#> 154 Number of features: 57 Max AUC: 0.864 AUC: 0.793 Z: 0.570 Rdelta: 0.00670
#> 155 Number of features: 57 Max AUC: 0.864 AUC: 0.801 Z: 0.590 Rdelta: 0.00536
#> 156 Number of features: 57 Max AUC: 0.864 AUC: 0.836 Z: 0.396 Rdelta: 0.00429
#> 157 Number of features: 57 Max AUC: 0.864 AUC: 0.822 Z: 0.491 Rdelta: 0.00343
#> 158 Number of features: 57 Max AUC: 0.864 AUC: 0.841 Z: 0.592 Rdelta: 0.00274
#> 159 Number of features: 57 Max AUC: 0.864 AUC: 0.839 Z: 0.569 Rdelta: 0.00219
#> 160 Number of features: 57 Max AUC: 0.864 AUC: 0.796 Z: 0.416 Rdelta: 0.00176
#> 161 Number of features: 57 Max AUC: 0.864 AUC: 0.828 Z: 0.532 Rdelta: 0.00140
#> 162 Number of features: 57 Max AUC: 0.864 AUC: 0.824 Z: 0.366 Rdelta: 0.00112
#> 163 Number of features: 57 Max AUC: 0.864 AUC: 0.835 Z: 0.566 Rdelta: 0.00090
#> 164 Number of features: 57 Max AUC: 0.864 AUC: 0.807 Z: 0.389 Rdelta: 0.00072
#> 165 Number of features: 57 Max AUC: 0.864 AUC: 0.823 Z: 0.515 Rdelta: 0.00058
#> 166 Number of features: 57 Max AUC: 0.864 AUC: 0.827 Z: 0.310 Rdelta: 0.00046
#> 167 Number of features: 57 Max AUC: 0.864 AUC: 0.819 Z: 0.541 Rdelta: 0.00037
#> 168 Number of features: 57 Max AUC: 0.864 AUC: 0.846 Z: 0.608 Rdelta: 0.00029
#> 169 Number of features: 57 Max AUC: 0.864 AUC: 0.830 Z: 0.514 Rdelta: 0.00024
#> 170 Number of features: 57 Max AUC: 0.864 AUC: 0.834 Z: 0.301 Rdelta: 0.00019
#> 171 Number of features: 57 Max AUC: 0.864 AUC: 0.812 Z: 0.104 Rdelta: 0.00015
#> 172 Number of features: 57 Max AUC: 0.864 AUC: 0.822 Z: 0.556 Rdelta: 0.00012
#> 173 Number of features: 57 Max AUC: 0.864 AUC: 0.830 Z: 0.598 Rdelta: 0.00010
#> user system elapsed
#> 93.21 0.00 93.32
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"MAN")+signatureDistance(signature$controlTemplate,validLabeled,"MAN")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
#############################################################################################
varlist <- names(arceneCVTwo$bagging$frequencyTable)
#############################################################################################
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="pearson"))
#> 7 Number of features: 7 Max AUC: 0.525 AUC: 0.525 Z: 0.083 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.525 AUC: 0.525 Z: 0.066 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.525 AUC: 0.517 Z: 0.062 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.534 AUC: 0.534 Z: 0.066 Rdelta: 0.10000
#> 11 Number of features: 10 Max AUC: 0.534 AUC: 0.522 Z: 0.098 Rdelta: 0.08000
#> 12 Number of features: 10 Max AUC: 0.534 AUC: 0.504 Z: -0.009 Rdelta: 0.06400
#> 13 Number of features: 11 Max AUC: 0.534 AUC: 0.524 Z: 0.100 Rdelta: 0.06760
#> 14 Number of features: 12 Max AUC: 0.534 AUC: 0.520 Z: 0.068 Rdelta: 0.07084
#> 15 Number of features: 13 Max AUC: 0.534 AUC: 0.526 Z: 0.111 Rdelta: 0.07376
#> 16 Number of features: 14 Max AUC: 0.540 AUC: 0.540 Z: 0.161 Rdelta: 0.07638
#> 17 Number of features: 15 Max AUC: 0.560 AUC: 0.560 Z: 0.273 Rdelta: 0.07874
#> 18 Number of features: 15 Max AUC: 0.560 AUC: 0.543 Z: 0.174 Rdelta: 0.06299
#> 19 Number of features: 15 Max AUC: 0.560 AUC: 0.545 Z: 0.191 Rdelta: 0.05040
#> 20 Number of features: 15 Max AUC: 0.560 AUC: 0.540 Z: 0.105 Rdelta: 0.04032
#> 21 Number of features: 15 Max AUC: 0.560 AUC: 0.529 Z: 0.085 Rdelta: 0.03225
#> 22 Number of features: 15 Max AUC: 0.560 AUC: 0.530 Z: 0.135 Rdelta: 0.02580
#> 23 Number of features: 15 Max AUC: 0.560 AUC: 0.527 Z: 0.109 Rdelta: 0.02064
#> 24 Number of features: 16 Max AUC: 0.706 AUC: 0.706 Z: 0.880 Rdelta: 0.02858
#> 25 Number of features: 17 Max AUC: 0.708 AUC: 0.708 Z: 0.948 Rdelta: 0.03572
#> 26 Number of features: 18 Max AUC: 0.713 AUC: 0.713 Z: 0.962 Rdelta: 0.04215
#> 27 Number of features: 19 Max AUC: 0.713 AUC: 0.713 Z: 0.964 Rdelta: 0.04793
#> 28 Number of features: 20 Max AUC: 0.713 AUC: 0.710 Z: 0.965 Rdelta: 0.05314
#> 29 Number of features: 21 Max AUC: 0.716 AUC: 0.716 Z: 1.002 Rdelta: 0.05783
#> 30 Number of features: 22 Max AUC: 0.716 AUC: 0.716 Z: 1.059 Rdelta: 0.06204
#> 31 Number of features: 23 Max AUC: 0.716 AUC: 0.716 Z: 1.027 Rdelta: 0.06584
#> 32 Number of features: 24 Max AUC: 0.726 AUC: 0.726 Z: 1.069 Rdelta: 0.06926
#> 33 Number of features: 25 Max AUC: 0.726 AUC: 0.722 Z: 1.038 Rdelta: 0.07233
#> 34 Number of features: 26 Max AUC: 0.726 AUC: 0.717 Z: 1.014 Rdelta: 0.07510
#> 35 Number of features: 27 Max AUC: 0.726 AUC: 0.723 Z: 1.033 Rdelta: 0.07759
#> 36 Number of features: 28 Max AUC: 0.726 AUC: 0.719 Z: 0.994 Rdelta: 0.07983
#> 37 Number of features: 29 Max AUC: 0.727 AUC: 0.727 Z: 1.099 Rdelta: 0.08185
#> 38 Number of features: 29 Max AUC: 0.727 AUC: 0.717 Z: 1.047 Rdelta: 0.06548
#> 39 Number of features: 29 Max AUC: 0.727 AUC: 0.708 Z: 1.058 Rdelta: 0.05238
#> 40 Number of features: 30 Max AUC: 0.737 AUC: 0.737 Z: 1.095 Rdelta: 0.05714
#> 41 Number of features: 31 Max AUC: 0.743 AUC: 0.743 Z: 1.131 Rdelta: 0.06143
#> 42 Number of features: 31 Max AUC: 0.743 AUC: 0.724 Z: 1.146 Rdelta: 0.04914
#> 43 Number of features: 31 Max AUC: 0.743 AUC: 0.720 Z: 1.178 Rdelta: 0.03931
#> 44 Number of features: 31 Max AUC: 0.743 AUC: 0.733 Z: 1.195 Rdelta: 0.03145
#> 45 Number of features: 32 Max AUC: 0.751 AUC: 0.751 Z: 1.320 Rdelta: 0.03831
#> 46 Number of features: 33 Max AUC: 0.751 AUC: 0.748 Z: 1.395 Rdelta: 0.04448
#> 47 Number of features: 34 Max AUC: 0.756 AUC: 0.756 Z: 1.362 Rdelta: 0.05003
#> 48 Number of features: 35 Max AUC: 0.756 AUC: 0.756 Z: 1.422 Rdelta: 0.05503
#> 49 Number of features: 35 Max AUC: 0.756 AUC: 0.697 Z: 0.993 Rdelta: 0.04402
#> 50 Number of features: 36 Max AUC: 0.756 AUC: 0.749 Z: 1.453 Rdelta: 0.04962
#> 51 Number of features: 36 Max AUC: 0.756 AUC: 0.734 Z: 1.329 Rdelta: 0.03969
#> 52 Number of features: 36 Max AUC: 0.756 AUC: 0.737 Z: 1.375 Rdelta: 0.03176
#> 53 Number of features: 37 Max AUC: 0.756 AUC: 0.751 Z: 1.395 Rdelta: 0.03858
#> 54 Number of features: 38 Max AUC: 0.756 AUC: 0.753 Z: 1.405 Rdelta: 0.04472
#> 55 Number of features: 39 Max AUC: 0.761 AUC: 0.761 Z: 1.440 Rdelta: 0.05025
#> 56 Number of features: 39 Max AUC: 0.761 AUC: 0.745 Z: 1.392 Rdelta: 0.04020
#> 57 Number of features: 40 Max AUC: 0.765 AUC: 0.765 Z: 1.442 Rdelta: 0.04618
#> 58 Number of features: 40 Max AUC: 0.765 AUC: 0.727 Z: 1.381 Rdelta: 0.03694
#> 59 Number of features: 41 Max AUC: 0.765 AUC: 0.765 Z: 1.426 Rdelta: 0.04325
#> 60 Number of features: 41 Max AUC: 0.765 AUC: 0.751 Z: 1.415 Rdelta: 0.03460
#> 61 Number of features: 42 Max AUC: 0.765 AUC: 0.761 Z: 1.471 Rdelta: 0.04114
#> 62 Number of features: 42 Max AUC: 0.765 AUC: 0.744 Z: 1.502 Rdelta: 0.03291
#> 63 Number of features: 42 Max AUC: 0.765 AUC: 0.754 Z: 1.438 Rdelta: 0.02633
#> 64 Number of features: 43 Max AUC: 0.765 AUC: 0.764 Z: 1.451 Rdelta: 0.03370
#> 65 Number of features: 44 Max AUC: 0.765 AUC: 0.758 Z: 1.448 Rdelta: 0.04033
#> 66 Number of features: 45 Max AUC: 0.765 AUC: 0.760 Z: 1.474 Rdelta: 0.04629
#> 67 Number of features: 45 Max AUC: 0.765 AUC: 0.752 Z: 1.453 Rdelta: 0.03704
#> 68 Number of features: 46 Max AUC: 0.771 AUC: 0.771 Z: 1.456 Rdelta: 0.04333
#> 69 Number of features: 47 Max AUC: 0.771 AUC: 0.771 Z: 1.490 Rdelta: 0.04900
#> 70 Number of features: 47 Max AUC: 0.771 AUC: 0.751 Z: 1.441 Rdelta: 0.03920
#> 71 Number of features: 48 Max AUC: 0.775 AUC: 0.775 Z: 1.491 Rdelta: 0.04528
#> 72 Number of features: 48 Max AUC: 0.775 AUC: 0.748 Z: 1.432 Rdelta: 0.03622
#> 73 Number of features: 48 Max AUC: 0.775 AUC: 0.760 Z: 1.467 Rdelta: 0.02898
#> 74 Number of features: 48 Max AUC: 0.775 AUC: 0.749 Z: 1.478 Rdelta: 0.02318
#> 75 Number of features: 48 Max AUC: 0.775 AUC: 0.759 Z: 1.400 Rdelta: 0.01855
#> 76 Number of features: 48 Max AUC: 0.775 AUC: 0.765 Z: 1.526 Rdelta: 0.01484
#> 77 Number of features: 48 Max AUC: 0.775 AUC: 0.752 Z: 1.401 Rdelta: 0.01187
#> 78 Number of features: 48 Max AUC: 0.775 AUC: 0.711 Z: 1.068 Rdelta: 0.00950
#> 79 Number of features: 48 Max AUC: 0.775 AUC: 0.761 Z: 1.427 Rdelta: 0.00760
#> 80 Number of features: 49 Max AUC: 0.775 AUC: 0.769 Z: 1.430 Rdelta: 0.01684
#> 81 Number of features: 49 Max AUC: 0.775 AUC: 0.759 Z: 1.435 Rdelta: 0.01347
#> 82 Number of features: 49 Max AUC: 0.775 AUC: 0.755 Z: 1.357 Rdelta: 0.01078
#> 83 Number of features: 49 Max AUC: 0.775 AUC: 0.722 Z: 1.151 Rdelta: 0.00862
#> 84 Number of features: 49 Max AUC: 0.775 AUC: 0.750 Z: 1.367 Rdelta: 0.00690
#> 85 Number of features: 49 Max AUC: 0.775 AUC: 0.759 Z: 1.447 Rdelta: 0.00552
#> 86 Number of features: 49 Max AUC: 0.775 AUC: 0.762 Z: 1.439 Rdelta: 0.00441
#> 87 Number of features: 49 Max AUC: 0.775 AUC: 0.684 Z: 0.952 Rdelta: 0.00353
#> 88 Number of features: 49 Max AUC: 0.775 AUC: 0.745 Z: 1.352 Rdelta: 0.00282
#> 89 Number of features: 49 Max AUC: 0.775 AUC: 0.747 Z: 1.401 Rdelta: 0.00226
#> 90 Number of features: 49 Max AUC: 0.775 AUC: 0.744 Z: 1.318 Rdelta: 0.00181
#> 91 Number of features: 49 Max AUC: 0.775 AUC: 0.762 Z: 1.466 Rdelta: 0.00145
#> 92 Number of features: 49 Max AUC: 0.775 AUC: 0.747 Z: 1.361 Rdelta: 0.00116
#> 93 Number of features: 49 Max AUC: 0.775 AUC: 0.742 Z: 1.333 Rdelta: 0.00093
#> 94 Number of features: 49 Max AUC: 0.775 AUC: 0.760 Z: 1.388 Rdelta: 0.00074
#> 95 Number of features: 49 Max AUC: 0.775 AUC: 0.679 Z: 0.950 Rdelta: 0.00059
#> 96 Number of features: 49 Max AUC: 0.775 AUC: 0.730 Z: 1.146 Rdelta: 0.00047
#> 97 Number of features: 49 Max AUC: 0.775 AUC: 0.675 Z: 0.953 Rdelta: 0.00038
#> 98 Number of features: 49 Max AUC: 0.775 AUC: 0.689 Z: 0.931 Rdelta: 0.00030
#> 99 Number of features: 49 Max AUC: 0.775 AUC: 0.732 Z: 1.217 Rdelta: 0.00024
#> 100 Number of features: 49 Max AUC: 0.775 AUC: 0.724 Z: 1.284 Rdelta: 0.00019
#> 101 Number of features: 49 Max AUC: 0.775 AUC: 0.751 Z: 1.392 Rdelta: 0.00016
#> 102 Number of features: 49 Max AUC: 0.775 AUC: 0.751 Z: 1.384 Rdelta: 0.00012
#> 103 Number of features: 50 Max AUC: 0.775 AUC: 0.775 Z: 1.473 Rdelta: 0.01011
#> 104 Number of features: 50 Max AUC: 0.775 AUC: 0.731 Z: 1.143 Rdelta: 0.00809
#> 105 Number of features: 50 Max AUC: 0.775 AUC: 0.754 Z: 1.411 Rdelta: 0.00647
#> 106 Number of features: 50 Max AUC: 0.775 AUC: 0.675 Z: 0.916 Rdelta: 0.00518
#> 107 Number of features: 50 Max AUC: 0.775 AUC: 0.746 Z: 1.393 Rdelta: 0.00414
#> 108 Number of features: 50 Max AUC: 0.775 AUC: 0.741 Z: 1.324 Rdelta: 0.00331
#> 109 Number of features: 50 Max AUC: 0.775 AUC: 0.750 Z: 1.386 Rdelta: 0.00265
#> 110 Number of features: 51 Max AUC: 0.775 AUC: 0.767 Z: 1.450 Rdelta: 0.01239
#> 111 Number of features: 51 Max AUC: 0.775 AUC: 0.759 Z: 1.444 Rdelta: 0.00991
#> 112 Number of features: 51 Max AUC: 0.775 AUC: 0.763 Z: 1.413 Rdelta: 0.00793
#> 113 Number of features: 51 Max AUC: 0.775 AUC: 0.751 Z: 1.361 Rdelta: 0.00634
#> 114 Number of features: 51 Max AUC: 0.775 AUC: 0.750 Z: 1.377 Rdelta: 0.00507
#> 115 Number of features: 51 Max AUC: 0.775 AUC: 0.719 Z: 1.319 Rdelta: 0.00406
#> 116 Number of features: 52 Max AUC: 0.775 AUC: 0.768 Z: 1.390 Rdelta: 0.01365
#> 117 Number of features: 52 Max AUC: 0.775 AUC: 0.742 Z: 1.390 Rdelta: 0.01092
#> 118 Number of features: 52 Max AUC: 0.775 AUC: 0.758 Z: 1.424 Rdelta: 0.00874
#> 119 Number of features: 53 Max AUC: 0.775 AUC: 0.769 Z: 1.404 Rdelta: 0.01786
#> 120 Number of features: 53 Max AUC: 0.775 AUC: 0.762 Z: 1.420 Rdelta: 0.01429
#> 121 Number of features: 53 Max AUC: 0.775 AUC: 0.744 Z: 1.394 Rdelta: 0.01143
#> 122 Number of features: 53 Max AUC: 0.775 AUC: 0.754 Z: 1.442 Rdelta: 0.00915
#> 123 Number of features: 53 Max AUC: 0.775 AUC: 0.761 Z: 1.394 Rdelta: 0.00732
#> 124 Number of features: 53 Max AUC: 0.775 AUC: 0.727 Z: 1.249 Rdelta: 0.00585
#> 125 Number of features: 54 Max AUC: 0.775 AUC: 0.767 Z: 1.478 Rdelta: 0.01527
#> 126 Number of features: 54 Max AUC: 0.775 AUC: 0.764 Z: 1.448 Rdelta: 0.01221
#> 127 Number of features: 54 Max AUC: 0.775 AUC: 0.736 Z: 1.138 Rdelta: 0.00977
#> 128 Number of features: 54 Max AUC: 0.775 AUC: 0.758 Z: 1.430 Rdelta: 0.00782
#> 129 Number of features: 54 Max AUC: 0.775 AUC: 0.757 Z: 1.451 Rdelta: 0.00625
#> 130 Number of features: 54 Max AUC: 0.775 AUC: 0.755 Z: 1.366 Rdelta: 0.00500
#> 131 Number of features: 54 Max AUC: 0.775 AUC: 0.756 Z: 1.426 Rdelta: 0.00400
#> 132 Number of features: 54 Max AUC: 0.775 AUC: 0.699 Z: 1.077 Rdelta: 0.00320
#> 133 Number of features: 54 Max AUC: 0.775 AUC: 0.763 Z: 1.413 Rdelta: 0.00256
#> 134 Number of features: 54 Max AUC: 0.775 AUC: 0.759 Z: 1.426 Rdelta: 0.00205
#> 135 Number of features: 54 Max AUC: 0.775 AUC: 0.758 Z: 1.431 Rdelta: 0.00164
#> 136 Number of features: 54 Max AUC: 0.775 AUC: 0.728 Z: 1.181 Rdelta: 0.00131
#> 137 Number of features: 54 Max AUC: 0.775 AUC: 0.752 Z: 1.395 Rdelta: 0.00105
#> 138 Number of features: 55 Max AUC: 0.775 AUC: 0.769 Z: 1.447 Rdelta: 0.01094
#> 139 Number of features: 55 Max AUC: 0.775 AUC: 0.765 Z: 1.426 Rdelta: 0.00876
#> 140 Number of features: 55 Max AUC: 0.775 AUC: 0.746 Z: 1.188 Rdelta: 0.00700
#> 141 Number of features: 55 Max AUC: 0.775 AUC: 0.746 Z: 1.371 Rdelta: 0.00560
#> 142 Number of features: 55 Max AUC: 0.775 AUC: 0.747 Z: 1.376 Rdelta: 0.00448
#> 143 Number of features: 56 Max AUC: 0.775 AUC: 0.772 Z: 1.233 Rdelta: 0.01403
#> 144 Number of features: 57 Max AUC: 0.775 AUC: 0.772 Z: 1.245 Rdelta: 0.02263
#> 145 Number of features: 57 Max AUC: 0.775 AUC: 0.674 Z: 0.937 Rdelta: 0.01810
#> 146 Number of features: 58 Max AUC: 0.775 AUC: 0.772 Z: 1.278 Rdelta: 0.02629
#> 147 Number of features: 58 Max AUC: 0.775 AUC: 0.751 Z: 1.135 Rdelta: 0.02104
#> 148 Number of features: 58 Max AUC: 0.775 AUC: 0.750 Z: 1.212 Rdelta: 0.01683
#> 149 Number of features: 59 Max AUC: 0.776 AUC: 0.776 Z: 1.249 Rdelta: 0.02515
#> 150 Number of features: 60 Max AUC: 0.776 AUC: 0.769 Z: 1.205 Rdelta: 0.03263
#> 151 Number of features: 60 Max AUC: 0.776 AUC: 0.713 Z: 1.014 Rdelta: 0.02610
#> 152 Number of features: 60 Max AUC: 0.776 AUC: 0.753 Z: 1.196 Rdelta: 0.02088
#> 153 Number of features: 60 Max AUC: 0.776 AUC: 0.736 Z: 1.109 Rdelta: 0.01671
#> 154 Number of features: 61 Max AUC: 0.776 AUC: 0.771 Z: 1.234 Rdelta: 0.02504
#> 155 Number of features: 61 Max AUC: 0.776 AUC: 0.755 Z: 1.198 Rdelta: 0.02003
#> 156 Number of features: 61 Max AUC: 0.776 AUC: 0.750 Z: 1.135 Rdelta: 0.01602
#> 157 Number of features: 61 Max AUC: 0.776 AUC: 0.724 Z: 1.139 Rdelta: 0.01282
#> 158 Number of features: 61 Max AUC: 0.776 AUC: 0.761 Z: 1.226 Rdelta: 0.01025
#> 159 Number of features: 61 Max AUC: 0.776 AUC: 0.765 Z: 1.250 Rdelta: 0.00820
#> 160 Number of features: 61 Max AUC: 0.776 AUC: 0.750 Z: 1.182 Rdelta: 0.00656
#> 161 Number of features: 61 Max AUC: 0.776 AUC: 0.759 Z: 1.166 Rdelta: 0.00525
#> 162 Number of features: 61 Max AUC: 0.776 AUC: 0.761 Z: 1.212 Rdelta: 0.00420
#> 163 Number of features: 61 Max AUC: 0.776 AUC: 0.746 Z: 1.175 Rdelta: 0.00336
#> 164 Number of features: 61 Max AUC: 0.776 AUC: 0.734 Z: 1.167 Rdelta: 0.00269
#> 165 Number of features: 61 Max AUC: 0.776 AUC: 0.762 Z: 1.213 Rdelta: 0.00215
#> 166 Number of features: 62 Max AUC: 0.776 AUC: 0.770 Z: 1.225 Rdelta: 0.01194
#> 167 Number of features: 62 Max AUC: 0.776 AUC: 0.744 Z: 1.131 Rdelta: 0.00955
#> 168 Number of features: 62 Max AUC: 0.776 AUC: 0.705 Z: 0.839 Rdelta: 0.00764
#> 169 Number of features: 62 Max AUC: 0.776 AUC: 0.730 Z: 1.123 Rdelta: 0.00611
#> 170 Number of features: 62 Max AUC: 0.776 AUC: 0.757 Z: 1.171 Rdelta: 0.00489
#> 171 Number of features: 62 Max AUC: 0.776 AUC: 0.765 Z: 1.208 Rdelta: 0.00391
#> 172 Number of features: 62 Max AUC: 0.776 AUC: 0.756 Z: 1.171 Rdelta: 0.00313
#> 173 Number of features: 62 Max AUC: 0.776 AUC: 0.759 Z: 1.212 Rdelta: 0.00250
#> 174 Number of features: 62 Max AUC: 0.776 AUC: 0.757 Z: 1.195 Rdelta: 0.00200
#> 175 Number of features: 62 Max AUC: 0.776 AUC: 0.750 Z: 1.152 Rdelta: 0.00160
#> 176 Number of features: 62 Max AUC: 0.776 AUC: 0.756 Z: 1.188 Rdelta: 0.00128
#> 177 Number of features: 62 Max AUC: 0.776 AUC: 0.766 Z: 1.157 Rdelta: 0.00103
#> 178 Number of features: 62 Max AUC: 0.776 AUC: 0.757 Z: 1.190 Rdelta: 0.00082
#> 179 Number of features: 62 Max AUC: 0.776 AUC: 0.757 Z: 1.151 Rdelta: 0.00066
#> 180 Number of features: 62 Max AUC: 0.776 AUC: 0.754 Z: 1.226 Rdelta: 0.00052
#> 181 Number of features: 62 Max AUC: 0.776 AUC: 0.754 Z: 1.217 Rdelta: 0.00042
#> 182 Number of features: 62 Max AUC: 0.776 AUC: 0.735 Z: 0.995 Rdelta: 0.00034
#> 183 Number of features: 62 Max AUC: 0.776 AUC: 0.763 Z: 1.234 Rdelta: 0.00027
#> 184 Number of features: 62 Max AUC: 0.776 AUC: 0.694 Z: 1.070 Rdelta: 0.00022
#> 185 Number of features: 62 Max AUC: 0.776 AUC: 0.738 Z: 1.143 Rdelta: 0.00017
#> 186 Number of features: 62 Max AUC: 0.776 AUC: 0.745 Z: 1.116 Rdelta: 0.00014
#> 187 Number of features: 62 Max AUC: 0.776 AUC: 0.756 Z: 1.177 Rdelta: 0.00011
#> 188 Number of features: 62 Max AUC: 0.776 AUC: 0.764 Z: 1.221 Rdelta: 0.00009
#> user system elapsed
#> 109.02 0.01 109.09
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"pearson")+signatureDistance(signature$controlTemplate,validLabeled,"pearson")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS"))
#> 7 Number of features: 7 Max AUC: 0.715 AUC: 0.715 Z: 0.952 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.715 AUC: 0.709 Z: 0.962 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.715 AUC: 0.709 Z: 1.200 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.715 AUC: 0.707 Z: 0.954 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.726 AUC: 0.726 Z: 1.209 Rdelta: 0.10000
#> 12 Number of features: 11 Max AUC: 0.726 AUC: 0.712 Z: 1.139 Rdelta: 0.08000
#> 13 Number of features: 12 Max AUC: 0.726 AUC: 0.717 Z: 1.227 Rdelta: 0.08200
#> 14 Number of features: 13 Max AUC: 0.743 AUC: 0.743 Z: 1.374 Rdelta: 0.08380
#> 15 Number of features: 14 Max AUC: 0.748 AUC: 0.748 Z: 1.500 Rdelta: 0.08542
#> 16 Number of features: 14 Max AUC: 0.748 AUC: 0.744 Z: 0.525 Rdelta: 0.06834
#> 17 Number of features: 15 Max AUC: 0.750 AUC: 0.750 Z: 0.218 Rdelta: 0.07150
#> 18 Number of features: 16 Max AUC: 0.756 AUC: 0.756 Z: 0.239 Rdelta: 0.07435
#> 19 Number of features: 17 Max AUC: 0.768 AUC: 0.768 Z: 0.235 Rdelta: 0.07692
#> 20 Number of features: 18 Max AUC: 0.774 AUC: 0.774 Z: 0.221 Rdelta: 0.07923
#> 21 Number of features: 19 Max AUC: 0.781 AUC: 0.781 Z: 0.233 Rdelta: 0.08130
#> 22 Number of features: 19 Max AUC: 0.781 AUC: 0.759 Z: 0.288 Rdelta: 0.06504
#> 23 Number of features: 20 Max AUC: 0.781 AUC: 0.776 Z: 0.263 Rdelta: 0.06854
#> 24 Number of features: 20 Max AUC: 0.781 AUC: 0.719 Z: 0.227 Rdelta: 0.05483
#> 25 Number of features: 21 Max AUC: 0.781 AUC: 0.779 Z: 0.263 Rdelta: 0.05935
#> 26 Number of features: 21 Max AUC: 0.781 AUC: 0.755 Z: 0.231 Rdelta: 0.04748
#> 27 Number of features: 22 Max AUC: 0.785 AUC: 0.785 Z: 0.225 Rdelta: 0.05273
#> 28 Number of features: 22 Max AUC: 0.785 AUC: 0.748 Z: 0.246 Rdelta: 0.04218
#> 29 Number of features: 23 Max AUC: 0.785 AUC: 0.781 Z: 0.260 Rdelta: 0.04797
#> 30 Number of features: 23 Max AUC: 0.785 AUC: 0.770 Z: 0.233 Rdelta: 0.03837
#> 31 Number of features: 23 Max AUC: 0.785 AUC: 0.750 Z: 0.260 Rdelta: 0.03070
#> 32 Number of features: 24 Max AUC: 0.785 AUC: 0.783 Z: 0.225 Rdelta: 0.03763
#> 33 Number of features: 24 Max AUC: 0.785 AUC: 0.766 Z: 0.237 Rdelta: 0.03010
#> 34 Number of features: 24 Max AUC: 0.785 AUC: 0.764 Z: 0.231 Rdelta: 0.02408
#> 35 Number of features: 24 Max AUC: 0.785 AUC: 0.764 Z: 0.257 Rdelta: 0.01927
#> 36 Number of features: 25 Max AUC: 0.788 AUC: 0.788 Z: 0.224 Rdelta: 0.02734
#> 37 Number of features: 26 Max AUC: 0.795 AUC: 0.795 Z: 0.254 Rdelta: 0.03461
#> 38 Number of features: 26 Max AUC: 0.795 AUC: 0.785 Z: 0.252 Rdelta: 0.02768
#> 39 Number of features: 26 Max AUC: 0.795 AUC: 0.785 Z: 0.259 Rdelta: 0.02215
#> 40 Number of features: 26 Max AUC: 0.795 AUC: 0.755 Z: 0.265 Rdelta: 0.01772
#> 41 Number of features: 27 Max AUC: 0.795 AUC: 0.791 Z: 0.282 Rdelta: 0.02595
#> 42 Number of features: 27 Max AUC: 0.795 AUC: 0.776 Z: 0.328 Rdelta: 0.02076
#> 43 Number of features: 27 Max AUC: 0.795 AUC: 0.776 Z: 0.341 Rdelta: 0.01661
#> 44 Number of features: 28 Max AUC: 0.795 AUC: 0.785 Z: 0.323 Rdelta: 0.02494
#> 45 Number of features: 28 Max AUC: 0.795 AUC: 0.776 Z: 0.304 Rdelta: 0.01996
#> 46 Number of features: 28 Max AUC: 0.795 AUC: 0.779 Z: 0.360 Rdelta: 0.01596
#> 47 Number of features: 28 Max AUC: 0.795 AUC: 0.773 Z: 0.325 Rdelta: 0.01277
#> 48 Number of features: 29 Max AUC: 0.795 AUC: 0.793 Z: 0.307 Rdelta: 0.02149
#> 49 Number of features: 29 Max AUC: 0.795 AUC: 0.748 Z: 0.290 Rdelta: 0.01720
#> 50 Number of features: 29 Max AUC: 0.795 AUC: 0.775 Z: 0.265 Rdelta: 0.01376
#> 51 Number of features: 29 Max AUC: 0.795 AUC: 0.774 Z: 0.286 Rdelta: 0.01101
#> 52 Number of features: 30 Max AUC: 0.795 AUC: 0.790 Z: 0.286 Rdelta: 0.01990
#> 53 Number of features: 30 Max AUC: 0.795 AUC: 0.756 Z: 0.268 Rdelta: 0.01592
#> 54 Number of features: 30 Max AUC: 0.795 AUC: 0.763 Z: 0.266 Rdelta: 0.01274
#> 55 Number of features: 30 Max AUC: 0.795 AUC: 0.776 Z: 0.273 Rdelta: 0.01019
#> 56 Number of features: 31 Max AUC: 0.795 AUC: 0.791 Z: 0.311 Rdelta: 0.01917
#> 57 Number of features: 31 Max AUC: 0.795 AUC: 0.766 Z: 0.267 Rdelta: 0.01534
#> 58 Number of features: 31 Max AUC: 0.795 AUC: 0.760 Z: 0.274 Rdelta: 0.01227
#> 59 Number of features: 31 Max AUC: 0.795 AUC: 0.758 Z: 0.270 Rdelta: 0.00982
#> 60 Number of features: 31 Max AUC: 0.795 AUC: 0.779 Z: 0.268 Rdelta: 0.00785
#> 61 Number of features: 31 Max AUC: 0.795 AUC: 0.773 Z: 0.323 Rdelta: 0.00628
#> 62 Number of features: 32 Max AUC: 0.795 AUC: 0.790 Z: 0.290 Rdelta: 0.01565
#> 63 Number of features: 33 Max AUC: 0.795 AUC: 0.791 Z: 0.176 Rdelta: 0.02409
#> 64 Number of features: 33 Max AUC: 0.795 AUC: 0.774 Z: 0.191 Rdelta: 0.01927
#> 65 Number of features: 33 Max AUC: 0.795 AUC: 0.781 Z: 0.215 Rdelta: 0.01542
#> 66 Number of features: 33 Max AUC: 0.795 AUC: 0.774 Z: 0.223 Rdelta: 0.01233
#> 67 Number of features: 33 Max AUC: 0.795 AUC: 0.770 Z: 0.203 Rdelta: 0.00987
#> 68 Number of features: 34 Max AUC: 0.797 AUC: 0.797 Z: 0.213 Rdelta: 0.01888
#> 69 Number of features: 34 Max AUC: 0.797 AUC: 0.767 Z: 0.236 Rdelta: 0.01510
#> 70 Number of features: 34 Max AUC: 0.797 AUC: 0.773 Z: 0.204 Rdelta: 0.01208
#> 71 Number of features: 34 Max AUC: 0.797 AUC: 0.773 Z: 0.196 Rdelta: 0.00967
#> 72 Number of features: 34 Max AUC: 0.797 AUC: 0.767 Z: 0.223 Rdelta: 0.00773
#> 73 Number of features: 34 Max AUC: 0.797 AUC: 0.779 Z: 0.222 Rdelta: 0.00619
#> 74 Number of features: 34 Max AUC: 0.797 AUC: 0.767 Z: 0.206 Rdelta: 0.00495
#> 75 Number of features: 34 Max AUC: 0.797 AUC: 0.749 Z: 0.200 Rdelta: 0.00396
#> 76 Number of features: 34 Max AUC: 0.797 AUC: 0.771 Z: 0.210 Rdelta: 0.00317
#> 77 Number of features: 34 Max AUC: 0.797 AUC: 0.786 Z: 0.211 Rdelta: 0.00253
#> 78 Number of features: 34 Max AUC: 0.797 AUC: 0.767 Z: 0.223 Rdelta: 0.00203
#> 79 Number of features: 34 Max AUC: 0.797 AUC: 0.783 Z: 0.288 Rdelta: 0.00162
#> 80 Number of features: 34 Max AUC: 0.797 AUC: 0.769 Z: 0.215 Rdelta: 0.00130
#> 81 Number of features: 35 Max AUC: 0.797 AUC: 0.790 Z: 0.282 Rdelta: 0.01117
#> 82 Number of features: 36 Max AUC: 0.797 AUC: 0.788 Z: 0.279 Rdelta: 0.02005
#> 83 Number of features: 36 Max AUC: 0.797 AUC: 0.779 Z: 0.276 Rdelta: 0.01604
#> 84 Number of features: 36 Max AUC: 0.797 AUC: 0.776 Z: 0.306 Rdelta: 0.01283
#> 85 Number of features: 36 Max AUC: 0.797 AUC: 0.763 Z: 0.315 Rdelta: 0.01027
#> 86 Number of features: 36 Max AUC: 0.797 AUC: 0.780 Z: 0.318 Rdelta: 0.00821
#> 87 Number of features: 36 Max AUC: 0.797 AUC: 0.709 Z: 0.335 Rdelta: 0.00657
#> 88 Number of features: 36 Max AUC: 0.797 AUC: 0.778 Z: 0.358 Rdelta: 0.00526
#> 89 Number of features: 37 Max AUC: 0.804 AUC: 0.804 Z: 0.316 Rdelta: 0.01473
#> 90 Number of features: 37 Max AUC: 0.804 AUC: 0.794 Z: 0.293 Rdelta: 0.01178
#> 91 Number of features: 38 Max AUC: 0.804 AUC: 0.798 Z: 0.325 Rdelta: 0.02061
#> 92 Number of features: 39 Max AUC: 0.806 AUC: 0.806 Z: 0.281 Rdelta: 0.02855
#> 93 Number of features: 39 Max AUC: 0.806 AUC: 0.789 Z: 0.349 Rdelta: 0.02284
#> 94 Number of features: 39 Max AUC: 0.806 AUC: 0.784 Z: 0.339 Rdelta: 0.01827
#> 95 Number of features: 39 Max AUC: 0.806 AUC: 0.771 Z: 0.279 Rdelta: 0.01462
#> 96 Number of features: 39 Max AUC: 0.806 AUC: 0.741 Z: 0.348 Rdelta: 0.01169
#> 97 Number of features: 39 Max AUC: 0.806 AUC: 0.759 Z: 0.353 Rdelta: 0.00935
#> 98 Number of features: 39 Max AUC: 0.806 AUC: 0.715 Z: 0.403 Rdelta: 0.00748
#> 99 Number of features: 39 Max AUC: 0.806 AUC: 0.788 Z: 0.293 Rdelta: 0.00599
#> 100 Number of features: 39 Max AUC: 0.806 AUC: 0.738 Z: 0.348 Rdelta: 0.00479
#> 101 Number of features: 39 Max AUC: 0.806 AUC: 0.779 Z: 0.292 Rdelta: 0.00383
#> 102 Number of features: 39 Max AUC: 0.806 AUC: 0.782 Z: 0.297 Rdelta: 0.00307
#> 103 Number of features: 39 Max AUC: 0.806 AUC: 0.774 Z: 0.330 Rdelta: 0.00245
#> 104 Number of features: 39 Max AUC: 0.806 AUC: 0.689 Z: 0.343 Rdelta: 0.00196
#> 105 Number of features: 39 Max AUC: 0.806 AUC: 0.794 Z: 0.329 Rdelta: 0.00157
#> 106 Number of features: 39 Max AUC: 0.806 AUC: 0.756 Z: 0.352 Rdelta: 0.00126
#> 107 Number of features: 39 Max AUC: 0.806 AUC: 0.792 Z: 0.303 Rdelta: 0.00100
#> 108 Number of features: 39 Max AUC: 0.806 AUC: 0.789 Z: 0.327 Rdelta: 0.00080
#> 109 Number of features: 39 Max AUC: 0.806 AUC: 0.774 Z: 0.349 Rdelta: 0.00064
#> 110 Number of features: 39 Max AUC: 0.806 AUC: 0.775 Z: 0.396 Rdelta: 0.00051
#> 111 Number of features: 39 Max AUC: 0.806 AUC: 0.758 Z: 0.290 Rdelta: 0.00041
#> 112 Number of features: 39 Max AUC: 0.806 AUC: 0.785 Z: 0.403 Rdelta: 0.00033
#> 113 Number of features: 39 Max AUC: 0.806 AUC: 0.753 Z: 0.304 Rdelta: 0.00026
#> 114 Number of features: 39 Max AUC: 0.806 AUC: 0.788 Z: 0.426 Rdelta: 0.00021
#> 115 Number of features: 39 Max AUC: 0.806 AUC: 0.741 Z: 0.310 Rdelta: 0.00017
#> 116 Number of features: 39 Max AUC: 0.806 AUC: 0.775 Z: 0.308 Rdelta: 0.00013
#> 117 Number of features: 39 Max AUC: 0.806 AUC: 0.796 Z: 0.348 Rdelta: 0.00011
#> 118 Number of features: 39 Max AUC: 0.806 AUC: 0.791 Z: 0.336 Rdelta: 0.00009
#> user system elapsed
#> 48.47 0.00 48.50
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
testDistance_cotrol <- signatureDistance(signature$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Case"))
#> 7 Number of features: 7 Max AUC: 0.876 AUC: 0.872 Z: 1.020 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.904 AUC: 0.904 Z: 1.345 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.904 AUC: 0.887 Z: 1.129 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.904 AUC: 0.897 Z: 1.411 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.904 AUC: 0.899 Z: 1.457 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.904 AUC: 0.892 Z: 1.400 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.916 AUC: 0.916 Z: 1.374 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.920 AUC: 0.920 Z: 1.372 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.942 AUC: 0.942 Z: 1.726 Rdelta: 0.10000
#> 16 Number of features: 16 Max AUC: 0.948 AUC: 0.948 Z: 0.928 Rdelta: 0.10000
#> 17 Number of features: 17 Max AUC: 0.953 AUC: 0.953 Z: 0.344 Rdelta: 0.10000
#> 18 Number of features: 18 Max AUC: 0.955 AUC: 0.955 Z: 0.323 Rdelta: 0.10000
#> 19 Number of features: 19 Max AUC: 0.955 AUC: 0.951 Z: 0.293 Rdelta: 0.10000
#> 20 Number of features: 20 Max AUC: 0.955 AUC: 0.952 Z: 0.289 Rdelta: 0.10000
#> 21 Number of features: 21 Max AUC: 0.955 AUC: 0.942 Z: 0.336 Rdelta: 0.10000
#> 22 Number of features: 22 Max AUC: 0.955 AUC: 0.952 Z: 0.300 Rdelta: 0.10000
#> 23 Number of features: 23 Max AUC: 0.955 AUC: 0.942 Z: 0.313 Rdelta: 0.10000
#> 24 Number of features: 23 Max AUC: 0.955 AUC: 0.886 Z: 0.423 Rdelta: 0.08000
#> 25 Number of features: 24 Max AUC: 0.955 AUC: 0.940 Z: 0.381 Rdelta: 0.08200
#> 26 Number of features: 25 Max AUC: 0.955 AUC: 0.942 Z: 0.348 Rdelta: 0.08380
#> 27 Number of features: 26 Max AUC: 0.955 AUC: 0.947 Z: 0.346 Rdelta: 0.08542
#> 28 Number of features: 27 Max AUC: 0.955 AUC: 0.949 Z: 0.296 Rdelta: 0.08688
#> 29 Number of features: 28 Max AUC: 0.955 AUC: 0.939 Z: 0.258 Rdelta: 0.08819
#> 30 Number of features: 29 Max AUC: 0.955 AUC: 0.950 Z: 0.318 Rdelta: 0.08937
#> 31 Number of features: 30 Max AUC: 0.955 AUC: 0.946 Z: 0.309 Rdelta: 0.09043
#> 32 Number of features: 31 Max AUC: 0.955 AUC: 0.949 Z: 0.279 Rdelta: 0.09139
#> 33 Number of features: 31 Max AUC: 0.955 AUC: 0.932 Z: 0.271 Rdelta: 0.07311
#> 34 Number of features: 32 Max AUC: 0.955 AUC: 0.942 Z: 0.249 Rdelta: 0.07580
#> 35 Number of features: 33 Max AUC: 0.955 AUC: 0.943 Z: 0.307 Rdelta: 0.07822
#> 36 Number of features: 34 Max AUC: 0.955 AUC: 0.943 Z: 0.238 Rdelta: 0.08040
#> 37 Number of features: 35 Max AUC: 0.955 AUC: 0.942 Z: 0.255 Rdelta: 0.08236
#> 38 Number of features: 35 Max AUC: 0.955 AUC: 0.928 Z: 0.213 Rdelta: 0.06589
#> 39 Number of features: 36 Max AUC: 0.955 AUC: 0.938 Z: 0.278 Rdelta: 0.06930
#> 40 Number of features: 37 Max AUC: 0.955 AUC: 0.950 Z: 0.298 Rdelta: 0.07237
#> 41 Number of features: 38 Max AUC: 0.955 AUC: 0.946 Z: 0.437 Rdelta: 0.07513
#> 42 Number of features: 39 Max AUC: 0.955 AUC: 0.939 Z: 0.328 Rdelta: 0.07762
#> 43 Number of features: 40 Max AUC: 0.955 AUC: 0.940 Z: 0.369 Rdelta: 0.07986
#> 44 Number of features: 40 Max AUC: 0.955 AUC: 0.934 Z: 0.354 Rdelta: 0.06389
#> 45 Number of features: 41 Max AUC: 0.955 AUC: 0.938 Z: 0.331 Rdelta: 0.06750
#> 46 Number of features: 42 Max AUC: 0.955 AUC: 0.938 Z: 0.344 Rdelta: 0.07075
#> 47 Number of features: 43 Max AUC: 0.955 AUC: 0.939 Z: 0.338 Rdelta: 0.07367
#> 48 Number of features: 44 Max AUC: 0.955 AUC: 0.939 Z: 0.366 Rdelta: 0.07631
#> 49 Number of features: 44 Max AUC: 0.955 AUC: 0.926 Z: 0.287 Rdelta: 0.06104
#> 50 Number of features: 45 Max AUC: 0.955 AUC: 0.935 Z: 0.337 Rdelta: 0.06494
#> 51 Number of features: 45 Max AUC: 0.955 AUC: 0.930 Z: 0.262 Rdelta: 0.05195
#> 52 Number of features: 45 Max AUC: 0.955 AUC: 0.933 Z: 0.368 Rdelta: 0.04156
#> 53 Number of features: 46 Max AUC: 0.955 AUC: 0.936 Z: 0.337 Rdelta: 0.04741
#> 54 Number of features: 47 Max AUC: 0.955 AUC: 0.937 Z: 0.357 Rdelta: 0.05266
#> 55 Number of features: 48 Max AUC: 0.955 AUC: 0.935 Z: 0.345 Rdelta: 0.05740
#> 56 Number of features: 49 Max AUC: 0.955 AUC: 0.940 Z: 0.334 Rdelta: 0.06166
#> 57 Number of features: 49 Max AUC: 0.955 AUC: 0.928 Z: 0.352 Rdelta: 0.04933
#> 58 Number of features: 49 Max AUC: 0.955 AUC: 0.927 Z: 0.387 Rdelta: 0.03946
#> 59 Number of features: 50 Max AUC: 0.955 AUC: 0.943 Z: 0.347 Rdelta: 0.04552
#> 60 Number of features: 50 Max AUC: 0.955 AUC: 0.932 Z: 0.382 Rdelta: 0.03641
#> 61 Number of features: 50 Max AUC: 0.955 AUC: 0.928 Z: 0.314 Rdelta: 0.02913
#> 62 Number of features: 50 Max AUC: 0.955 AUC: 0.922 Z: 0.373 Rdelta: 0.02330
#> 63 Number of features: 50 Max AUC: 0.955 AUC: 0.932 Z: 0.283 Rdelta: 0.01864
#> 64 Number of features: 51 Max AUC: 0.955 AUC: 0.938 Z: 0.375 Rdelta: 0.02678
#> 65 Number of features: 51 Max AUC: 0.955 AUC: 0.933 Z: 0.352 Rdelta: 0.02142
#> 66 Number of features: 51 Max AUC: 0.955 AUC: 0.918 Z: 0.417 Rdelta: 0.01714
#> 67 Number of features: 51 Max AUC: 0.955 AUC: 0.921 Z: 0.346 Rdelta: 0.01371
#> 68 Number of features: 52 Max AUC: 0.955 AUC: 0.936 Z: 0.377 Rdelta: 0.02234
#> 69 Number of features: 52 Max AUC: 0.955 AUC: 0.926 Z: 0.401 Rdelta: 0.01787
#> 70 Number of features: 52 Max AUC: 0.955 AUC: 0.916 Z: 0.356 Rdelta: 0.01430
#> 71 Number of features: 53 Max AUC: 0.955 AUC: 0.933 Z: 0.373 Rdelta: 0.02287
#> 72 Number of features: 53 Max AUC: 0.955 AUC: 0.931 Z: 0.328 Rdelta: 0.01829
#> 73 Number of features: 53 Max AUC: 0.955 AUC: 0.924 Z: 0.235 Rdelta: 0.01464
#> 74 Number of features: 53 Max AUC: 0.955 AUC: 0.931 Z: 0.383 Rdelta: 0.01171
#> 75 Number of features: 53 Max AUC: 0.955 AUC: 0.925 Z: 0.384 Rdelta: 0.00937
#> 76 Number of features: 53 Max AUC: 0.955 AUC: 0.925 Z: 0.371 Rdelta: 0.00749
#> 77 Number of features: 54 Max AUC: 0.955 AUC: 0.933 Z: 0.294 Rdelta: 0.01674
#> 78 Number of features: 55 Max AUC: 0.955 AUC: 0.941 Z: 0.320 Rdelta: 0.02507
#> 79 Number of features: 56 Max AUC: 0.955 AUC: 0.937 Z: 0.411 Rdelta: 0.03256
#> 80 Number of features: 57 Max AUC: 0.955 AUC: 0.938 Z: 0.441 Rdelta: 0.03931
#> 81 Number of features: 58 Max AUC: 0.955 AUC: 0.931 Z: 0.468 Rdelta: 0.04538
#> 82 Number of features: 58 Max AUC: 0.955 AUC: 0.927 Z: 0.361 Rdelta: 0.03630
#> 83 Number of features: 59 Max AUC: 0.955 AUC: 0.933 Z: 0.430 Rdelta: 0.04267
#> 84 Number of features: 59 Max AUC: 0.955 AUC: 0.929 Z: 0.410 Rdelta: 0.03414
#> 85 Number of features: 60 Max AUC: 0.955 AUC: 0.931 Z: 0.393 Rdelta: 0.04072
#> 86 Number of features: 61 Max AUC: 0.955 AUC: 0.932 Z: 0.465 Rdelta: 0.04665
#> 87 Number of features: 61 Max AUC: 0.955 AUC: 0.862 Z: 0.246 Rdelta: 0.03732
#> 88 Number of features: 61 Max AUC: 0.955 AUC: 0.925 Z: 0.474 Rdelta: 0.02986
#> 89 Number of features: 62 Max AUC: 0.955 AUC: 0.941 Z: 0.471 Rdelta: 0.03687
#> 90 Number of features: 62 Max AUC: 0.955 AUC: 0.925 Z: 0.380 Rdelta: 0.02950
#> 91 Number of features: 62 Max AUC: 0.955 AUC: 0.901 Z: 0.454 Rdelta: 0.02360
#> 92 Number of features: 62 Max AUC: 0.955 AUC: 0.919 Z: 0.460 Rdelta: 0.01888
#> 93 Number of features: 62 Max AUC: 0.955 AUC: 0.931 Z: 0.473 Rdelta: 0.01510
#> 94 Number of features: 62 Max AUC: 0.955 AUC: 0.929 Z: 0.468 Rdelta: 0.01208
#> 95 Number of features: 62 Max AUC: 0.955 AUC: 0.925 Z: 0.447 Rdelta: 0.00967
#> 96 Number of features: 62 Max AUC: 0.955 AUC: 0.899 Z: 0.054 Rdelta: 0.00773
#> 97 Number of features: 62 Max AUC: 0.955 AUC: 0.915 Z: 0.464 Rdelta: 0.00619
#> 98 Number of features: 62 Max AUC: 0.955 AUC: 0.856 Z: 0.088 Rdelta: 0.00495
#> 99 Number of features: 62 Max AUC: 0.955 AUC: 0.927 Z: 0.366 Rdelta: 0.00396
#> 100 Number of features: 62 Max AUC: 0.955 AUC: 0.867 Z: 0.427 Rdelta: 0.00317
#> 101 Number of features: 62 Max AUC: 0.955 AUC: 0.931 Z: 0.424 Rdelta: 0.00253
#> 102 Number of features: 62 Max AUC: 0.955 AUC: 0.927 Z: 0.394 Rdelta: 0.00203
#> 103 Number of features: 63 Max AUC: 0.955 AUC: 0.934 Z: 0.447 Rdelta: 0.01182
#> 104 Number of features: 63 Max AUC: 0.955 AUC: 0.895 Z: 0.448 Rdelta: 0.00946
#> 105 Number of features: 63 Max AUC: 0.955 AUC: 0.922 Z: 0.249 Rdelta: 0.00757
#> 106 Number of features: 63 Max AUC: 0.955 AUC: 0.909 Z: 0.405 Rdelta: 0.00605
#> 107 Number of features: 64 Max AUC: 0.955 AUC: 0.937 Z: 0.421 Rdelta: 0.01545
#> 108 Number of features: 64 Max AUC: 0.955 AUC: 0.930 Z: 0.387 Rdelta: 0.01236
#> 109 Number of features: 64 Max AUC: 0.955 AUC: 0.926 Z: 0.408 Rdelta: 0.00989
#> 110 Number of features: 64 Max AUC: 0.955 AUC: 0.924 Z: 0.376 Rdelta: 0.00791
#> 111 Number of features: 64 Max AUC: 0.955 AUC: 0.919 Z: 0.403 Rdelta: 0.00633
#> 112 Number of features: 64 Max AUC: 0.955 AUC: 0.918 Z: 0.359 Rdelta: 0.00506
#> 113 Number of features: 64 Max AUC: 0.955 AUC: 0.892 Z: 0.227 Rdelta: 0.00405
#> 114 Number of features: 64 Max AUC: 0.955 AUC: 0.917 Z: 0.337 Rdelta: 0.00324
#> 115 Number of features: 64 Max AUC: 0.955 AUC: 0.890 Z: 0.392 Rdelta: 0.00259
#> 116 Number of features: 64 Max AUC: 0.955 AUC: 0.926 Z: 0.374 Rdelta: 0.00207
#> 117 Number of features: 64 Max AUC: 0.955 AUC: 0.928 Z: 0.405 Rdelta: 0.00166
#> 118 Number of features: 64 Max AUC: 0.955 AUC: 0.929 Z: 0.422 Rdelta: 0.00133
#> 119 Number of features: 64 Max AUC: 0.955 AUC: 0.931 Z: 0.415 Rdelta: 0.00106
#> 120 Number of features: 64 Max AUC: 0.955 AUC: 0.925 Z: 0.408 Rdelta: 0.00085
#> 121 Number of features: 64 Max AUC: 0.955 AUC: 0.930 Z: 0.387 Rdelta: 0.00068
#> 122 Number of features: 65 Max AUC: 0.955 AUC: 0.933 Z: 0.406 Rdelta: 0.01061
#> 123 Number of features: 65 Max AUC: 0.955 AUC: 0.926 Z: 0.429 Rdelta: 0.00849
#> 124 Number of features: 65 Max AUC: 0.955 AUC: 0.857 Z: 0.109 Rdelta: 0.00679
#> 125 Number of features: 65 Max AUC: 0.955 AUC: 0.919 Z: 0.293 Rdelta: 0.00543
#> 126 Number of features: 65 Max AUC: 0.955 AUC: 0.919 Z: 0.309 Rdelta: 0.00435
#> 127 Number of features: 65 Max AUC: 0.955 AUC: 0.931 Z: 0.356 Rdelta: 0.00348
#> 128 Number of features: 65 Max AUC: 0.955 AUC: 0.916 Z: 0.338 Rdelta: 0.00278
#> 129 Number of features: 66 Max AUC: 0.955 AUC: 0.935 Z: 0.343 Rdelta: 0.01250
#> 130 Number of features: 66 Max AUC: 0.955 AUC: 0.925 Z: 0.307 Rdelta: 0.01000
#> 131 Number of features: 67 Max AUC: 0.955 AUC: 0.932 Z: 0.354 Rdelta: 0.01900
#> 132 Number of features: 67 Max AUC: 0.955 AUC: 0.863 Z: 0.437 Rdelta: 0.01520
#> 133 Number of features: 67 Max AUC: 0.955 AUC: 0.925 Z: 0.262 Rdelta: 0.01216
#> 134 Number of features: 67 Max AUC: 0.955 AUC: 0.926 Z: 0.382 Rdelta: 0.00973
#> 135 Number of features: 67 Max AUC: 0.955 AUC: 0.904 Z: 0.358 Rdelta: 0.00778
#> 136 Number of features: 67 Max AUC: 0.955 AUC: 0.912 Z: 0.402 Rdelta: 0.00623
#> 137 Number of features: 67 Max AUC: 0.955 AUC: 0.909 Z: 0.306 Rdelta: 0.00498
#> 138 Number of features: 67 Max AUC: 0.955 AUC: 0.914 Z: 0.337 Rdelta: 0.00399
#> 139 Number of features: 67 Max AUC: 0.955 AUC: 0.929 Z: 0.400 Rdelta: 0.00319
#> 140 Number of features: 67 Max AUC: 0.955 AUC: 0.877 Z: 0.307 Rdelta: 0.00255
#> 141 Number of features: 67 Max AUC: 0.955 AUC: 0.920 Z: 0.275 Rdelta: 0.00204
#> 142 Number of features: 67 Max AUC: 0.955 AUC: 0.921 Z: 0.366 Rdelta: 0.00163
#> 143 Number of features: 67 Max AUC: 0.955 AUC: 0.883 Z: 0.409 Rdelta: 0.00131
#> 144 Number of features: 68 Max AUC: 0.955 AUC: 0.933 Z: 0.296 Rdelta: 0.01118
#> 145 Number of features: 68 Max AUC: 0.955 AUC: 0.900 Z: 0.350 Rdelta: 0.00894
#> 146 Number of features: 68 Max AUC: 0.955 AUC: 0.926 Z: 0.462 Rdelta: 0.00715
#> 147 Number of features: 68 Max AUC: 0.955 AUC: 0.925 Z: 0.301 Rdelta: 0.00572
#> 148 Number of features: 68 Max AUC: 0.955 AUC: 0.928 Z: 0.374 Rdelta: 0.00458
#> 149 Number of features: 68 Max AUC: 0.955 AUC: 0.929 Z: 0.296 Rdelta: 0.00366
#> 150 Number of features: 69 Max AUC: 0.955 AUC: 0.932 Z: 0.389 Rdelta: 0.01330
#> 151 Number of features: 69 Max AUC: 0.955 AUC: 0.903 Z: 0.302 Rdelta: 0.01064
#> 152 Number of features: 69 Max AUC: 0.955 AUC: 0.928 Z: 0.277 Rdelta: 0.00851
#> 153 Number of features: 69 Max AUC: 0.955 AUC: 0.923 Z: 0.307 Rdelta: 0.00681
#> 154 Number of features: 69 Max AUC: 0.955 AUC: 0.930 Z: 0.306 Rdelta: 0.00545
#> 155 Number of features: 69 Max AUC: 0.955 AUC: 0.926 Z: 0.311 Rdelta: 0.00436
#> 156 Number of features: 69 Max AUC: 0.955 AUC: 0.917 Z: 0.295 Rdelta: 0.00349
#> 157 Number of features: 69 Max AUC: 0.955 AUC: 0.907 Z: 0.323 Rdelta: 0.00279
#> 158 Number of features: 69 Max AUC: 0.955 AUC: 0.928 Z: 0.381 Rdelta: 0.00223
#> 159 Number of features: 70 Max AUC: 0.955 AUC: 0.934 Z: 0.374 Rdelta: 0.01201
#> 160 Number of features: 71 Max AUC: 0.955 AUC: 0.934 Z: 0.431 Rdelta: 0.02081
#> 161 Number of features: 71 Max AUC: 0.955 AUC: 0.928 Z: 0.318 Rdelta: 0.01665
#> 162 Number of features: 71 Max AUC: 0.955 AUC: 0.923 Z: 0.246 Rdelta: 0.01332
#> 163 Number of features: 71 Max AUC: 0.955 AUC: 0.925 Z: 0.326 Rdelta: 0.01065
#> 164 Number of features: 71 Max AUC: 0.955 AUC: 0.920 Z: 0.347 Rdelta: 0.00852
#> 165 Number of features: 71 Max AUC: 0.955 AUC: 0.922 Z: 0.384 Rdelta: 0.00682
#> 166 Number of features: 71 Max AUC: 0.955 AUC: 0.927 Z: 0.340 Rdelta: 0.00545
#> 167 Number of features: 71 Max AUC: 0.955 AUC: 0.931 Z: 0.291 Rdelta: 0.00436
#> 168 Number of features: 71 Max AUC: 0.955 AUC: 0.910 Z: 0.337 Rdelta: 0.00349
#> 169 Number of features: 72 Max AUC: 0.955 AUC: 0.933 Z: 0.347 Rdelta: 0.01314
#> 170 Number of features: 72 Max AUC: 0.955 AUC: 0.929 Z: 0.429 Rdelta: 0.01051
#> 171 Number of features: 72 Max AUC: 0.955 AUC: 0.924 Z: 0.398 Rdelta: 0.00841
#> 172 Number of features: 72 Max AUC: 0.955 AUC: 0.928 Z: 0.406 Rdelta: 0.00673
#> 173 Number of features: 72 Max AUC: 0.955 AUC: 0.906 Z: 0.336 Rdelta: 0.00538
#> 174 Number of features: 72 Max AUC: 0.955 AUC: 0.922 Z: 0.282 Rdelta: 0.00431
#> 175 Number of features: 72 Max AUC: 0.955 AUC: 0.928 Z: 0.387 Rdelta: 0.00345
#> 176 Number of features: 72 Max AUC: 0.955 AUC: 0.926 Z: 0.285 Rdelta: 0.00276
#> 177 Number of features: 72 Max AUC: 0.955 AUC: 0.922 Z: 0.369 Rdelta: 0.00220
#> 178 Number of features: 72 Max AUC: 0.955 AUC: 0.916 Z: 0.398 Rdelta: 0.00176
#> 179 Number of features: 72 Max AUC: 0.955 AUC: 0.928 Z: 0.392 Rdelta: 0.00141
#> 180 Number of features: 72 Max AUC: 0.955 AUC: 0.928 Z: 0.337 Rdelta: 0.00113
#> 181 Number of features: 72 Max AUC: 0.955 AUC: 0.922 Z: 0.440 Rdelta: 0.00090
#> 182 Number of features: 72 Max AUC: 0.955 AUC: 0.906 Z: 0.382 Rdelta: 0.00072
#> 183 Number of features: 72 Max AUC: 0.955 AUC: 0.915 Z: 0.286 Rdelta: 0.00058
#> 184 Number of features: 72 Max AUC: 0.955 AUC: 0.903 Z: 0.427 Rdelta: 0.00046
#> 185 Number of features: 72 Max AUC: 0.955 AUC: 0.922 Z: 0.360 Rdelta: 0.00037
#> 186 Number of features: 72 Max AUC: 0.955 AUC: 0.930 Z: 0.317 Rdelta: 0.00030
#> 187 Number of features: 72 Max AUC: 0.955 AUC: 0.925 Z: 0.400 Rdelta: 0.00024
#> 188 Number of features: 72 Max AUC: 0.955 AUC: 0.919 Z: 0.218 Rdelta: 0.00019
#> 189 Number of features: 72 Max AUC: 0.955 AUC: 0.908 Z: 0.308 Rdelta: 0.00015
#> 190 Number of features: 72 Max AUC: 0.955 AUC: 0.914 Z: 0.381 Rdelta: 0.00012
#> 191 Number of features: 72 Max AUC: 0.955 AUC: 0.916 Z: 0.272 Rdelta: 0.00010
#> user system elapsed
#> 127.22 0.00 127.26
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
system.time(signatureControl <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Control"))
#> 7 Number of features: 7 Max AUC: 0.321 AUC: 0.321 Z: -0.467 Rdelta: 0.10000
#> 8 Number of features: 7 Max AUC: 0.321 AUC: 0.283 Z: -0.607 Rdelta: 0.08000
#> 9 Number of features: 8 Max AUC: 0.323 AUC: 0.323 Z: -0.369 Rdelta: 0.08200
#> 10 Number of features: 9 Max AUC: 0.326 AUC: 0.326 Z: -0.506 Rdelta: 0.08380
#> 11 Number of features: 10 Max AUC: 0.326 AUC: 0.325 Z: -0.502 Rdelta: 0.08542
#> 12 Number of features: 10 Max AUC: 0.326 AUC: 0.309 Z: -0.519 Rdelta: 0.06834
#> 13 Number of features: 10 Max AUC: 0.326 AUC: 0.309 Z: -0.544 Rdelta: 0.05467
#> 14 Number of features: 10 Max AUC: 0.326 AUC: 0.316 Z: -0.465 Rdelta: 0.04374
#> 15 Number of features: 10 Max AUC: 0.326 AUC: 0.309 Z: -0.538 Rdelta: 0.03499
#> 16 Number of features: 10 Max AUC: 0.326 AUC: 0.291 Z: -0.530 Rdelta: 0.02799
#> 17 Number of features: 10 Max AUC: 0.326 AUC: 0.293 Z: -0.600 Rdelta: 0.02239
#> 18 Number of features: 11 Max AUC: 0.336 AUC: 0.336 Z: -0.454 Rdelta: 0.03015
#> 19 Number of features: 12 Max AUC: 0.336 AUC: 0.334 Z: -0.464 Rdelta: 0.03714
#> 20 Number of features: 13 Max AUC: 0.338 AUC: 0.338 Z: -0.434 Rdelta: 0.04342
#> 21 Number of features: 13 Max AUC: 0.338 AUC: 0.300 Z: -0.537 Rdelta: 0.03474
#> 22 Number of features: 13 Max AUC: 0.338 AUC: 0.304 Z: -0.530 Rdelta: 0.02779
#> 23 Number of features: 14 Max AUC: 0.338 AUC: 0.334 Z: -0.480 Rdelta: 0.03501
#> 24 Number of features: 15 Max AUC: 0.448 AUC: 0.448 Z: -0.062 Rdelta: 0.04151
#> 25 Number of features: 15 Max AUC: 0.448 AUC: 0.431 Z: -0.016 Rdelta: 0.03321
#> 26 Number of features: 15 Max AUC: 0.448 AUC: 0.437 Z: -0.136 Rdelta: 0.02657
#> 27 Number of features: 16 Max AUC: 0.448 AUC: 0.444 Z: -0.131 Rdelta: 0.03391
#> 28 Number of features: 16 Max AUC: 0.448 AUC: 0.434 Z: -0.068 Rdelta: 0.02713
#> 29 Number of features: 16 Max AUC: 0.448 AUC: 0.428 Z: -0.194 Rdelta: 0.02170
#> 30 Number of features: 16 Max AUC: 0.448 AUC: 0.427 Z: -0.104 Rdelta: 0.01736
#> 31 Number of features: 16 Max AUC: 0.448 AUC: 0.435 Z: -0.184 Rdelta: 0.01389
#> 32 Number of features: 16 Max AUC: 0.448 AUC: 0.437 Z: -0.146 Rdelta: 0.01111
#> 33 Number of features: 17 Max AUC: 0.448 AUC: 0.446 Z: -0.042 Rdelta: 0.02000
#> 34 Number of features: 17 Max AUC: 0.448 AUC: 0.426 Z: -0.090 Rdelta: 0.01600
#> 35 Number of features: 17 Max AUC: 0.448 AUC: 0.430 Z: -0.140 Rdelta: 0.01280
#> 36 Number of features: 17 Max AUC: 0.448 AUC: 0.434 Z: -0.144 Rdelta: 0.01024
#> 37 Number of features: 17 Max AUC: 0.448 AUC: 0.417 Z: -0.183 Rdelta: 0.00819
#> 38 Number of features: 17 Max AUC: 0.448 AUC: 0.417 Z: -0.196 Rdelta: 0.00655
#> 39 Number of features: 17 Max AUC: 0.448 AUC: 0.413 Z: -0.117 Rdelta: 0.00524
#> 40 Number of features: 17 Max AUC: 0.448 AUC: 0.439 Z: -0.086 Rdelta: 0.00419
#> 41 Number of features: 17 Max AUC: 0.448 AUC: 0.432 Z: -0.080 Rdelta: 0.00336
#> 42 Number of features: 18 Max AUC: 0.448 AUC: 0.447 Z: -0.056 Rdelta: 0.01302
#> 43 Number of features: 18 Max AUC: 0.448 AUC: 0.425 Z: -0.151 Rdelta: 0.01042
#> 44 Number of features: 18 Max AUC: 0.448 AUC: 0.425 Z: -0.189 Rdelta: 0.00833
#> 45 Number of features: 18 Max AUC: 0.448 AUC: 0.403 Z: -0.199 Rdelta: 0.00667
#> 46 Number of features: 18 Max AUC: 0.448 AUC: 0.411 Z: -0.234 Rdelta: 0.00533
#> 47 Number of features: 18 Max AUC: 0.448 AUC: 0.419 Z: -0.215 Rdelta: 0.00427
#> 48 Number of features: 18 Max AUC: 0.448 AUC: 0.440 Z: -0.079 Rdelta: 0.00341
#> 49 Number of features: 19 Max AUC: 0.482 AUC: 0.482 Z: 0.017 Rdelta: 0.01307
#> 50 Number of features: 20 Max AUC: 0.482 AUC: 0.482 Z: 0.005 Rdelta: 0.02176
#> 51 Number of features: 20 Max AUC: 0.482 AUC: 0.472 Z: -0.002 Rdelta: 0.01741
#> 52 Number of features: 21 Max AUC: 0.494 AUC: 0.494 Z: 0.022 Rdelta: 0.02567
#> 53 Number of features: 21 Max AUC: 0.494 AUC: 0.474 Z: -0.002 Rdelta: 0.02054
#> 54 Number of features: 21 Max AUC: 0.494 AUC: 0.472 Z: 0.005 Rdelta: 0.01643
#> 55 Number of features: 21 Max AUC: 0.494 AUC: 0.459 Z: -0.017 Rdelta: 0.01314
#> 56 Number of features: 21 Max AUC: 0.494 AUC: 0.450 Z: -0.045 Rdelta: 0.01051
#> 57 Number of features: 21 Max AUC: 0.494 AUC: 0.452 Z: -0.015 Rdelta: 0.00841
#> 58 Number of features: 21 Max AUC: 0.494 AUC: 0.470 Z: -0.056 Rdelta: 0.00673
#> 59 Number of features: 21 Max AUC: 0.494 AUC: 0.457 Z: 0.001 Rdelta: 0.00538
#> 60 Number of features: 21 Max AUC: 0.494 AUC: 0.479 Z: 0.014 Rdelta: 0.00431
#> 61 Number of features: 21 Max AUC: 0.494 AUC: 0.441 Z: -0.106 Rdelta: 0.00345
#> 62 Number of features: 21 Max AUC: 0.494 AUC: 0.487 Z: 0.017 Rdelta: 0.00276
#> 63 Number of features: 21 Max AUC: 0.494 AUC: 0.421 Z: -0.061 Rdelta: 0.00221
#> 64 Number of features: 21 Max AUC: 0.494 AUC: 0.468 Z: -0.002 Rdelta: 0.00176
#> 65 Number of features: 21 Max AUC: 0.494 AUC: 0.450 Z: -0.019 Rdelta: 0.00141
#> 66 Number of features: 21 Max AUC: 0.494 AUC: 0.458 Z: -0.061 Rdelta: 0.00113
#> 67 Number of features: 21 Max AUC: 0.494 AUC: 0.439 Z: -0.004 Rdelta: 0.00090
#> 68 Number of features: 21 Max AUC: 0.494 AUC: 0.462 Z: -0.007 Rdelta: 0.00072
#> 69 Number of features: 21 Max AUC: 0.494 AUC: 0.440 Z: -0.023 Rdelta: 0.00058
#> 70 Number of features: 21 Max AUC: 0.494 AUC: 0.441 Z: -0.132 Rdelta: 0.00046
#> 71 Number of features: 21 Max AUC: 0.494 AUC: 0.462 Z: -0.004 Rdelta: 0.00037
#> 72 Number of features: 21 Max AUC: 0.494 AUC: 0.474 Z: 0.011 Rdelta: 0.00030
#> 73 Number of features: 21 Max AUC: 0.494 AUC: 0.464 Z: 0.009 Rdelta: 0.00024
#> 74 Number of features: 21 Max AUC: 0.494 AUC: 0.459 Z: -0.019 Rdelta: 0.00019
#> 75 Number of features: 21 Max AUC: 0.494 AUC: 0.474 Z: 0.000 Rdelta: 0.00015
#> 76 Number of features: 21 Max AUC: 0.494 AUC: 0.459 Z: -0.017 Rdelta: 0.00012
#> 77 Number of features: 21 Max AUC: 0.494 AUC: 0.477 Z: 0.035 Rdelta: 0.00010
#> user system elapsed
#> 21.34 0.00 21.36
testDistance_control <- signatureDistance(signatureControl$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="MAN"))
#> 7 Number of features: 7 Max AUC: 0.733 AUC: 0.733 Z: 1.279 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.733 AUC: 0.720 Z: 1.491 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.733 AUC: 0.715 Z: 1.383 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.762 AUC: 0.762 Z: 1.620 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.781 AUC: 0.781 Z: 1.532 Rdelta: 0.10000
#> 12 Number of features: 11 Max AUC: 0.781 AUC: 0.755 Z: 1.497 Rdelta: 0.08000
#> 13 Number of features: 12 Max AUC: 0.792 AUC: 0.792 Z: 1.742 Rdelta: 0.08200
#> 14 Number of features: 13 Max AUC: 0.804 AUC: 0.804 Z: 1.665 Rdelta: 0.08380
#> 15 Number of features: 14 Max AUC: 0.821 AUC: 0.821 Z: 1.880 Rdelta: 0.08542
#> 16 Number of features: 15 Max AUC: 0.825 AUC: 0.825 Z: 0.806 Rdelta: 0.08688
#> 17 Number of features: 16 Max AUC: 0.825 AUC: 0.825 Z: 0.326 Rdelta: 0.08819
#> 18 Number of features: 17 Max AUC: 0.832 AUC: 0.832 Z: 0.407 Rdelta: 0.08937
#> 19 Number of features: 18 Max AUC: 0.832 AUC: 0.825 Z: 0.458 Rdelta: 0.09043
#> 20 Number of features: 19 Max AUC: 0.840 AUC: 0.840 Z: 0.447 Rdelta: 0.09139
#> 21 Number of features: 20 Max AUC: 0.840 AUC: 0.840 Z: 0.310 Rdelta: 0.09225
#> 22 Number of features: 20 Max AUC: 0.840 AUC: 0.823 Z: 0.388 Rdelta: 0.07380
#> 23 Number of features: 21 Max AUC: 0.840 AUC: 0.831 Z: 0.313 Rdelta: 0.07642
#> 24 Number of features: 21 Max AUC: 0.840 AUC: 0.813 Z: 0.349 Rdelta: 0.06114
#> 25 Number of features: 22 Max AUC: 0.841 AUC: 0.841 Z: 0.341 Rdelta: 0.06502
#> 26 Number of features: 23 Max AUC: 0.841 AUC: 0.833 Z: 0.359 Rdelta: 0.06852
#> 27 Number of features: 24 Max AUC: 0.841 AUC: 0.841 Z: 0.410 Rdelta: 0.07167
#> 28 Number of features: 25 Max AUC: 0.856 AUC: 0.856 Z: 0.303 Rdelta: 0.07450
#> 29 Number of features: 26 Max AUC: 0.856 AUC: 0.851 Z: 0.376 Rdelta: 0.07705
#> 30 Number of features: 26 Max AUC: 0.856 AUC: 0.840 Z: 0.307 Rdelta: 0.06164
#> 31 Number of features: 26 Max AUC: 0.856 AUC: 0.844 Z: 0.328 Rdelta: 0.04931
#> 32 Number of features: 27 Max AUC: 0.856 AUC: 0.850 Z: 0.309 Rdelta: 0.05438
#> 33 Number of features: 28 Max AUC: 0.856 AUC: 0.853 Z: 0.306 Rdelta: 0.05894
#> 34 Number of features: 28 Max AUC: 0.856 AUC: 0.843 Z: 0.307 Rdelta: 0.04715
#> 35 Number of features: 29 Max AUC: 0.856 AUC: 0.850 Z: 0.338 Rdelta: 0.05244
#> 36 Number of features: 30 Max AUC: 0.856 AUC: 0.848 Z: 0.313 Rdelta: 0.05720
#> 37 Number of features: 31 Max AUC: 0.862 AUC: 0.862 Z: 0.294 Rdelta: 0.06148
#> 38 Number of features: 32 Max AUC: 0.862 AUC: 0.861 Z: 0.326 Rdelta: 0.06533
#> 39 Number of features: 33 Max AUC: 0.866 AUC: 0.866 Z: 0.380 Rdelta: 0.06880
#> 40 Number of features: 34 Max AUC: 0.867 AUC: 0.867 Z: 0.330 Rdelta: 0.07192
#> 41 Number of features: 34 Max AUC: 0.867 AUC: 0.856 Z: 0.363 Rdelta: 0.05753
#> 42 Number of features: 34 Max AUC: 0.867 AUC: 0.853 Z: 0.378 Rdelta: 0.04603
#> 43 Number of features: 34 Max AUC: 0.867 AUC: 0.849 Z: 0.397 Rdelta: 0.03682
#> 44 Number of features: 35 Max AUC: 0.867 AUC: 0.859 Z: 0.383 Rdelta: 0.04314
#> 45 Number of features: 36 Max AUC: 0.867 AUC: 0.862 Z: 0.328 Rdelta: 0.04882
#> 46 Number of features: 36 Max AUC: 0.867 AUC: 0.845 Z: 0.355 Rdelta: 0.03906
#> 47 Number of features: 36 Max AUC: 0.867 AUC: 0.850 Z: 0.404 Rdelta: 0.03125
#> 48 Number of features: 36 Max AUC: 0.867 AUC: 0.850 Z: 0.366 Rdelta: 0.02500
#> 49 Number of features: 36 Max AUC: 0.867 AUC: 0.828 Z: 0.371 Rdelta: 0.02000
#> 50 Number of features: 36 Max AUC: 0.867 AUC: 0.844 Z: 0.405 Rdelta: 0.01600
#> 51 Number of features: 37 Max AUC: 0.867 AUC: 0.857 Z: 0.374 Rdelta: 0.02440
#> 52 Number of features: 37 Max AUC: 0.867 AUC: 0.845 Z: 0.370 Rdelta: 0.01952
#> 53 Number of features: 38 Max AUC: 0.867 AUC: 0.857 Z: 0.430 Rdelta: 0.02757
#> 54 Number of features: 38 Max AUC: 0.867 AUC: 0.853 Z: 0.403 Rdelta: 0.02205
#> 55 Number of features: 39 Max AUC: 0.867 AUC: 0.857 Z: 0.338 Rdelta: 0.02985
#> 56 Number of features: 40 Max AUC: 0.867 AUC: 0.855 Z: 0.408 Rdelta: 0.03686
#> 57 Number of features: 40 Max AUC: 0.867 AUC: 0.847 Z: 0.374 Rdelta: 0.02949
#> 58 Number of features: 40 Max AUC: 0.867 AUC: 0.844 Z: 0.348 Rdelta: 0.02359
#> 59 Number of features: 41 Max AUC: 0.867 AUC: 0.853 Z: 0.353 Rdelta: 0.03123
#> 60 Number of features: 42 Max AUC: 0.867 AUC: 0.853 Z: 0.368 Rdelta: 0.03811
#> 61 Number of features: 42 Max AUC: 0.867 AUC: 0.846 Z: 0.459 Rdelta: 0.03049
#> 62 Number of features: 42 Max AUC: 0.867 AUC: 0.842 Z: 0.363 Rdelta: 0.02439
#> 63 Number of features: 42 Max AUC: 0.867 AUC: 0.852 Z: 0.241 Rdelta: 0.01951
#> 64 Number of features: 42 Max AUC: 0.867 AUC: 0.834 Z: 0.379 Rdelta: 0.01561
#> 65 Number of features: 43 Max AUC: 0.867 AUC: 0.858 Z: 0.411 Rdelta: 0.02405
#> 66 Number of features: 43 Max AUC: 0.867 AUC: 0.836 Z: 0.419 Rdelta: 0.01924
#> 67 Number of features: 43 Max AUC: 0.867 AUC: 0.838 Z: 0.438 Rdelta: 0.01539
#> 68 Number of features: 44 Max AUC: 0.867 AUC: 0.860 Z: 0.379 Rdelta: 0.02385
#> 69 Number of features: 45 Max AUC: 0.867 AUC: 0.853 Z: 0.332 Rdelta: 0.03147
#> 70 Number of features: 45 Max AUC: 0.867 AUC: 0.839 Z: 0.400 Rdelta: 0.02517
#> 71 Number of features: 46 Max AUC: 0.867 AUC: 0.852 Z: 0.370 Rdelta: 0.03266
#> 72 Number of features: 46 Max AUC: 0.867 AUC: 0.840 Z: 0.378 Rdelta: 0.02612
#> 73 Number of features: 46 Max AUC: 0.867 AUC: 0.847 Z: 0.355 Rdelta: 0.02090
#> 74 Number of features: 46 Max AUC: 0.867 AUC: 0.846 Z: 0.400 Rdelta: 0.01672
#> 75 Number of features: 46 Max AUC: 0.867 AUC: 0.825 Z: 0.348 Rdelta: 0.01338
#> 76 Number of features: 46 Max AUC: 0.867 AUC: 0.834 Z: 0.380 Rdelta: 0.01070
#> 77 Number of features: 47 Max AUC: 0.867 AUC: 0.853 Z: 0.400 Rdelta: 0.01963
#> 78 Number of features: 47 Max AUC: 0.867 AUC: 0.824 Z: 0.404 Rdelta: 0.01570
#> 79 Number of features: 47 Max AUC: 0.867 AUC: 0.837 Z: 0.451 Rdelta: 0.01256
#> 80 Number of features: 47 Max AUC: 0.867 AUC: 0.834 Z: 0.418 Rdelta: 0.01005
#> 81 Number of features: 47 Max AUC: 0.867 AUC: 0.842 Z: 0.449 Rdelta: 0.00804
#> 82 Number of features: 48 Max AUC: 0.867 AUC: 0.850 Z: 0.405 Rdelta: 0.01724
#> 83 Number of features: 48 Max AUC: 0.867 AUC: 0.825 Z: 0.457 Rdelta: 0.01379
#> 84 Number of features: 48 Max AUC: 0.867 AUC: 0.836 Z: 0.450 Rdelta: 0.01103
#> 85 Number of features: 48 Max AUC: 0.867 AUC: 0.839 Z: 0.439 Rdelta: 0.00883
#> 86 Number of features: 48 Max AUC: 0.867 AUC: 0.847 Z: 0.442 Rdelta: 0.00706
#> 87 Number of features: 48 Max AUC: 0.867 AUC: 0.751 Z: 0.482 Rdelta: 0.00565
#> 88 Number of features: 48 Max AUC: 0.867 AUC: 0.831 Z: 0.465 Rdelta: 0.00452
#> 89 Number of features: 49 Max AUC: 0.867 AUC: 0.849 Z: 0.500 Rdelta: 0.01407
#> 90 Number of features: 49 Max AUC: 0.867 AUC: 0.839 Z: 0.461 Rdelta: 0.01125
#> 91 Number of features: 49 Max AUC: 0.867 AUC: 0.846 Z: 0.460 Rdelta: 0.00900
#> 92 Number of features: 49 Max AUC: 0.867 AUC: 0.840 Z: 0.450 Rdelta: 0.00720
#> 93 Number of features: 49 Max AUC: 0.867 AUC: 0.838 Z: 0.442 Rdelta: 0.00576
#> 94 Number of features: 49 Max AUC: 0.867 AUC: 0.842 Z: 0.423 Rdelta: 0.00461
#> 95 Number of features: 49 Max AUC: 0.867 AUC: 0.828 Z: 0.425 Rdelta: 0.00369
#> 96 Number of features: 49 Max AUC: 0.867 AUC: 0.837 Z: 0.443 Rdelta: 0.00295
#> 97 Number of features: 49 Max AUC: 0.867 AUC: 0.803 Z: 0.461 Rdelta: 0.00236
#> 98 Number of features: 49 Max AUC: 0.867 AUC: 0.759 Z: 0.459 Rdelta: 0.00189
#> 99 Number of features: 49 Max AUC: 0.867 AUC: 0.848 Z: 0.451 Rdelta: 0.00151
#> 100 Number of features: 49 Max AUC: 0.867 AUC: 0.840 Z: 0.520 Rdelta: 0.00121
#> 101 Number of features: 49 Max AUC: 0.867 AUC: 0.836 Z: 0.400 Rdelta: 0.00097
#> 102 Number of features: 49 Max AUC: 0.867 AUC: 0.824 Z: 0.489 Rdelta: 0.00077
#> 103 Number of features: 49 Max AUC: 0.867 AUC: 0.837 Z: 0.316 Rdelta: 0.00062
#> 104 Number of features: 49 Max AUC: 0.867 AUC: 0.785 Z: 0.456 Rdelta: 0.00049
#> 105 Number of features: 49 Max AUC: 0.867 AUC: 0.833 Z: 0.415 Rdelta: 0.00040
#> 106 Number of features: 49 Max AUC: 0.867 AUC: 0.803 Z: 0.299 Rdelta: 0.00032
#> 107 Number of features: 49 Max AUC: 0.867 AUC: 0.835 Z: 0.395 Rdelta: 0.00025
#> 108 Number of features: 49 Max AUC: 0.867 AUC: 0.847 Z: 0.492 Rdelta: 0.00020
#> 109 Number of features: 49 Max AUC: 0.867 AUC: 0.845 Z: 0.448 Rdelta: 0.00016
#> 110 Number of features: 49 Max AUC: 0.867 AUC: 0.839 Z: 0.516 Rdelta: 0.00013
#> 111 Number of features: 49 Max AUC: 0.867 AUC: 0.830 Z: 0.447 Rdelta: 0.00010
#> 112 Number of features: 49 Max AUC: 0.867 AUC: 0.836 Z: 0.553 Rdelta: 0.00008
#> user system elapsed
#> 53.15 0.00 53.20
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"MAN")+signatureDistance(signature$controlTemplate,validLabeled,"MAN")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
#############################################################################################
varlist <- names(arceneCVThree$bagging$frequencyTable)
#############################################################################################
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="pearson"))
#> 7 Number of features: 7 Max AUC: 0.480 AUC: 0.480 Z: -0.174 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.488 AUC: 0.488 Z: -0.092 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.528 AUC: 0.528 Z: 0.109 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.539 AUC: 0.539 Z: 0.161 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.558 AUC: 0.558 Z: 0.212 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.566 AUC: 0.566 Z: 0.247 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.573 AUC: 0.573 Z: 0.240 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.582 AUC: 0.582 Z: 0.309 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.582 AUC: 0.577 Z: 0.279 Rdelta: 0.10000
#> 16 Number of features: 15 Max AUC: 0.582 AUC: 0.567 Z: 0.329 Rdelta: 0.08000
#> 17 Number of features: 16 Max AUC: 0.582 AUC: 0.579 Z: 0.326 Rdelta: 0.08200
#> 18 Number of features: 17 Max AUC: 0.586 AUC: 0.586 Z: 0.330 Rdelta: 0.08380
#> 19 Number of features: 17 Max AUC: 0.586 AUC: 0.566 Z: 0.271 Rdelta: 0.06704
#> 20 Number of features: 18 Max AUC: 0.586 AUC: 0.584 Z: 0.311 Rdelta: 0.07034
#> 21 Number of features: 18 Max AUC: 0.586 AUC: 0.574 Z: 0.362 Rdelta: 0.05627
#> 22 Number of features: 19 Max AUC: 0.704 AUC: 0.704 Z: 1.287 Rdelta: 0.06064
#> 23 Number of features: 20 Max AUC: 0.704 AUC: 0.702 Z: 1.362 Rdelta: 0.06458
#> 24 Number of features: 21 Max AUC: 0.708 AUC: 0.708 Z: 1.376 Rdelta: 0.06812
#> 25 Number of features: 22 Max AUC: 0.711 AUC: 0.711 Z: 1.386 Rdelta: 0.07131
#> 26 Number of features: 22 Max AUC: 0.711 AUC: 0.702 Z: 1.252 Rdelta: 0.05705
#> 27 Number of features: 23 Max AUC: 0.711 AUC: 0.704 Z: 1.320 Rdelta: 0.06134
#> 28 Number of features: 24 Max AUC: 0.711 AUC: 0.706 Z: 1.288 Rdelta: 0.06521
#> 29 Number of features: 24 Max AUC: 0.711 AUC: 0.689 Z: 1.331 Rdelta: 0.05217
#> 30 Number of features: 24 Max AUC: 0.711 AUC: 0.682 Z: 1.208 Rdelta: 0.04173
#> 31 Number of features: 24 Max AUC: 0.711 AUC: 0.701 Z: 1.276 Rdelta: 0.03339
#> 32 Number of features: 24 Max AUC: 0.711 AUC: 0.691 Z: 1.246 Rdelta: 0.02671
#> 33 Number of features: 24 Max AUC: 0.711 AUC: 0.693 Z: 1.300 Rdelta: 0.02137
#> 34 Number of features: 24 Max AUC: 0.711 AUC: 0.691 Z: 1.130 Rdelta: 0.01709
#> 35 Number of features: 24 Max AUC: 0.711 AUC: 0.699 Z: 1.290 Rdelta: 0.01368
#> 36 Number of features: 25 Max AUC: 0.711 AUC: 0.700 Z: 1.324 Rdelta: 0.02231
#> 37 Number of features: 26 Max AUC: 0.711 AUC: 0.709 Z: 1.379 Rdelta: 0.03008
#> 38 Number of features: 27 Max AUC: 0.711 AUC: 0.706 Z: 1.233 Rdelta: 0.03707
#> 39 Number of features: 27 Max AUC: 0.711 AUC: 0.695 Z: 1.234 Rdelta: 0.02966
#> 40 Number of features: 27 Max AUC: 0.711 AUC: 0.676 Z: 1.060 Rdelta: 0.02372
#> 41 Number of features: 28 Max AUC: 0.735 AUC: 0.735 Z: 1.244 Rdelta: 0.03135
#> 42 Number of features: 29 Max AUC: 0.738 AUC: 0.738 Z: 1.314 Rdelta: 0.03822
#> 43 Number of features: 30 Max AUC: 0.738 AUC: 0.732 Z: 1.309 Rdelta: 0.04439
#> 44 Number of features: 30 Max AUC: 0.738 AUC: 0.729 Z: 1.252 Rdelta: 0.03552
#> 45 Number of features: 30 Max AUC: 0.738 AUC: 0.730 Z: 1.230 Rdelta: 0.02841
#> 46 Number of features: 30 Max AUC: 0.738 AUC: 0.730 Z: 1.176 Rdelta: 0.02273
#> 47 Number of features: 31 Max AUC: 0.738 AUC: 0.731 Z: 1.230 Rdelta: 0.03046
#> 48 Number of features: 31 Max AUC: 0.738 AUC: 0.729 Z: 1.225 Rdelta: 0.02437
#> 49 Number of features: 32 Max AUC: 0.766 AUC: 0.766 Z: 1.441 Rdelta: 0.03193
#> 50 Number of features: 33 Max AUC: 0.775 AUC: 0.775 Z: 1.597 Rdelta: 0.03874
#> 51 Number of features: 34 Max AUC: 0.775 AUC: 0.772 Z: 1.567 Rdelta: 0.04486
#> 52 Number of features: 34 Max AUC: 0.775 AUC: 0.766 Z: 1.591 Rdelta: 0.03589
#> 53 Number of features: 35 Max AUC: 0.775 AUC: 0.774 Z: 1.519 Rdelta: 0.04230
#> 54 Number of features: 35 Max AUC: 0.775 AUC: 0.764 Z: 1.566 Rdelta: 0.03384
#> 55 Number of features: 36 Max AUC: 0.786 AUC: 0.786 Z: 1.727 Rdelta: 0.04046
#> 56 Number of features: 36 Max AUC: 0.786 AUC: 0.778 Z: 1.690 Rdelta: 0.03237
#> 57 Number of features: 37 Max AUC: 0.786 AUC: 0.785 Z: 1.736 Rdelta: 0.03913
#> 58 Number of features: 37 Max AUC: 0.786 AUC: 0.778 Z: 1.681 Rdelta: 0.03130
#> 59 Number of features: 38 Max AUC: 0.786 AUC: 0.785 Z: 1.712 Rdelta: 0.03817
#> 60 Number of features: 38 Max AUC: 0.786 AUC: 0.774 Z: 1.589 Rdelta: 0.03054
#> 61 Number of features: 39 Max AUC: 0.786 AUC: 0.782 Z: 1.674 Rdelta: 0.03748
#> 62 Number of features: 39 Max AUC: 0.786 AUC: 0.768 Z: 1.654 Rdelta: 0.02999
#> 63 Number of features: 40 Max AUC: 0.786 AUC: 0.781 Z: 1.673 Rdelta: 0.03699
#> 64 Number of features: 40 Max AUC: 0.786 AUC: 0.773 Z: 1.684 Rdelta: 0.02959
#> 65 Number of features: 41 Max AUC: 0.786 AUC: 0.781 Z: 1.646 Rdelta: 0.03663
#> 66 Number of features: 42 Max AUC: 0.786 AUC: 0.779 Z: 1.672 Rdelta: 0.04297
#> 67 Number of features: 42 Max AUC: 0.786 AUC: 0.767 Z: 1.656 Rdelta: 0.03437
#> 68 Number of features: 43 Max AUC: 0.786 AUC: 0.777 Z: 1.685 Rdelta: 0.04094
#> 69 Number of features: 44 Max AUC: 0.786 AUC: 0.785 Z: 1.798 Rdelta: 0.04684
#> 70 Number of features: 45 Max AUC: 0.786 AUC: 0.782 Z: 1.790 Rdelta: 0.05216
#> 71 Number of features: 46 Max AUC: 0.812 AUC: 0.812 Z: 1.901 Rdelta: 0.05694
#> 72 Number of features: 46 Max AUC: 0.812 AUC: 0.801 Z: 1.821 Rdelta: 0.04555
#> 73 Number of features: 47 Max AUC: 0.812 AUC: 0.812 Z: 1.918 Rdelta: 0.05100
#> 74 Number of features: 48 Max AUC: 0.814 AUC: 0.814 Z: 1.948 Rdelta: 0.05590
#> 75 Number of features: 49 Max AUC: 0.814 AUC: 0.808 Z: 1.915 Rdelta: 0.06031
#> 76 Number of features: 50 Max AUC: 0.814 AUC: 0.806 Z: 1.869 Rdelta: 0.06428
#> 77 Number of features: 51 Max AUC: 0.814 AUC: 0.806 Z: 1.848 Rdelta: 0.06785
#> 78 Number of features: 51 Max AUC: 0.814 AUC: 0.798 Z: 1.899 Rdelta: 0.05428
#> 79 Number of features: 51 Max AUC: 0.814 AUC: 0.799 Z: 1.849 Rdelta: 0.04342
#> 80 Number of features: 51 Max AUC: 0.814 AUC: 0.804 Z: 1.864 Rdelta: 0.03474
#> 81 Number of features: 52 Max AUC: 0.814 AUC: 0.806 Z: 1.823 Rdelta: 0.04127
#> 82 Number of features: 53 Max AUC: 0.814 AUC: 0.808 Z: 1.857 Rdelta: 0.04714
#> 83 Number of features: 53 Max AUC: 0.814 AUC: 0.750 Z: 1.478 Rdelta: 0.03771
#> 84 Number of features: 54 Max AUC: 0.814 AUC: 0.806 Z: 1.840 Rdelta: 0.04394
#> 85 Number of features: 55 Max AUC: 0.814 AUC: 0.806 Z: 1.898 Rdelta: 0.04955
#> 86 Number of features: 55 Max AUC: 0.814 AUC: 0.798 Z: 1.821 Rdelta: 0.03964
#> 87 Number of features: 55 Max AUC: 0.814 AUC: 0.796 Z: 1.817 Rdelta: 0.03171
#> 88 Number of features: 55 Max AUC: 0.814 AUC: 0.799 Z: 1.899 Rdelta: 0.02537
#> 89 Number of features: 55 Max AUC: 0.814 AUC: 0.794 Z: 1.780 Rdelta: 0.02029
#> 90 Number of features: 56 Max AUC: 0.814 AUC: 0.805 Z: 1.842 Rdelta: 0.02826
#> 91 Number of features: 56 Max AUC: 0.814 AUC: 0.798 Z: 1.852 Rdelta: 0.02261
#> 92 Number of features: 56 Max AUC: 0.814 AUC: 0.788 Z: 1.833 Rdelta: 0.01809
#> 93 Number of features: 56 Max AUC: 0.814 AUC: 0.793 Z: 1.826 Rdelta: 0.01447
#> 94 Number of features: 56 Max AUC: 0.814 AUC: 0.787 Z: 1.775 Rdelta: 0.01158
#> 95 Number of features: 56 Max AUC: 0.814 AUC: 0.791 Z: 1.787 Rdelta: 0.00926
#> 96 Number of features: 56 Max AUC: 0.814 AUC: 0.793 Z: 1.764 Rdelta: 0.00741
#> 97 Number of features: 56 Max AUC: 0.814 AUC: 0.802 Z: 1.881 Rdelta: 0.00593
#> 98 Number of features: 56 Max AUC: 0.814 AUC: 0.793 Z: 1.810 Rdelta: 0.00474
#> 99 Number of features: 56 Max AUC: 0.814 AUC: 0.782 Z: 1.763 Rdelta: 0.00379
#> 100 Number of features: 56 Max AUC: 0.814 AUC: 0.791 Z: 1.792 Rdelta: 0.00303
#> 101 Number of features: 56 Max AUC: 0.814 AUC: 0.803 Z: 1.828 Rdelta: 0.00243
#> 102 Number of features: 57 Max AUC: 0.814 AUC: 0.804 Z: 1.856 Rdelta: 0.01219
#> 103 Number of features: 57 Max AUC: 0.814 AUC: 0.797 Z: 1.850 Rdelta: 0.00975
#> 104 Number of features: 57 Max AUC: 0.814 AUC: 0.789 Z: 1.859 Rdelta: 0.00780
#> 105 Number of features: 57 Max AUC: 0.814 AUC: 0.794 Z: 1.821 Rdelta: 0.00624
#> 106 Number of features: 57 Max AUC: 0.814 AUC: 0.802 Z: 1.929 Rdelta: 0.00499
#> 107 Number of features: 58 Max AUC: 0.814 AUC: 0.811 Z: 1.909 Rdelta: 0.01449
#> 108 Number of features: 58 Max AUC: 0.814 AUC: 0.792 Z: 1.833 Rdelta: 0.01159
#> 109 Number of features: 58 Max AUC: 0.814 AUC: 0.801 Z: 1.904 Rdelta: 0.00927
#> 110 Number of features: 59 Max AUC: 0.816 AUC: 0.816 Z: 1.959 Rdelta: 0.01835
#> 111 Number of features: 59 Max AUC: 0.816 AUC: 0.804 Z: 1.871 Rdelta: 0.01468
#> 112 Number of features: 60 Max AUC: 0.816 AUC: 0.810 Z: 1.973 Rdelta: 0.02321
#> 113 Number of features: 60 Max AUC: 0.816 AUC: 0.800 Z: 1.651 Rdelta: 0.01857
#> 114 Number of features: 60 Max AUC: 0.816 AUC: 0.798 Z: 1.924 Rdelta: 0.01485
#> 115 Number of features: 60 Max AUC: 0.816 AUC: 0.807 Z: 1.971 Rdelta: 0.01188
#> 116 Number of features: 60 Max AUC: 0.816 AUC: 0.807 Z: 1.970 Rdelta: 0.00951
#> 117 Number of features: 61 Max AUC: 0.816 AUC: 0.812 Z: 2.044 Rdelta: 0.01856
#> 118 Number of features: 61 Max AUC: 0.816 AUC: 0.806 Z: 1.972 Rdelta: 0.01484
#> 119 Number of features: 62 Max AUC: 0.820 AUC: 0.820 Z: 2.071 Rdelta: 0.02336
#> 120 Number of features: 63 Max AUC: 0.824 AUC: 0.824 Z: 2.094 Rdelta: 0.03102
#> 121 Number of features: 64 Max AUC: 0.824 AUC: 0.817 Z: 2.064 Rdelta: 0.03792
#> 122 Number of features: 64 Max AUC: 0.824 AUC: 0.809 Z: 2.051 Rdelta: 0.03034
#> 123 Number of features: 64 Max AUC: 0.824 AUC: 0.803 Z: 1.909 Rdelta: 0.02427
#> 124 Number of features: 64 Max AUC: 0.824 AUC: 0.810 Z: 2.067 Rdelta: 0.01942
#> 125 Number of features: 65 Max AUC: 0.826 AUC: 0.826 Z: 2.075 Rdelta: 0.02747
#> 126 Number of features: 65 Max AUC: 0.826 AUC: 0.809 Z: 2.062 Rdelta: 0.02198
#> 127 Number of features: 66 Max AUC: 0.826 AUC: 0.822 Z: 1.763 Rdelta: 0.02978
#> 128 Number of features: 67 Max AUC: 0.828 AUC: 0.828 Z: 1.829 Rdelta: 0.03680
#> 129 Number of features: 68 Max AUC: 0.831 AUC: 0.831 Z: 1.872 Rdelta: 0.04312
#> 130 Number of features: 68 Max AUC: 0.831 AUC: 0.819 Z: 1.837 Rdelta: 0.03450
#> 131 Number of features: 68 Max AUC: 0.831 AUC: 0.780 Z: 1.456 Rdelta: 0.02760
#> 132 Number of features: 68 Max AUC: 0.831 AUC: 0.818 Z: 1.851 Rdelta: 0.02208
#> 133 Number of features: 69 Max AUC: 0.831 AUC: 0.827 Z: 1.898 Rdelta: 0.02987
#> 134 Number of features: 70 Max AUC: 0.831 AUC: 0.828 Z: 1.873 Rdelta: 0.03688
#> 135 Number of features: 71 Max AUC: 0.831 AUC: 0.822 Z: 1.868 Rdelta: 0.04320
#> 136 Number of features: 71 Max AUC: 0.831 AUC: 0.769 Z: 1.567 Rdelta: 0.03456
#> 137 Number of features: 72 Max AUC: 0.831 AUC: 0.831 Z: 1.863 Rdelta: 0.04110
#> 138 Number of features: 73 Max AUC: 0.831 AUC: 0.824 Z: 1.879 Rdelta: 0.04699
#> 139 Number of features: 73 Max AUC: 0.831 AUC: 0.821 Z: 1.884 Rdelta: 0.03759
#> 140 Number of features: 73 Max AUC: 0.831 AUC: 0.817 Z: 1.813 Rdelta: 0.03007
#> 141 Number of features: 74 Max AUC: 0.831 AUC: 0.823 Z: 1.833 Rdelta: 0.03707
#> 142 Number of features: 74 Max AUC: 0.831 AUC: 0.813 Z: 1.790 Rdelta: 0.02965
#> 143 Number of features: 74 Max AUC: 0.831 AUC: 0.819 Z: 1.798 Rdelta: 0.02372
#> 144 Number of features: 74 Max AUC: 0.831 AUC: 0.819 Z: 1.815 Rdelta: 0.01898
#> 145 Number of features: 74 Max AUC: 0.831 AUC: 0.770 Z: 1.484 Rdelta: 0.01518
#> 146 Number of features: 74 Max AUC: 0.831 AUC: 0.819 Z: 1.774 Rdelta: 0.01215
#> 147 Number of features: 74 Max AUC: 0.831 AUC: 0.731 Z: 1.077 Rdelta: 0.00972
#> 148 Number of features: 74 Max AUC: 0.831 AUC: 0.819 Z: 1.812 Rdelta: 0.00777
#> 149 Number of features: 75 Max AUC: 0.832 AUC: 0.832 Z: 1.862 Rdelta: 0.01700
#> 150 Number of features: 76 Max AUC: 0.832 AUC: 0.825 Z: 1.842 Rdelta: 0.02530
#> 151 Number of features: 76 Max AUC: 0.832 AUC: 0.727 Z: 1.077 Rdelta: 0.02024
#> 152 Number of features: 76 Max AUC: 0.832 AUC: 0.796 Z: 1.801 Rdelta: 0.01619
#> 153 Number of features: 76 Max AUC: 0.832 AUC: 0.819 Z: 1.829 Rdelta: 0.01295
#> 154 Number of features: 76 Max AUC: 0.832 AUC: 0.751 Z: 1.386 Rdelta: 0.01036
#> 155 Number of features: 77 Max AUC: 0.832 AUC: 0.825 Z: 1.831 Rdelta: 0.01933
#> 156 Number of features: 77 Max AUC: 0.832 AUC: 0.817 Z: 1.820 Rdelta: 0.01546
#> 157 Number of features: 77 Max AUC: 0.832 AUC: 0.819 Z: 1.837 Rdelta: 0.01237
#> 158 Number of features: 77 Max AUC: 0.832 AUC: 0.775 Z: 1.547 Rdelta: 0.00989
#> 159 Number of features: 77 Max AUC: 0.832 AUC: 0.820 Z: 1.823 Rdelta: 0.00792
#> 160 Number of features: 77 Max AUC: 0.832 AUC: 0.809 Z: 1.775 Rdelta: 0.00633
#> 161 Number of features: 77 Max AUC: 0.832 AUC: 0.812 Z: 1.755 Rdelta: 0.00507
#> 162 Number of features: 77 Max AUC: 0.832 AUC: 0.813 Z: 1.708 Rdelta: 0.00405
#> 163 Number of features: 77 Max AUC: 0.832 AUC: 0.817 Z: 1.840 Rdelta: 0.00324
#> 164 Number of features: 77 Max AUC: 0.832 AUC: 0.813 Z: 1.773 Rdelta: 0.00259
#> 165 Number of features: 78 Max AUC: 0.832 AUC: 0.824 Z: 1.830 Rdelta: 0.01233
#> 166 Number of features: 78 Max AUC: 0.832 AUC: 0.816 Z: 1.826 Rdelta: 0.00987
#> 167 Number of features: 78 Max AUC: 0.832 AUC: 0.807 Z: 1.837 Rdelta: 0.00789
#> 168 Number of features: 79 Max AUC: 0.832 AUC: 0.829 Z: 1.866 Rdelta: 0.01710
#> 169 Number of features: 80 Max AUC: 0.832 AUC: 0.825 Z: 1.902 Rdelta: 0.02539
#> 170 Number of features: 80 Max AUC: 0.832 AUC: 0.808 Z: 1.789 Rdelta: 0.02032
#> 171 Number of features: 80 Max AUC: 0.832 AUC: 0.819 Z: 1.798 Rdelta: 0.01625
#> 172 Number of features: 81 Max AUC: 0.832 AUC: 0.824 Z: 1.865 Rdelta: 0.02463
#> 173 Number of features: 82 Max AUC: 0.841 AUC: 0.841 Z: 1.969 Rdelta: 0.03216
#> 174 Number of features: 83 Max AUC: 0.841 AUC: 0.836 Z: 1.994 Rdelta: 0.03895
#> 175 Number of features: 84 Max AUC: 0.841 AUC: 0.836 Z: 1.958 Rdelta: 0.04505
#> 176 Number of features: 84 Max AUC: 0.841 AUC: 0.806 Z: 1.772 Rdelta: 0.03604
#> 177 Number of features: 84 Max AUC: 0.841 AUC: 0.803 Z: 1.781 Rdelta: 0.02883
#> 178 Number of features: 85 Max AUC: 0.841 AUC: 0.837 Z: 1.898 Rdelta: 0.03595
#> 179 Number of features: 86 Max AUC: 0.841 AUC: 0.841 Z: 1.881 Rdelta: 0.04236
#> 180 Number of features: 86 Max AUC: 0.841 AUC: 0.831 Z: 1.827 Rdelta: 0.03388
#> 181 Number of features: 86 Max AUC: 0.841 AUC: 0.755 Z: 1.497 Rdelta: 0.02711
#> 182 Number of features: 86 Max AUC: 0.841 AUC: 0.750 Z: 1.373 Rdelta: 0.02169
#> 183 Number of features: 86 Max AUC: 0.841 AUC: 0.826 Z: 1.820 Rdelta: 0.01735
#> 184 Number of features: 86 Max AUC: 0.841 AUC: 0.833 Z: 1.849 Rdelta: 0.01388
#> 185 Number of features: 87 Max AUC: 0.841 AUC: 0.838 Z: 1.865 Rdelta: 0.02249
#> 186 Number of features: 88 Max AUC: 0.841 AUC: 0.838 Z: 1.867 Rdelta: 0.03024
#> 187 Number of features: 88 Max AUC: 0.841 AUC: 0.832 Z: 1.812 Rdelta: 0.02419
#> 188 Number of features: 88 Max AUC: 0.841 AUC: 0.831 Z: 1.704 Rdelta: 0.01935
#> 189 Number of features: 88 Max AUC: 0.841 AUC: 0.822 Z: 1.791 Rdelta: 0.01548
#> 190 Number of features: 89 Max AUC: 0.841 AUC: 0.836 Z: 1.824 Rdelta: 0.02394
#> 191 Number of features: 90 Max AUC: 0.841 AUC: 0.833 Z: 1.889 Rdelta: 0.03154
#> 192 Number of features: 91 Max AUC: 0.841 AUC: 0.836 Z: 1.868 Rdelta: 0.03839
#> 193 Number of features: 91 Max AUC: 0.841 AUC: 0.783 Z: 1.621 Rdelta: 0.03071
#> 194 Number of features: 92 Max AUC: 0.841 AUC: 0.837 Z: 1.856 Rdelta: 0.03764
#> 195 Number of features: 93 Max AUC: 0.841 AUC: 0.835 Z: 1.867 Rdelta: 0.04388
#> 196 Number of features: 94 Max AUC: 0.845 AUC: 0.845 Z: 1.890 Rdelta: 0.04949
#> 197 Number of features: 94 Max AUC: 0.845 AUC: 0.834 Z: 1.856 Rdelta: 0.03959
#> 198 Number of features: 94 Max AUC: 0.845 AUC: 0.789 Z: 1.556 Rdelta: 0.03167
#> 199 Number of features: 94 Max AUC: 0.845 AUC: 0.828 Z: 1.847 Rdelta: 0.02534
#> 200 Number of features: 95 Max AUC: 0.846 AUC: 0.846 Z: 1.859 Rdelta: 0.03280
#> 201 Number of features: 95 Max AUC: 0.846 AUC: 0.828 Z: 1.845 Rdelta: 0.02624
#> 202 Number of features: 95 Max AUC: 0.846 AUC: 0.837 Z: 1.840 Rdelta: 0.02099
#> 203 Number of features: 95 Max AUC: 0.846 AUC: 0.835 Z: 1.834 Rdelta: 0.01680
#> 204 Number of features: 95 Max AUC: 0.846 AUC: 0.829 Z: 1.808 Rdelta: 0.01344
#> 205 Number of features: 95 Max AUC: 0.846 AUC: 0.828 Z: 1.837 Rdelta: 0.01075
#> 206 Number of features: 96 Max AUC: 0.846 AUC: 0.837 Z: 1.863 Rdelta: 0.01967
#> 207 Number of features: 96 Max AUC: 0.846 AUC: 0.837 Z: 1.902 Rdelta: 0.01574
#> 208 Number of features: 96 Max AUC: 0.846 AUC: 0.831 Z: 1.828 Rdelta: 0.01259
#> 209 Number of features: 96 Max AUC: 0.846 AUC: 0.750 Z: 1.317 Rdelta: 0.01007
#> 210 Number of features: 96 Max AUC: 0.846 AUC: 0.827 Z: 1.843 Rdelta: 0.00806
#> 211 Number of features: 96 Max AUC: 0.846 AUC: 0.836 Z: 1.882 Rdelta: 0.00645
#> 212 Number of features: 97 Max AUC: 0.846 AUC: 0.838 Z: 1.818 Rdelta: 0.01580
#> 213 Number of features: 98 Max AUC: 0.846 AUC: 0.838 Z: 1.905 Rdelta: 0.02422
#> 214 Number of features: 98 Max AUC: 0.846 AUC: 0.831 Z: 1.831 Rdelta: 0.01938
#> 215 Number of features: 98 Max AUC: 0.846 AUC: 0.823 Z: 1.719 Rdelta: 0.01550
#> 216 Number of features: 98 Max AUC: 0.846 AUC: 0.813 Z: 1.730 Rdelta: 0.01240
#> 217 Number of features: 98 Max AUC: 0.846 AUC: 0.818 Z: 1.842 Rdelta: 0.00992
#> 218 Number of features: 98 Max AUC: 0.846 AUC: 0.805 Z: 1.801 Rdelta: 0.00794
#> 219 Number of features: 98 Max AUC: 0.846 AUC: 0.803 Z: 1.767 Rdelta: 0.00635
#> 220 Number of features: 98 Max AUC: 0.846 AUC: 0.833 Z: 1.851 Rdelta: 0.00508
#> 221 Number of features: 98 Max AUC: 0.846 AUC: 0.828 Z: 1.842 Rdelta: 0.00406
#> 222 Number of features: 99 Max AUC: 0.846 AUC: 0.843 Z: 1.834 Rdelta: 0.01366
#> 223 Number of features: 99 Max AUC: 0.846 AUC: 0.822 Z: 1.806 Rdelta: 0.01093
#> 224 Number of features: 99 Max AUC: 0.846 AUC: 0.836 Z: 1.866 Rdelta: 0.00874
#> 225 Number of features: 99 Max AUC: 0.846 AUC: 0.823 Z: 1.861 Rdelta: 0.00699
#> 226 Number of features: 99 Max AUC: 0.846 AUC: 0.824 Z: 1.878 Rdelta: 0.00559
#> 227 Number of features: 99 Max AUC: 0.846 AUC: 0.836 Z: 1.822 Rdelta: 0.00448
#> 228 Number of features: 99 Max AUC: 0.846 AUC: 0.829 Z: 1.795 Rdelta: 0.00358
#> 229 Number of features: 99 Max AUC: 0.846 AUC: 0.818 Z: 1.730 Rdelta: 0.00286
#> 230 Number of features: 99 Max AUC: 0.846 AUC: 0.806 Z: 1.720 Rdelta: 0.00229
#> 231 Number of features: 100 Max AUC: 0.846 AUC: 0.840 Z: 1.840 Rdelta: 0.01206
#> 232 Number of features: 100 Max AUC: 0.846 AUC: 0.831 Z: 1.872 Rdelta: 0.00965
#> 233 Number of features: 100 Max AUC: 0.846 AUC: 0.745 Z: 1.394 Rdelta: 0.00772
#> 234 Number of features: 100 Max AUC: 0.846 AUC: 0.828 Z: 1.869 Rdelta: 0.00618
#> 235 Number of features: 100 Max AUC: 0.846 AUC: 0.801 Z: 1.732 Rdelta: 0.00494
#> 236 Number of features: 100 Max AUC: 0.846 AUC: 0.733 Z: 1.346 Rdelta: 0.00395
#> 237 Number of features: 100 Max AUC: 0.846 AUC: 0.836 Z: 1.923 Rdelta: 0.00316
#> 238 Number of features: 100 Max AUC: 0.846 AUC: 0.836 Z: 1.889 Rdelta: 0.00253
#> 239 Number of features: 101 Max AUC: 0.846 AUC: 0.841 Z: 1.872 Rdelta: 0.01228
#> 240 Number of features: 101 Max AUC: 0.846 AUC: 0.821 Z: 1.777 Rdelta: 0.00982
#> 241 Number of features: 101 Max AUC: 0.846 AUC: 0.834 Z: 1.874 Rdelta: 0.00786
#> 242 Number of features: 101 Max AUC: 0.846 AUC: 0.830 Z: 1.852 Rdelta: 0.00629
#> 243 Number of features: 101 Max AUC: 0.846 AUC: 0.834 Z: 1.844 Rdelta: 0.00503
#> 244 Number of features: 102 Max AUC: 0.846 AUC: 0.838 Z: 1.867 Rdelta: 0.01453
#> 245 Number of features: 102 Max AUC: 0.846 AUC: 0.833 Z: 1.855 Rdelta: 0.01162
#> 246 Number of features: 102 Max AUC: 0.846 AUC: 0.800 Z: 1.749 Rdelta: 0.00930
#> 247 Number of features: 102 Max AUC: 0.846 AUC: 0.836 Z: 1.876 Rdelta: 0.00744
#> 248 Number of features: 102 Max AUC: 0.846 AUC: 0.835 Z: 1.870 Rdelta: 0.00595
#> 249 Number of features: 102 Max AUC: 0.846 AUC: 0.835 Z: 1.849 Rdelta: 0.00476
#> 250 Number of features: 102 Max AUC: 0.846 AUC: 0.750 Z: 1.503 Rdelta: 0.00381
#> 251 Number of features: 102 Max AUC: 0.846 AUC: 0.832 Z: 1.802 Rdelta: 0.00305
#> 252 Number of features: 102 Max AUC: 0.846 AUC: 0.820 Z: 1.837 Rdelta: 0.00244
#> 253 Number of features: 102 Max AUC: 0.846 AUC: 0.797 Z: 1.812 Rdelta: 0.00195
#> 254 Number of features: 102 Max AUC: 0.846 AUC: 0.823 Z: 1.877 Rdelta: 0.00156
#> 255 Number of features: 102 Max AUC: 0.846 AUC: 0.774 Z: 1.534 Rdelta: 0.00125
#> 256 Number of features: 102 Max AUC: 0.846 AUC: 0.827 Z: 1.895 Rdelta: 0.00100
#> 257 Number of features: 102 Max AUC: 0.846 AUC: 0.816 Z: 1.627 Rdelta: 0.00080
#> 258 Number of features: 102 Max AUC: 0.846 AUC: 0.708 Z: 1.344 Rdelta: 0.00064
#> 259 Number of features: 102 Max AUC: 0.846 AUC: 0.829 Z: 1.822 Rdelta: 0.00051
#> 260 Number of features: 102 Max AUC: 0.846 AUC: 0.771 Z: 1.563 Rdelta: 0.00041
#> 261 Number of features: 102 Max AUC: 0.846 AUC: 0.748 Z: 1.294 Rdelta: 0.00033
#> 262 Number of features: 102 Max AUC: 0.846 AUC: 0.833 Z: 1.841 Rdelta: 0.00026
#> 263 Number of features: 102 Max AUC: 0.846 AUC: 0.831 Z: 1.851 Rdelta: 0.00021
#> 264 Number of features: 102 Max AUC: 0.846 AUC: 0.826 Z: 1.863 Rdelta: 0.00017
#> 265 Number of features: 103 Max AUC: 0.846 AUC: 0.840 Z: 1.850 Rdelta: 0.01015
#> 266 Number of features: 103 Max AUC: 0.846 AUC: 0.827 Z: 1.814 Rdelta: 0.00812
#> 267 Number of features: 103 Max AUC: 0.846 AUC: 0.830 Z: 1.848 Rdelta: 0.00650
#> 268 Number of features: 103 Max AUC: 0.846 AUC: 0.824 Z: 1.775 Rdelta: 0.00520
#> 269 Number of features: 103 Max AUC: 0.846 AUC: 0.820 Z: 1.772 Rdelta: 0.00416
#> 270 Number of features: 103 Max AUC: 0.846 AUC: 0.821 Z: 1.794 Rdelta: 0.00333
#> 271 Number of features: 103 Max AUC: 0.846 AUC: 0.827 Z: 1.730 Rdelta: 0.00266
#> 272 Number of features: 103 Max AUC: 0.846 AUC: 0.825 Z: 1.873 Rdelta: 0.00213
#> 273 Number of features: 104 Max AUC: 0.846 AUC: 0.840 Z: 1.880 Rdelta: 0.01192
#> 274 Number of features: 104 Max AUC: 0.846 AUC: 0.768 Z: 1.467 Rdelta: 0.00953
#> 275 Number of features: 104 Max AUC: 0.846 AUC: 0.761 Z: 1.482 Rdelta: 0.00763
#> 276 Number of features: 104 Max AUC: 0.846 AUC: 0.766 Z: 1.495 Rdelta: 0.00610
#> 277 Number of features: 104 Max AUC: 0.846 AUC: 0.784 Z: 1.503 Rdelta: 0.00488
#> 278 Number of features: 104 Max AUC: 0.846 AUC: 0.800 Z: 1.646 Rdelta: 0.00390
#> 279 Number of features: 104 Max AUC: 0.846 AUC: 0.768 Z: 1.572 Rdelta: 0.00312
#> 280 Number of features: 104 Max AUC: 0.846 AUC: 0.836 Z: 1.861 Rdelta: 0.00250
#> 281 Number of features: 104 Max AUC: 0.846 AUC: 0.748 Z: 1.013 Rdelta: 0.00200
#> 282 Number of features: 104 Max AUC: 0.846 AUC: 0.836 Z: 1.900 Rdelta: 0.00160
#> 283 Number of features: 104 Max AUC: 0.846 AUC: 0.819 Z: 1.850 Rdelta: 0.00128
#> 284 Number of features: 104 Max AUC: 0.846 AUC: 0.798 Z: 1.671 Rdelta: 0.00102
#> 285 Number of features: 104 Max AUC: 0.846 AUC: 0.828 Z: 1.831 Rdelta: 0.00082
#> 286 Number of features: 104 Max AUC: 0.846 AUC: 0.778 Z: 1.532 Rdelta: 0.00066
#> 287 Number of features: 104 Max AUC: 0.846 AUC: 0.796 Z: 1.603 Rdelta: 0.00052
#> 288 Number of features: 104 Max AUC: 0.846 AUC: 0.812 Z: 1.685 Rdelta: 0.00042
#> 289 Number of features: 104 Max AUC: 0.846 AUC: 0.739 Z: 1.351 Rdelta: 0.00034
#> 290 Number of features: 104 Max AUC: 0.846 AUC: 0.818 Z: 1.798 Rdelta: 0.00027
#> 291 Number of features: 104 Max AUC: 0.846 AUC: 0.831 Z: 1.720 Rdelta: 0.00021
#> 292 Number of features: 104 Max AUC: 0.846 AUC: 0.796 Z: 1.669 Rdelta: 0.00017
#> 293 Number of features: 104 Max AUC: 0.846 AUC: 0.812 Z: 1.763 Rdelta: 0.00014
#> 294 Number of features: 104 Max AUC: 0.846 AUC: 0.706 Z: 0.827 Rdelta: 0.00011
#> 295 Number of features: 104 Max AUC: 0.846 AUC: 0.823 Z: 1.790 Rdelta: 0.00009
#> user system elapsed
#> 246.84 0.00 246.99
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"pearson")+signatureDistance(signature$controlTemplate,validLabeled,"pearson")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS"))
#> 7 Number of features: 7 Max AUC: 0.761 AUC: 0.761 Z: 1.054 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.768 AUC: 0.768 Z: 0.700 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.768 AUC: 0.755 Z: 0.661 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.772 AUC: 0.772 Z: 0.599 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.772 AUC: 0.765 Z: 0.779 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.772 AUC: 0.761 Z: 0.642 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.772 AUC: 0.763 Z: 0.361 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.772 AUC: 0.769 Z: 0.351 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.772 AUC: 0.767 Z: 0.422 Rdelta: 0.10000
#> 16 Number of features: 15 Max AUC: 0.772 AUC: 0.748 Z: 0.289 Rdelta: 0.08000
#> 17 Number of features: 16 Max AUC: 0.772 AUC: 0.759 Z: 0.507 Rdelta: 0.08200
#> 18 Number of features: 17 Max AUC: 0.781 AUC: 0.781 Z: 0.346 Rdelta: 0.08380
#> 19 Number of features: 17 Max AUC: 0.781 AUC: 0.760 Z: 0.437 Rdelta: 0.06704
#> 20 Number of features: 18 Max AUC: 0.781 AUC: 0.779 Z: 0.467 Rdelta: 0.07034
#> 21 Number of features: 18 Max AUC: 0.781 AUC: 0.767 Z: 0.325 Rdelta: 0.05627
#> 22 Number of features: 19 Max AUC: 0.781 AUC: 0.779 Z: 0.382 Rdelta: 0.06064
#> 23 Number of features: 20 Max AUC: 0.781 AUC: 0.777 Z: 0.286 Rdelta: 0.06458
#> 24 Number of features: 21 Max AUC: 0.783 AUC: 0.783 Z: 0.442 Rdelta: 0.06812
#> 25 Number of features: 22 Max AUC: 0.783 AUC: 0.783 Z: 0.426 Rdelta: 0.07131
#> 26 Number of features: 23 Max AUC: 0.794 AUC: 0.794 Z: 0.506 Rdelta: 0.07418
#> 27 Number of features: 24 Max AUC: 0.794 AUC: 0.789 Z: 0.470 Rdelta: 0.07676
#> 28 Number of features: 25 Max AUC: 0.803 AUC: 0.803 Z: 0.470 Rdelta: 0.07908
#> 29 Number of features: 26 Max AUC: 0.803 AUC: 0.801 Z: 0.459 Rdelta: 0.08118
#> 30 Number of features: 26 Max AUC: 0.803 AUC: 0.781 Z: 0.355 Rdelta: 0.06494
#> 31 Number of features: 27 Max AUC: 0.803 AUC: 0.802 Z: 0.290 Rdelta: 0.06845
#> 32 Number of features: 27 Max AUC: 0.803 AUC: 0.787 Z: 0.432 Rdelta: 0.05476
#> 33 Number of features: 27 Max AUC: 0.803 AUC: 0.784 Z: 0.246 Rdelta: 0.04381
#> 34 Number of features: 27 Max AUC: 0.803 AUC: 0.748 Z: 0.503 Rdelta: 0.03504
#> 35 Number of features: 27 Max AUC: 0.803 AUC: 0.784 Z: 0.308 Rdelta: 0.02804
#> 36 Number of features: 27 Max AUC: 0.803 AUC: 0.794 Z: 0.187 Rdelta: 0.02243
#> 37 Number of features: 27 Max AUC: 0.803 AUC: 0.788 Z: 0.357 Rdelta: 0.01794
#> 38 Number of features: 27 Max AUC: 0.803 AUC: 0.782 Z: 0.383 Rdelta: 0.01435
#> 39 Number of features: 27 Max AUC: 0.803 AUC: 0.777 Z: 0.258 Rdelta: 0.01148
#> 40 Number of features: 27 Max AUC: 0.803 AUC: 0.735 Z: 0.237 Rdelta: 0.00919
#> 41 Number of features: 27 Max AUC: 0.803 AUC: 0.764 Z: 0.426 Rdelta: 0.00735
#> 42 Number of features: 28 Max AUC: 0.803 AUC: 0.798 Z: 0.407 Rdelta: 0.01661
#> 43 Number of features: 29 Max AUC: 0.803 AUC: 0.800 Z: 0.377 Rdelta: 0.02495
#> 44 Number of features: 30 Max AUC: 0.812 AUC: 0.812 Z: 0.359 Rdelta: 0.03246
#> 45 Number of features: 30 Max AUC: 0.812 AUC: 0.801 Z: 0.327 Rdelta: 0.02597
#> 46 Number of features: 30 Max AUC: 0.812 AUC: 0.743 Z: 0.313 Rdelta: 0.02077
#> 47 Number of features: 30 Max AUC: 0.812 AUC: 0.784 Z: 0.470 Rdelta: 0.01662
#> 48 Number of features: 30 Max AUC: 0.812 AUC: 0.795 Z: 0.335 Rdelta: 0.01329
#> 49 Number of features: 30 Max AUC: 0.812 AUC: 0.791 Z: 0.552 Rdelta: 0.01064
#> 50 Number of features: 30 Max AUC: 0.812 AUC: 0.792 Z: 0.362 Rdelta: 0.00851
#> 51 Number of features: 31 Max AUC: 0.812 AUC: 0.807 Z: 0.502 Rdelta: 0.01766
#> 52 Number of features: 32 Max AUC: 0.812 AUC: 0.811 Z: 0.361 Rdelta: 0.02589
#> 53 Number of features: 33 Max AUC: 0.812 AUC: 0.800 Z: 0.639 Rdelta: 0.03330
#> 54 Number of features: 34 Max AUC: 0.812 AUC: 0.812 Z: 0.403 Rdelta: 0.03997
#> 55 Number of features: 34 Max AUC: 0.812 AUC: 0.788 Z: 0.451 Rdelta: 0.03198
#> 56 Number of features: 35 Max AUC: 0.815 AUC: 0.815 Z: 0.598 Rdelta: 0.03878
#> 57 Number of features: 35 Max AUC: 0.815 AUC: 0.800 Z: 0.423 Rdelta: 0.03102
#> 58 Number of features: 35 Max AUC: 0.815 AUC: 0.798 Z: 0.421 Rdelta: 0.02482
#> 59 Number of features: 35 Max AUC: 0.815 AUC: 0.788 Z: 0.460 Rdelta: 0.01986
#> 60 Number of features: 35 Max AUC: 0.815 AUC: 0.789 Z: 0.361 Rdelta: 0.01588
#> 61 Number of features: 35 Max AUC: 0.815 AUC: 0.758 Z: 0.511 Rdelta: 0.01271
#> 62 Number of features: 35 Max AUC: 0.815 AUC: 0.770 Z: 0.600 Rdelta: 0.01017
#> 63 Number of features: 36 Max AUC: 0.815 AUC: 0.809 Z: 0.345 Rdelta: 0.01915
#> 64 Number of features: 36 Max AUC: 0.815 AUC: 0.791 Z: 0.485 Rdelta: 0.01532
#> 65 Number of features: 36 Max AUC: 0.815 AUC: 0.793 Z: 0.648 Rdelta: 0.01226
#> 66 Number of features: 36 Max AUC: 0.815 AUC: 0.776 Z: 0.485 Rdelta: 0.00980
#> 67 Number of features: 36 Max AUC: 0.815 AUC: 0.746 Z: 0.344 Rdelta: 0.00784
#> 68 Number of features: 37 Max AUC: 0.815 AUC: 0.811 Z: 0.404 Rdelta: 0.01706
#> 69 Number of features: 37 Max AUC: 0.815 AUC: 0.786 Z: 0.189 Rdelta: 0.01365
#> 70 Number of features: 37 Max AUC: 0.815 AUC: 0.795 Z: 0.321 Rdelta: 0.01092
#> 71 Number of features: 37 Max AUC: 0.815 AUC: 0.792 Z: 0.377 Rdelta: 0.00873
#> 72 Number of features: 37 Max AUC: 0.815 AUC: 0.786 Z: 0.365 Rdelta: 0.00699
#> 73 Number of features: 37 Max AUC: 0.815 AUC: 0.772 Z: 0.689 Rdelta: 0.00559
#> 74 Number of features: 37 Max AUC: 0.815 AUC: 0.765 Z: 0.451 Rdelta: 0.00447
#> 75 Number of features: 37 Max AUC: 0.815 AUC: 0.785 Z: 0.525 Rdelta: 0.00358
#> 76 Number of features: 37 Max AUC: 0.815 AUC: 0.802 Z: 0.517 Rdelta: 0.00286
#> 77 Number of features: 37 Max AUC: 0.815 AUC: 0.777 Z: 0.565 Rdelta: 0.00229
#> 78 Number of features: 37 Max AUC: 0.815 AUC: 0.794 Z: 0.408 Rdelta: 0.00183
#> 79 Number of features: 37 Max AUC: 0.815 AUC: 0.796 Z: 0.481 Rdelta: 0.00147
#> 80 Number of features: 38 Max AUC: 0.815 AUC: 0.812 Z: 0.336 Rdelta: 0.01132
#> 81 Number of features: 39 Max AUC: 0.821 AUC: 0.821 Z: 0.356 Rdelta: 0.02019
#> 82 Number of features: 39 Max AUC: 0.821 AUC: 0.807 Z: 0.425 Rdelta: 0.01615
#> 83 Number of features: 39 Max AUC: 0.821 AUC: 0.800 Z: 0.242 Rdelta: 0.01292
#> 84 Number of features: 39 Max AUC: 0.821 AUC: 0.806 Z: 0.358 Rdelta: 0.01034
#> 85 Number of features: 39 Max AUC: 0.821 AUC: 0.796 Z: 0.529 Rdelta: 0.00827
#> 86 Number of features: 39 Max AUC: 0.821 AUC: 0.803 Z: 0.521 Rdelta: 0.00661
#> 87 Number of features: 39 Max AUC: 0.821 AUC: 0.800 Z: 0.317 Rdelta: 0.00529
#> 88 Number of features: 39 Max AUC: 0.821 AUC: 0.786 Z: 0.397 Rdelta: 0.00423
#> 89 Number of features: 39 Max AUC: 0.821 AUC: 0.788 Z: 0.533 Rdelta: 0.00339
#> 90 Number of features: 39 Max AUC: 0.821 AUC: 0.795 Z: 0.480 Rdelta: 0.00271
#> 91 Number of features: 39 Max AUC: 0.821 AUC: 0.793 Z: 0.520 Rdelta: 0.00217
#> 92 Number of features: 39 Max AUC: 0.821 AUC: 0.776 Z: 0.354 Rdelta: 0.00173
#> 93 Number of features: 39 Max AUC: 0.821 AUC: 0.803 Z: 0.264 Rdelta: 0.00139
#> 94 Number of features: 40 Max AUC: 0.821 AUC: 0.811 Z: 0.489 Rdelta: 0.01125
#> 95 Number of features: 41 Max AUC: 0.828 AUC: 0.828 Z: 0.406 Rdelta: 0.02012
#> 96 Number of features: 41 Max AUC: 0.828 AUC: 0.800 Z: 0.429 Rdelta: 0.01610
#> 97 Number of features: 41 Max AUC: 0.828 AUC: 0.809 Z: 0.371 Rdelta: 0.01288
#> 98 Number of features: 42 Max AUC: 0.834 AUC: 0.834 Z: 0.358 Rdelta: 0.02159
#> 99 Number of features: 42 Max AUC: 0.834 AUC: 0.782 Z: 0.402 Rdelta: 0.01727
#> 100 Number of features: 42 Max AUC: 0.834 AUC: 0.801 Z: 0.244 Rdelta: 0.01382
#> 101 Number of features: 43 Max AUC: 0.834 AUC: 0.825 Z: 0.497 Rdelta: 0.02244
#> 102 Number of features: 44 Max AUC: 0.837 AUC: 0.837 Z: 0.389 Rdelta: 0.03019
#> 103 Number of features: 45 Max AUC: 0.837 AUC: 0.830 Z: 0.492 Rdelta: 0.03717
#> 104 Number of features: 45 Max AUC: 0.837 AUC: 0.821 Z: 0.464 Rdelta: 0.02974
#> 105 Number of features: 45 Max AUC: 0.837 AUC: 0.819 Z: 0.461 Rdelta: 0.02379
#> 106 Number of features: 45 Max AUC: 0.837 AUC: 0.814 Z: 0.445 Rdelta: 0.01903
#> 107 Number of features: 45 Max AUC: 0.837 AUC: 0.826 Z: 0.381 Rdelta: 0.01523
#> 108 Number of features: 46 Max AUC: 0.837 AUC: 0.828 Z: 0.511 Rdelta: 0.02370
#> 109 Number of features: 47 Max AUC: 0.840 AUC: 0.840 Z: 0.458 Rdelta: 0.03133
#> 110 Number of features: 48 Max AUC: 0.840 AUC: 0.840 Z: 0.386 Rdelta: 0.03820
#> 111 Number of features: 49 Max AUC: 0.840 AUC: 0.832 Z: 0.580 Rdelta: 0.04438
#> 112 Number of features: 49 Max AUC: 0.840 AUC: 0.830 Z: 0.406 Rdelta: 0.03550
#> 113 Number of features: 49 Max AUC: 0.840 AUC: 0.805 Z: 0.425 Rdelta: 0.02840
#> 114 Number of features: 49 Max AUC: 0.840 AUC: 0.824 Z: 0.369 Rdelta: 0.02272
#> 115 Number of features: 50 Max AUC: 0.844 AUC: 0.844 Z: 0.218 Rdelta: 0.03045
#> 116 Number of features: 51 Max AUC: 0.844 AUC: 0.836 Z: 0.509 Rdelta: 0.03741
#> 117 Number of features: 51 Max AUC: 0.844 AUC: 0.809 Z: 0.477 Rdelta: 0.02992
#> 118 Number of features: 51 Max AUC: 0.844 AUC: 0.803 Z: 0.489 Rdelta: 0.02394
#> 119 Number of features: 51 Max AUC: 0.844 AUC: 0.819 Z: 0.395 Rdelta: 0.01915
#> 120 Number of features: 51 Max AUC: 0.844 AUC: 0.811 Z: 0.370 Rdelta: 0.01532
#> 121 Number of features: 51 Max AUC: 0.844 AUC: 0.820 Z: 0.422 Rdelta: 0.01226
#> 122 Number of features: 52 Max AUC: 0.846 AUC: 0.846 Z: 0.377 Rdelta: 0.02103
#> 123 Number of features: 53 Max AUC: 0.846 AUC: 0.836 Z: 0.498 Rdelta: 0.02893
#> 124 Number of features: 53 Max AUC: 0.846 AUC: 0.835 Z: 0.415 Rdelta: 0.02314
#> 125 Number of features: 53 Max AUC: 0.846 AUC: 0.824 Z: 0.498 Rdelta: 0.01851
#> 126 Number of features: 53 Max AUC: 0.846 AUC: 0.831 Z: 0.514 Rdelta: 0.01481
#> 127 Number of features: 53 Max AUC: 0.846 AUC: 0.748 Z: 0.311 Rdelta: 0.01185
#> 128 Number of features: 53 Max AUC: 0.846 AUC: 0.825 Z: 0.394 Rdelta: 0.00948
#> 129 Number of features: 53 Max AUC: 0.846 AUC: 0.821 Z: 0.526 Rdelta: 0.00758
#> 130 Number of features: 53 Max AUC: 0.846 AUC: 0.754 Z: 0.414 Rdelta: 0.00607
#> 131 Number of features: 54 Max AUC: 0.852 AUC: 0.852 Z: 0.547 Rdelta: 0.01546
#> 132 Number of features: 54 Max AUC: 0.852 AUC: 0.822 Z: 0.282 Rdelta: 0.01237
#> 133 Number of features: 54 Max AUC: 0.852 AUC: 0.817 Z: 0.600 Rdelta: 0.00989
#> 134 Number of features: 54 Max AUC: 0.852 AUC: 0.809 Z: 0.450 Rdelta: 0.00792
#> 135 Number of features: 54 Max AUC: 0.852 AUC: 0.841 Z: 0.446 Rdelta: 0.00633
#> 136 Number of features: 54 Max AUC: 0.852 AUC: 0.773 Z: 0.390 Rdelta: 0.00507
#> 137 Number of features: 55 Max AUC: 0.852 AUC: 0.846 Z: 0.392 Rdelta: 0.01456
#> 138 Number of features: 55 Max AUC: 0.852 AUC: 0.784 Z: 0.055 Rdelta: 0.01165
#> 139 Number of features: 55 Max AUC: 0.852 AUC: 0.807 Z: 0.251 Rdelta: 0.00932
#> 140 Number of features: 55 Max AUC: 0.852 AUC: 0.806 Z: 0.279 Rdelta: 0.00745
#> 141 Number of features: 55 Max AUC: 0.852 AUC: 0.824 Z: 0.240 Rdelta: 0.00596
#> 142 Number of features: 55 Max AUC: 0.852 AUC: 0.810 Z: 0.471 Rdelta: 0.00477
#> 143 Number of features: 55 Max AUC: 0.852 AUC: 0.827 Z: 0.377 Rdelta: 0.00382
#> 144 Number of features: 55 Max AUC: 0.852 AUC: 0.821 Z: 0.138 Rdelta: 0.00305
#> 145 Number of features: 55 Max AUC: 0.852 AUC: 0.836 Z: 0.214 Rdelta: 0.00244
#> 146 Number of features: 55 Max AUC: 0.852 AUC: 0.826 Z: 0.525 Rdelta: 0.00195
#> 147 Number of features: 55 Max AUC: 0.852 AUC: 0.830 Z: 0.183 Rdelta: 0.00156
#> 148 Number of features: 55 Max AUC: 0.852 AUC: 0.791 Z: 0.351 Rdelta: 0.00125
#> 149 Number of features: 55 Max AUC: 0.852 AUC: 0.840 Z: 0.422 Rdelta: 0.00100
#> 150 Number of features: 55 Max AUC: 0.852 AUC: 0.805 Z: 0.348 Rdelta: 0.00080
#> 151 Number of features: 55 Max AUC: 0.852 AUC: 0.825 Z: 0.486 Rdelta: 0.00064
#> 152 Number of features: 55 Max AUC: 0.852 AUC: 0.829 Z: 0.483 Rdelta: 0.00051
#> 153 Number of features: 55 Max AUC: 0.852 AUC: 0.822 Z: 0.430 Rdelta: 0.00041
#> 154 Number of features: 55 Max AUC: 0.852 AUC: 0.813 Z: 0.423 Rdelta: 0.00033
#> 155 Number of features: 55 Max AUC: 0.852 AUC: 0.841 Z: 0.420 Rdelta: 0.00026
#> 156 Number of features: 55 Max AUC: 0.852 AUC: 0.831 Z: 0.460 Rdelta: 0.00021
#> 157 Number of features: 55 Max AUC: 0.852 AUC: 0.818 Z: 0.224 Rdelta: 0.00017
#> 158 Number of features: 55 Max AUC: 0.852 AUC: 0.801 Z: 0.376 Rdelta: 0.00013
#> 159 Number of features: 55 Max AUC: 0.852 AUC: 0.812 Z: 0.423 Rdelta: 0.00011
#> 160 Number of features: 55 Max AUC: 0.852 AUC: 0.831 Z: 0.475 Rdelta: 0.00009
#> user system elapsed
#> 80.68 0.00 80.75
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
testDistance_cotrol <- signatureDistance(signature$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_cotrol-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Case"))
#> 7 Number of features: 7 Max AUC: 0.859 AUC: 0.859 Z: 1.076 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.892 AUC: 0.892 Z: 0.542 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.910 AUC: 0.910 Z: 0.651 Rdelta: 0.10000
#> 10 Number of features: 10 Max AUC: 0.914 AUC: 0.914 Z: 0.710 Rdelta: 0.10000
#> 11 Number of features: 11 Max AUC: 0.914 AUC: 0.908 Z: 0.711 Rdelta: 0.10000
#> 12 Number of features: 12 Max AUC: 0.914 AUC: 0.905 Z: 0.699 Rdelta: 0.10000
#> 13 Number of features: 13 Max AUC: 0.914 AUC: 0.909 Z: 0.675 Rdelta: 0.10000
#> 14 Number of features: 14 Max AUC: 0.914 AUC: 0.906 Z: 0.610 Rdelta: 0.10000
#> 15 Number of features: 15 Max AUC: 0.914 AUC: 0.905 Z: 0.543 Rdelta: 0.10000
#> 16 Number of features: 16 Max AUC: 0.914 AUC: 0.906 Z: 0.525 Rdelta: 0.10000
#> 17 Number of features: 17 Max AUC: 0.914 AUC: 0.906 Z: 0.579 Rdelta: 0.10000
#> 18 Number of features: 18 Max AUC: 0.917 AUC: 0.917 Z: 0.648 Rdelta: 0.10000
#> 19 Number of features: 19 Max AUC: 0.932 AUC: 0.932 Z: 0.576 Rdelta: 0.10000
#> 20 Number of features: 20 Max AUC: 0.932 AUC: 0.926 Z: 0.516 Rdelta: 0.10000
#> 21 Number of features: 21 Max AUC: 0.942 AUC: 0.942 Z: 0.383 Rdelta: 0.10000
#> 22 Number of features: 22 Max AUC: 0.942 AUC: 0.933 Z: 0.531 Rdelta: 0.10000
#> 23 Number of features: 23 Max AUC: 0.942 AUC: 0.933 Z: 0.422 Rdelta: 0.10000
#> 24 Number of features: 24 Max AUC: 0.942 AUC: 0.934 Z: 0.414 Rdelta: 0.10000
#> 25 Number of features: 25 Max AUC: 0.942 AUC: 0.933 Z: 0.484 Rdelta: 0.10000
#> 26 Number of features: 26 Max AUC: 0.942 AUC: 0.932 Z: 0.377 Rdelta: 0.10000
#> 27 Number of features: 27 Max AUC: 0.942 AUC: 0.940 Z: 0.441 Rdelta: 0.10000
#> 28 Number of features: 28 Max AUC: 0.942 AUC: 0.940 Z: 0.536 Rdelta: 0.10000
#> 29 Number of features: 29 Max AUC: 0.942 AUC: 0.934 Z: 0.346 Rdelta: 0.10000
#> 30 Number of features: 30 Max AUC: 0.942 AUC: 0.940 Z: 0.368 Rdelta: 0.10000
#> 31 Number of features: 31 Max AUC: 0.942 AUC: 0.940 Z: 0.343 Rdelta: 0.10000
#> 32 Number of features: 32 Max AUC: 0.951 AUC: 0.951 Z: 0.371 Rdelta: 0.10000
#> 33 Number of features: 33 Max AUC: 0.951 AUC: 0.947 Z: 0.486 Rdelta: 0.10000
#> 34 Number of features: 33 Max AUC: 0.951 AUC: 0.932 Z: 0.414 Rdelta: 0.08000
#> 35 Number of features: 34 Max AUC: 0.951 AUC: 0.944 Z: 0.532 Rdelta: 0.08200
#> 36 Number of features: 35 Max AUC: 0.951 AUC: 0.945 Z: 0.230 Rdelta: 0.08380
#> 37 Number of features: 36 Max AUC: 0.951 AUC: 0.943 Z: 0.381 Rdelta: 0.08542
#> 38 Number of features: 36 Max AUC: 0.951 AUC: 0.937 Z: 0.357 Rdelta: 0.06834
#> 39 Number of features: 37 Max AUC: 0.951 AUC: 0.944 Z: 0.395 Rdelta: 0.07150
#> 40 Number of features: 38 Max AUC: 0.951 AUC: 0.948 Z: 0.461 Rdelta: 0.07435
#> 41 Number of features: 38 Max AUC: 0.951 AUC: 0.934 Z: 0.246 Rdelta: 0.05948
#> 42 Number of features: 39 Max AUC: 0.951 AUC: 0.946 Z: 0.434 Rdelta: 0.06353
#> 43 Number of features: 40 Max AUC: 0.951 AUC: 0.947 Z: 0.293 Rdelta: 0.06718
#> 44 Number of features: 41 Max AUC: 0.951 AUC: 0.949 Z: 0.489 Rdelta: 0.07046
#> 45 Number of features: 42 Max AUC: 0.951 AUC: 0.948 Z: 0.362 Rdelta: 0.07342
#> 46 Number of features: 43 Max AUC: 0.951 AUC: 0.945 Z: 0.432 Rdelta: 0.07607
#> 47 Number of features: 44 Max AUC: 0.951 AUC: 0.946 Z: 0.587 Rdelta: 0.07847
#> 48 Number of features: 45 Max AUC: 0.951 AUC: 0.940 Z: 0.436 Rdelta: 0.08062
#> 49 Number of features: 46 Max AUC: 0.951 AUC: 0.945 Z: 0.494 Rdelta: 0.08256
#> 50 Number of features: 47 Max AUC: 0.951 AUC: 0.945 Z: 0.534 Rdelta: 0.08430
#> 51 Number of features: 48 Max AUC: 0.951 AUC: 0.940 Z: 0.547 Rdelta: 0.08587
#> 52 Number of features: 49 Max AUC: 0.951 AUC: 0.941 Z: 0.379 Rdelta: 0.08728
#> 53 Number of features: 50 Max AUC: 0.951 AUC: 0.947 Z: 0.475 Rdelta: 0.08856
#> 54 Number of features: 51 Max AUC: 0.956 AUC: 0.956 Z: 0.546 Rdelta: 0.08970
#> 55 Number of features: 52 Max AUC: 0.956 AUC: 0.950 Z: 0.676 Rdelta: 0.09073
#> 56 Number of features: 53 Max AUC: 0.956 AUC: 0.948 Z: 0.426 Rdelta: 0.09166
#> 57 Number of features: 54 Max AUC: 0.956 AUC: 0.956 Z: 0.681 Rdelta: 0.09249
#> 58 Number of features: 54 Max AUC: 0.956 AUC: 0.925 Z: 0.357 Rdelta: 0.07399
#> 59 Number of features: 54 Max AUC: 0.956 AUC: 0.939 Z: 0.526 Rdelta: 0.05919
#> 60 Number of features: 54 Max AUC: 0.956 AUC: 0.939 Z: 0.441 Rdelta: 0.04736
#> 61 Number of features: 54 Max AUC: 0.956 AUC: 0.906 Z: 0.404 Rdelta: 0.03788
#> 62 Number of features: 54 Max AUC: 0.956 AUC: 0.937 Z: 0.492 Rdelta: 0.03031
#> 63 Number of features: 54 Max AUC: 0.956 AUC: 0.938 Z: 0.492 Rdelta: 0.02425
#> 64 Number of features: 54 Max AUC: 0.956 AUC: 0.936 Z: 0.569 Rdelta: 0.01940
#> 65 Number of features: 54 Max AUC: 0.956 AUC: 0.929 Z: 0.479 Rdelta: 0.01552
#> 66 Number of features: 54 Max AUC: 0.956 AUC: 0.925 Z: 0.397 Rdelta: 0.01241
#> 67 Number of features: 54 Max AUC: 0.956 AUC: 0.939 Z: 0.525 Rdelta: 0.00993
#> 68 Number of features: 54 Max AUC: 0.956 AUC: 0.940 Z: 0.509 Rdelta: 0.00794
#> 69 Number of features: 55 Max AUC: 0.956 AUC: 0.948 Z: 0.579 Rdelta: 0.01715
#> 70 Number of features: 55 Max AUC: 0.956 AUC: 0.943 Z: 0.479 Rdelta: 0.01372
#> 71 Number of features: 55 Max AUC: 0.956 AUC: 0.928 Z: 0.444 Rdelta: 0.01098
#> 72 Number of features: 55 Max AUC: 0.956 AUC: 0.925 Z: 0.396 Rdelta: 0.00878
#> 73 Number of features: 55 Max AUC: 0.956 AUC: 0.931 Z: 0.427 Rdelta: 0.00702
#> 74 Number of features: 55 Max AUC: 0.956 AUC: 0.945 Z: 0.741 Rdelta: 0.00562
#> 75 Number of features: 55 Max AUC: 0.956 AUC: 0.934 Z: 0.473 Rdelta: 0.00450
#> 76 Number of features: 55 Max AUC: 0.956 AUC: 0.941 Z: 0.536 Rdelta: 0.00360
#> 77 Number of features: 55 Max AUC: 0.956 AUC: 0.946 Z: 0.513 Rdelta: 0.00288
#> 78 Number of features: 55 Max AUC: 0.956 AUC: 0.943 Z: 0.654 Rdelta: 0.00230
#> 79 Number of features: 55 Max AUC: 0.956 AUC: 0.939 Z: 0.617 Rdelta: 0.00184
#> 80 Number of features: 55 Max AUC: 0.956 AUC: 0.933 Z: 0.427 Rdelta: 0.00147
#> 81 Number of features: 55 Max AUC: 0.956 AUC: 0.937 Z: 0.426 Rdelta: 0.00118
#> 82 Number of features: 55 Max AUC: 0.956 AUC: 0.943 Z: 0.480 Rdelta: 0.00094
#> 83 Number of features: 55 Max AUC: 0.956 AUC: 0.941 Z: 0.600 Rdelta: 0.00075
#> 84 Number of features: 55 Max AUC: 0.956 AUC: 0.933 Z: 0.456 Rdelta: 0.00060
#> 85 Number of features: 55 Max AUC: 0.956 AUC: 0.944 Z: 0.668 Rdelta: 0.00048
#> 86 Number of features: 55 Max AUC: 0.956 AUC: 0.932 Z: 0.438 Rdelta: 0.00039
#> 87 Number of features: 55 Max AUC: 0.956 AUC: 0.943 Z: 0.491 Rdelta: 0.00031
#> 88 Number of features: 55 Max AUC: 0.956 AUC: 0.945 Z: 0.559 Rdelta: 0.00025
#> 89 Number of features: 55 Max AUC: 0.956 AUC: 0.925 Z: 0.495 Rdelta: 0.00020
#> 90 Number of features: 55 Max AUC: 0.956 AUC: 0.944 Z: 0.473 Rdelta: 0.00016
#> 91 Number of features: 55 Max AUC: 0.956 AUC: 0.940 Z: 0.558 Rdelta: 0.00013
#> 92 Number of features: 55 Max AUC: 0.956 AUC: 0.939 Z: 0.331 Rdelta: 0.00010
#> 93 Number of features: 55 Max AUC: 0.956 AUC: 0.938 Z: 0.453 Rdelta: 0.00008
#> user system elapsed
#> 47.18 0.00 47.24
testDistance_case <- signatureDistance(signature$caseTamplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_case))
system.time(signatureControl <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="RMS",target="Control"))
#> 7 Number of features: 7 Max AUC: 0.384 AUC: 0.373 Z: -0.449 Rdelta: 0.10000
#> 8 Number of features: 7 Max AUC: 0.384 AUC: 0.329 Z: -0.527 Rdelta: 0.08000
#> 9 Number of features: 7 Max AUC: 0.384 AUC: 0.321 Z: -0.487 Rdelta: 0.06400
#> 10 Number of features: 8 Max AUC: 0.384 AUC: 0.377 Z: -0.405 Rdelta: 0.06760
#> 11 Number of features: 8 Max AUC: 0.384 AUC: 0.367 Z: -0.414 Rdelta: 0.05408
#> 12 Number of features: 8 Max AUC: 0.384 AUC: 0.347 Z: -0.433 Rdelta: 0.04326
#> 13 Number of features: 8 Max AUC: 0.384 AUC: 0.337 Z: -0.485 Rdelta: 0.03461
#> 14 Number of features: 9 Max AUC: 0.384 AUC: 0.368 Z: -0.382 Rdelta: 0.04115
#> 15 Number of features: 10 Max AUC: 0.384 AUC: 0.371 Z: -0.412 Rdelta: 0.04704
#> 16 Number of features: 10 Max AUC: 0.384 AUC: 0.337 Z: -0.446 Rdelta: 0.03763
#> 17 Number of features: 10 Max AUC: 0.384 AUC: 0.335 Z: -0.438 Rdelta: 0.03010
#> 18 Number of features: 11 Max AUC: 0.384 AUC: 0.380 Z: -0.370 Rdelta: 0.03709
#> 19 Number of features: 11 Max AUC: 0.384 AUC: 0.344 Z: -0.459 Rdelta: 0.02967
#> 20 Number of features: 11 Max AUC: 0.384 AUC: 0.344 Z: -0.365 Rdelta: 0.02374
#> 21 Number of features: 11 Max AUC: 0.384 AUC: 0.359 Z: -0.459 Rdelta: 0.01899
#> 22 Number of features: 12 Max AUC: 0.448 AUC: 0.448 Z: -0.273 Rdelta: 0.02709
#> 23 Number of features: 12 Max AUC: 0.448 AUC: 0.433 Z: -0.139 Rdelta: 0.02167
#> 24 Number of features: 12 Max AUC: 0.448 AUC: 0.428 Z: -0.274 Rdelta: 0.01734
#> 25 Number of features: 12 Max AUC: 0.448 AUC: 0.411 Z: -0.347 Rdelta: 0.01387
#> 26 Number of features: 12 Max AUC: 0.448 AUC: 0.440 Z: -0.287 Rdelta: 0.01110
#> 27 Number of features: 12 Max AUC: 0.448 AUC: 0.435 Z: -0.340 Rdelta: 0.00888
#> 28 Number of features: 13 Max AUC: 0.449 AUC: 0.449 Z: -0.263 Rdelta: 0.01799
#> 29 Number of features: 13 Max AUC: 0.449 AUC: 0.414 Z: -0.354 Rdelta: 0.01439
#> 30 Number of features: 13 Max AUC: 0.449 AUC: 0.420 Z: -0.284 Rdelta: 0.01151
#> 31 Number of features: 13 Max AUC: 0.449 AUC: 0.404 Z: -0.356 Rdelta: 0.00921
#> 32 Number of features: 13 Max AUC: 0.449 AUC: 0.420 Z: -0.310 Rdelta: 0.00737
#> 33 Number of features: 13 Max AUC: 0.449 AUC: 0.426 Z: -0.277 Rdelta: 0.00589
#> 34 Number of features: 13 Max AUC: 0.449 AUC: 0.433 Z: -0.300 Rdelta: 0.00472
#> 35 Number of features: 13 Max AUC: 0.449 AUC: 0.419 Z: -0.267 Rdelta: 0.00377
#> 36 Number of features: 14 Max AUC: 0.453 AUC: 0.453 Z: -0.108 Rdelta: 0.01340
#> 37 Number of features: 14 Max AUC: 0.453 AUC: 0.388 Z: -0.368 Rdelta: 0.01072
#> 38 Number of features: 14 Max AUC: 0.453 AUC: 0.416 Z: -0.345 Rdelta: 0.00857
#> 39 Number of features: 14 Max AUC: 0.453 AUC: 0.415 Z: -0.301 Rdelta: 0.00686
#> 40 Number of features: 14 Max AUC: 0.453 AUC: 0.434 Z: -0.324 Rdelta: 0.00549
#> 41 Number of features: 15 Max AUC: 0.486 AUC: 0.486 Z: -0.028 Rdelta: 0.01494
#> 42 Number of features: 15 Max AUC: 0.486 AUC: 0.459 Z: -0.039 Rdelta: 0.01195
#> 43 Number of features: 15 Max AUC: 0.486 AUC: 0.428 Z: -0.085 Rdelta: 0.00956
#> 44 Number of features: 15 Max AUC: 0.486 AUC: 0.448 Z: -0.019 Rdelta: 0.00765
#> 45 Number of features: 15 Max AUC: 0.486 AUC: 0.430 Z: -0.029 Rdelta: 0.00612
#> 46 Number of features: 15 Max AUC: 0.486 AUC: 0.425 Z: -0.119 Rdelta: 0.00489
#> 47 Number of features: 15 Max AUC: 0.486 AUC: 0.412 Z: -0.112 Rdelta: 0.00392
#> 48 Number of features: 15 Max AUC: 0.486 AUC: 0.441 Z: -0.000 Rdelta: 0.00313
#> 49 Number of features: 15 Max AUC: 0.486 AUC: 0.452 Z: -0.016 Rdelta: 0.00251
#> 50 Number of features: 15 Max AUC: 0.486 AUC: 0.413 Z: -0.154 Rdelta: 0.00200
#> 51 Number of features: 15 Max AUC: 0.486 AUC: 0.424 Z: -0.095 Rdelta: 0.00160
#> 52 Number of features: 15 Max AUC: 0.486 AUC: 0.429 Z: -0.097 Rdelta: 0.00128
#> 53 Number of features: 15 Max AUC: 0.486 AUC: 0.455 Z: -0.105 Rdelta: 0.00103
#> 54 Number of features: 15 Max AUC: 0.486 AUC: 0.444 Z: -0.171 Rdelta: 0.00082
#> 55 Number of features: 15 Max AUC: 0.486 AUC: 0.474 Z: -0.052 Rdelta: 0.00066
#> 56 Number of features: 15 Max AUC: 0.486 AUC: 0.439 Z: -0.005 Rdelta: 0.00053
#> 57 Number of features: 15 Max AUC: 0.486 AUC: 0.455 Z: 0.000 Rdelta: 0.00042
#> 58 Number of features: 15 Max AUC: 0.486 AUC: 0.432 Z: -0.049 Rdelta: 0.00034
#> 59 Number of features: 15 Max AUC: 0.486 AUC: 0.438 Z: -0.018 Rdelta: 0.00027
#> 60 Number of features: 15 Max AUC: 0.486 AUC: 0.404 Z: -0.097 Rdelta: 0.00022
#> 61 Number of features: 16 Max AUC: 0.503 AUC: 0.503 Z: 0.004 Rdelta: 0.01019
#> 62 Number of features: 17 Max AUC: 0.521 AUC: 0.521 Z: 0.058 Rdelta: 0.01917
#> 63 Number of features: 17 Max AUC: 0.521 AUC: 0.466 Z: -0.087 Rdelta: 0.01534
#> 64 Number of features: 17 Max AUC: 0.521 AUC: 0.444 Z: -0.045 Rdelta: 0.01227
#> 65 Number of features: 17 Max AUC: 0.521 AUC: 0.472 Z: -0.099 Rdelta: 0.00982
#> 66 Number of features: 17 Max AUC: 0.521 AUC: 0.476 Z: -0.012 Rdelta: 0.00785
#> 67 Number of features: 17 Max AUC: 0.521 AUC: 0.496 Z: 0.012 Rdelta: 0.00628
#> 68 Number of features: 17 Max AUC: 0.521 AUC: 0.477 Z: -0.008 Rdelta: 0.00503
#> 69 Number of features: 17 Max AUC: 0.521 AUC: 0.503 Z: -0.004 Rdelta: 0.00402
#> 70 Number of features: 17 Max AUC: 0.521 AUC: 0.486 Z: -0.022 Rdelta: 0.00322
#> 71 Number of features: 17 Max AUC: 0.521 AUC: 0.481 Z: -0.093 Rdelta: 0.00257
#> 72 Number of features: 17 Max AUC: 0.521 AUC: 0.469 Z: -0.021 Rdelta: 0.00206
#> 73 Number of features: 17 Max AUC: 0.521 AUC: 0.489 Z: 0.031 Rdelta: 0.00165
#> 74 Number of features: 17 Max AUC: 0.521 AUC: 0.438 Z: -0.121 Rdelta: 0.00132
#> 75 Number of features: 17 Max AUC: 0.521 AUC: 0.481 Z: 0.001 Rdelta: 0.00105
#> 76 Number of features: 17 Max AUC: 0.521 AUC: 0.476 Z: -0.016 Rdelta: 0.00084
#> 77 Number of features: 17 Max AUC: 0.521 AUC: 0.453 Z: -0.080 Rdelta: 0.00067
#> 78 Number of features: 17 Max AUC: 0.521 AUC: 0.448 Z: -0.051 Rdelta: 0.00054
#> 79 Number of features: 17 Max AUC: 0.521 AUC: 0.471 Z: -0.025 Rdelta: 0.00043
#> 80 Number of features: 17 Max AUC: 0.521 AUC: 0.508 Z: 0.034 Rdelta: 0.00035
#> 81 Number of features: 17 Max AUC: 0.521 AUC: 0.485 Z: 0.014 Rdelta: 0.00028
#> 82 Number of features: 17 Max AUC: 0.521 AUC: 0.512 Z: 0.012 Rdelta: 0.00022
#> 83 Number of features: 17 Max AUC: 0.521 AUC: 0.481 Z: -0.015 Rdelta: 0.00018
#> 84 Number of features: 17 Max AUC: 0.521 AUC: 0.481 Z: -0.019 Rdelta: 0.00014
#> 85 Number of features: 17 Max AUC: 0.521 AUC: 0.456 Z: -0.036 Rdelta: 0.00011
#> 86 Number of features: 17 Max AUC: 0.521 AUC: 0.466 Z: -0.029 Rdelta: 0.00009
#> user system elapsed
#> 21.68 0.00 21.73
testDistance_control <- signatureDistance(signatureControl$controlTemplate,validLabeled,"RMS")
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control))
pm <-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance_control-testDistance_case))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
system.time(signature <- getSignature(data=trainLabeled,varlist=varlist,Outcome="Labels",method="MAN"))
#> 7 Number of features: 7 Max AUC: 0.751 AUC: 0.751 Z: 1.276 Rdelta: 0.10000
#> 8 Number of features: 8 Max AUC: 0.768 AUC: 0.768 Z: 0.850 Rdelta: 0.10000
#> 9 Number of features: 9 Max AUC: 0.792 AUC: 0.792 Z: 1.086 Rdelta: 0.10000
#> 10 Number of features: 9 Max AUC: 0.792 AUC: 0.761 Z: 0.947 Rdelta: 0.08000
#> 11 Number of features: 10 Max AUC: 0.792 AUC: 0.790 Z: 0.984 Rdelta: 0.08200
#> 12 Number of features: 11 Max AUC: 0.792 AUC: 0.781 Z: 1.022 Rdelta: 0.08380
#> 13 Number of features: 12 Max AUC: 0.792 AUC: 0.786 Z: 0.546 Rdelta: 0.08542
#> 14 Number of features: 13 Max AUC: 0.792 AUC: 0.774 Z: 0.580 Rdelta: 0.08688
#> 15 Number of features: 14 Max AUC: 0.792 AUC: 0.776 Z: 0.657 Rdelta: 0.08819
#> 16 Number of features: 14 Max AUC: 0.792 AUC: 0.767 Z: 0.180 Rdelta: 0.07055
#> 17 Number of features: 15 Max AUC: 0.792 AUC: 0.776 Z: 0.581 Rdelta: 0.07350
#> 18 Number of features: 16 Max AUC: 0.796 AUC: 0.796 Z: 0.446 Rdelta: 0.07615
#> 19 Number of features: 17 Max AUC: 0.805 AUC: 0.805 Z: 0.449 Rdelta: 0.07853
#> 20 Number of features: 17 Max AUC: 0.805 AUC: 0.784 Z: 0.520 Rdelta: 0.06283
#> 21 Number of features: 18 Max AUC: 0.824 AUC: 0.824 Z: 0.594 Rdelta: 0.06654
#> 22 Number of features: 18 Max AUC: 0.824 AUC: 0.808 Z: 0.517 Rdelta: 0.05323
#> 23 Number of features: 19 Max AUC: 0.824 AUC: 0.821 Z: 0.485 Rdelta: 0.05791
#> 24 Number of features: 20 Max AUC: 0.824 AUC: 0.822 Z: 0.632 Rdelta: 0.06212
#> 25 Number of features: 21 Max AUC: 0.824 AUC: 0.824 Z: 0.491 Rdelta: 0.06591
#> 26 Number of features: 22 Max AUC: 0.825 AUC: 0.825 Z: 0.487 Rdelta: 0.06932
#> 27 Number of features: 23 Max AUC: 0.840 AUC: 0.840 Z: 0.641 Rdelta: 0.07239
#> 28 Number of features: 24 Max AUC: 0.840 AUC: 0.834 Z: 0.495 Rdelta: 0.07515
#> 29 Number of features: 24 Max AUC: 0.840 AUC: 0.826 Z: 0.470 Rdelta: 0.06012
#> 30 Number of features: 25 Max AUC: 0.840 AUC: 0.837 Z: 0.488 Rdelta: 0.06411
#> 31 Number of features: 26 Max AUC: 0.840 AUC: 0.834 Z: 0.630 Rdelta: 0.06770
#> 32 Number of features: 27 Max AUC: 0.843 AUC: 0.843 Z: 0.507 Rdelta: 0.07093
#> 33 Number of features: 27 Max AUC: 0.843 AUC: 0.828 Z: 0.420 Rdelta: 0.05674
#> 34 Number of features: 27 Max AUC: 0.843 AUC: 0.808 Z: 0.576 Rdelta: 0.04539
#> 35 Number of features: 28 Max AUC: 0.848 AUC: 0.848 Z: 0.542 Rdelta: 0.05085
#> 36 Number of features: 28 Max AUC: 0.848 AUC: 0.836 Z: 0.496 Rdelta: 0.04068
#> 37 Number of features: 29 Max AUC: 0.848 AUC: 0.845 Z: 0.689 Rdelta: 0.04661
#> 38 Number of features: 30 Max AUC: 0.848 AUC: 0.844 Z: 0.516 Rdelta: 0.05195
#> 39 Number of features: 31 Max AUC: 0.848 AUC: 0.841 Z: 0.508 Rdelta: 0.05676
#> 40 Number of features: 31 Max AUC: 0.848 AUC: 0.798 Z: 0.425 Rdelta: 0.04541
#> 41 Number of features: 31 Max AUC: 0.848 AUC: 0.790 Z: 0.034 Rdelta: 0.03632
#> 42 Number of features: 32 Max AUC: 0.848 AUC: 0.843 Z: 0.421 Rdelta: 0.04269
#> 43 Number of features: 33 Max AUC: 0.848 AUC: 0.836 Z: 0.510 Rdelta: 0.04842
#> 44 Number of features: 34 Max AUC: 0.848 AUC: 0.845 Z: 0.388 Rdelta: 0.05358
#> 45 Number of features: 34 Max AUC: 0.848 AUC: 0.832 Z: 0.499 Rdelta: 0.04286
#> 46 Number of features: 34 Max AUC: 0.848 AUC: 0.800 Z: 0.526 Rdelta: 0.03429
#> 47 Number of features: 35 Max AUC: 0.850 AUC: 0.850 Z: 0.543 Rdelta: 0.04086
#> 48 Number of features: 35 Max AUC: 0.850 AUC: 0.835 Z: 0.394 Rdelta: 0.03269
#> 49 Number of features: 36 Max AUC: 0.850 AUC: 0.843 Z: 0.412 Rdelta: 0.03942
#> 50 Number of features: 36 Max AUC: 0.850 AUC: 0.834 Z: 0.412 Rdelta: 0.03154
#> 51 Number of features: 36 Max AUC: 0.850 AUC: 0.834 Z: 0.533 Rdelta: 0.02523
#> 52 Number of features: 36 Max AUC: 0.850 AUC: 0.837 Z: 0.447 Rdelta: 0.02018
#> 53 Number of features: 37 Max AUC: 0.852 AUC: 0.852 Z: 0.421 Rdelta: 0.02817
#> 54 Number of features: 38 Max AUC: 0.852 AUC: 0.849 Z: 0.317 Rdelta: 0.03535
#> 55 Number of features: 38 Max AUC: 0.852 AUC: 0.840 Z: 0.446 Rdelta: 0.02828
#> 56 Number of features: 39 Max AUC: 0.852 AUC: 0.844 Z: 0.504 Rdelta: 0.03545
#> 57 Number of features: 39 Max AUC: 0.852 AUC: 0.835 Z: 0.616 Rdelta: 0.02836
#> 58 Number of features: 39 Max AUC: 0.852 AUC: 0.830 Z: 0.532 Rdelta: 0.02269
#> 59 Number of features: 39 Max AUC: 0.852 AUC: 0.840 Z: 0.347 Rdelta: 0.01815
#> 60 Number of features: 39 Max AUC: 0.852 AUC: 0.841 Z: 0.393 Rdelta: 0.01452
#> 61 Number of features: 39 Max AUC: 0.852 AUC: 0.791 Z: 0.575 Rdelta: 0.01162
#> 62 Number of features: 39 Max AUC: 0.852 AUC: 0.827 Z: 0.419 Rdelta: 0.00929
#> 63 Number of features: 39 Max AUC: 0.852 AUC: 0.838 Z: 0.393 Rdelta: 0.00743
#> 64 Number of features: 39 Max AUC: 0.852 AUC: 0.821 Z: 0.356 Rdelta: 0.00595
#> 65 Number of features: 39 Max AUC: 0.852 AUC: 0.835 Z: 0.498 Rdelta: 0.00476
#> 66 Number of features: 40 Max AUC: 0.852 AUC: 0.841 Z: 0.588 Rdelta: 0.01428
#> 67 Number of features: 40 Max AUC: 0.852 AUC: 0.799 Z: 0.385 Rdelta: 0.01143
#> 68 Number of features: 40 Max AUC: 0.852 AUC: 0.827 Z: 0.530 Rdelta: 0.00914
#> 69 Number of features: 41 Max AUC: 0.852 AUC: 0.849 Z: 0.592 Rdelta: 0.01823
#> 70 Number of features: 42 Max AUC: 0.852 AUC: 0.849 Z: 0.528 Rdelta: 0.02640
#> 71 Number of features: 42 Max AUC: 0.852 AUC: 0.836 Z: 0.392 Rdelta: 0.02112
#> 72 Number of features: 42 Max AUC: 0.852 AUC: 0.840 Z: 0.548 Rdelta: 0.01690
#> 73 Number of features: 42 Max AUC: 0.852 AUC: 0.832 Z: 0.492 Rdelta: 0.01352
#> 74 Number of features: 42 Max AUC: 0.852 AUC: 0.830 Z: 0.554 Rdelta: 0.01082
#> 75 Number of features: 43 Max AUC: 0.852 AUC: 0.842 Z: 0.534 Rdelta: 0.01973
#> 76 Number of features: 44 Max AUC: 0.856 AUC: 0.856 Z: 0.381 Rdelta: 0.02776
#> 77 Number of features: 44 Max AUC: 0.856 AUC: 0.837 Z: 0.486 Rdelta: 0.02221
#> 78 Number of features: 44 Max AUC: 0.856 AUC: 0.829 Z: 0.585 Rdelta: 0.01777
#> 79 Number of features: 44 Max AUC: 0.856 AUC: 0.819 Z: 0.478 Rdelta: 0.01421
#> 80 Number of features: 44 Max AUC: 0.856 AUC: 0.843 Z: 0.621 Rdelta: 0.01137
#> 81 Number of features: 44 Max AUC: 0.856 AUC: 0.830 Z: 0.432 Rdelta: 0.00910
#> 82 Number of features: 45 Max AUC: 0.856 AUC: 0.847 Z: 0.596 Rdelta: 0.01819
#> 83 Number of features: 45 Max AUC: 0.856 AUC: 0.833 Z: 0.626 Rdelta: 0.01455
#> 84 Number of features: 45 Max AUC: 0.856 AUC: 0.839 Z: 0.432 Rdelta: 0.01164
#> 85 Number of features: 45 Max AUC: 0.856 AUC: 0.843 Z: 0.432 Rdelta: 0.00931
#> 86 Number of features: 45 Max AUC: 0.856 AUC: 0.829 Z: 0.515 Rdelta: 0.00745
#> 87 Number of features: 45 Max AUC: 0.856 AUC: 0.826 Z: 0.449 Rdelta: 0.00596
#> 88 Number of features: 45 Max AUC: 0.856 AUC: 0.822 Z: 0.414 Rdelta: 0.00477
#> 89 Number of features: 45 Max AUC: 0.856 AUC: 0.813 Z: 0.518 Rdelta: 0.00381
#> 90 Number of features: 45 Max AUC: 0.856 AUC: 0.826 Z: 0.539 Rdelta: 0.00305
#> 91 Number of features: 45 Max AUC: 0.856 AUC: 0.831 Z: 0.432 Rdelta: 0.00244
#> 92 Number of features: 45 Max AUC: 0.856 AUC: 0.834 Z: 0.451 Rdelta: 0.00195
#> 93 Number of features: 45 Max AUC: 0.856 AUC: 0.835 Z: 0.588 Rdelta: 0.00156
#> 94 Number of features: 46 Max AUC: 0.856 AUC: 0.855 Z: 0.313 Rdelta: 0.01141
#> 95 Number of features: 46 Max AUC: 0.856 AUC: 0.844 Z: 0.434 Rdelta: 0.00912
#> 96 Number of features: 46 Max AUC: 0.856 AUC: 0.838 Z: 0.509 Rdelta: 0.00730
#> 97 Number of features: 46 Max AUC: 0.856 AUC: 0.829 Z: 0.491 Rdelta: 0.00584
#> 98 Number of features: 47 Max AUC: 0.856 AUC: 0.845 Z: 0.486 Rdelta: 0.01526
#> 99 Number of features: 48 Max AUC: 0.856 AUC: 0.846 Z: 0.419 Rdelta: 0.02373
#> 100 Number of features: 48 Max AUC: 0.856 AUC: 0.825 Z: 0.478 Rdelta: 0.01898
#> 101 Number of features: 48 Max AUC: 0.856 AUC: 0.832 Z: 0.369 Rdelta: 0.01519
#> 102 Number of features: 48 Max AUC: 0.856 AUC: 0.844 Z: 0.363 Rdelta: 0.01215
#> 103 Number of features: 48 Max AUC: 0.856 AUC: 0.828 Z: 0.503 Rdelta: 0.00972
#> 104 Number of features: 48 Max AUC: 0.856 AUC: 0.824 Z: 0.364 Rdelta: 0.00778
#> 105 Number of features: 48 Max AUC: 0.856 AUC: 0.799 Z: 0.394 Rdelta: 0.00622
#> 106 Number of features: 48 Max AUC: 0.856 AUC: 0.835 Z: 0.427 Rdelta: 0.00498
#> 107 Number of features: 48 Max AUC: 0.856 AUC: 0.830 Z: 0.489 Rdelta: 0.00398
#> 108 Number of features: 48 Max AUC: 0.856 AUC: 0.835 Z: 0.396 Rdelta: 0.00319
#> 109 Number of features: 48 Max AUC: 0.856 AUC: 0.818 Z: 0.426 Rdelta: 0.00255
#> 110 Number of features: 48 Max AUC: 0.856 AUC: 0.835 Z: 0.452 Rdelta: 0.00204
#> 111 Number of features: 48 Max AUC: 0.856 AUC: 0.822 Z: 0.461 Rdelta: 0.00163
#> 112 Number of features: 48 Max AUC: 0.856 AUC: 0.828 Z: 0.466 Rdelta: 0.00130
#> 113 Number of features: 48 Max AUC: 0.856 AUC: 0.821 Z: 0.325 Rdelta: 0.00104
#> 114 Number of features: 48 Max AUC: 0.856 AUC: 0.806 Z: 0.255 Rdelta: 0.00083
#> 115 Number of features: 48 Max AUC: 0.856 AUC: 0.824 Z: 0.393 Rdelta: 0.00067
#> 116 Number of features: 49 Max AUC: 0.856 AUC: 0.846 Z: 0.489 Rdelta: 0.01060
#> 117 Number of features: 49 Max AUC: 0.856 AUC: 0.836 Z: 0.370 Rdelta: 0.00848
#> 118 Number of features: 49 Max AUC: 0.856 AUC: 0.817 Z: 0.451 Rdelta: 0.00678
#> 119 Number of features: 50 Max AUC: 0.856 AUC: 0.847 Z: 0.319 Rdelta: 0.01611
#> 120 Number of features: 50 Max AUC: 0.856 AUC: 0.826 Z: 0.423 Rdelta: 0.01289
#> 121 Number of features: 50 Max AUC: 0.856 AUC: 0.831 Z: 0.527 Rdelta: 0.01031
#> 122 Number of features: 50 Max AUC: 0.856 AUC: 0.823 Z: 0.392 Rdelta: 0.00825
#> 123 Number of features: 50 Max AUC: 0.856 AUC: 0.829 Z: 0.559 Rdelta: 0.00660
#> 124 Number of features: 51 Max AUC: 0.862 AUC: 0.862 Z: 0.485 Rdelta: 0.01594
#> 125 Number of features: 51 Max AUC: 0.862 AUC: 0.836 Z: 0.481 Rdelta: 0.01275
#> 126 Number of features: 51 Max AUC: 0.862 AUC: 0.839 Z: 0.493 Rdelta: 0.01020
#> 127 Number of features: 51 Max AUC: 0.862 AUC: 0.808 Z: 0.501 Rdelta: 0.00816
#> 128 Number of features: 51 Max AUC: 0.862 AUC: 0.831 Z: 0.483 Rdelta: 0.00653
#> 129 Number of features: 52 Max AUC: 0.862 AUC: 0.857 Z: 0.461 Rdelta: 0.01588
#> 130 Number of features: 52 Max AUC: 0.862 AUC: 0.798 Z: 0.371 Rdelta: 0.01270
#> 131 Number of features: 52 Max AUC: 0.862 AUC: 0.820 Z: 0.544 Rdelta: 0.01016
#> 132 Number of features: 52 Max AUC: 0.862 AUC: 0.840 Z: 0.404 Rdelta: 0.00813
#> 133 Number of features: 52 Max AUC: 0.862 AUC: 0.809 Z: 0.393 Rdelta: 0.00650
#> 134 Number of features: 52 Max AUC: 0.862 AUC: 0.836 Z: 0.332 Rdelta: 0.00520
#> 135 Number of features: 52 Max AUC: 0.862 AUC: 0.837 Z: 0.458 Rdelta: 0.00416
#> 136 Number of features: 52 Max AUC: 0.862 AUC: 0.804 Z: 0.428 Rdelta: 0.00333
#> 137 Number of features: 52 Max AUC: 0.862 AUC: 0.831 Z: 0.437 Rdelta: 0.00266
#> 138 Number of features: 52 Max AUC: 0.862 AUC: 0.834 Z: 0.414 Rdelta: 0.00213
#> 139 Number of features: 52 Max AUC: 0.862 AUC: 0.833 Z: 0.347 Rdelta: 0.00170
#> 140 Number of features: 52 Max AUC: 0.862 AUC: 0.843 Z: 0.451 Rdelta: 0.00136
#> 141 Number of features: 52 Max AUC: 0.862 AUC: 0.833 Z: 0.270 Rdelta: 0.00109
#> 142 Number of features: 53 Max AUC: 0.862 AUC: 0.853 Z: 0.466 Rdelta: 0.01098
#> 143 Number of features: 53 Max AUC: 0.862 AUC: 0.847 Z: 0.407 Rdelta: 0.00879
#> 144 Number of features: 53 Max AUC: 0.862 AUC: 0.842 Z: 0.492 Rdelta: 0.00703
#> 145 Number of features: 53 Max AUC: 0.862 AUC: 0.830 Z: 0.460 Rdelta: 0.00562
#> 146 Number of features: 53 Max AUC: 0.862 AUC: 0.846 Z: 0.416 Rdelta: 0.00450
#> 147 Number of features: 53 Max AUC: 0.862 AUC: 0.833 Z: 0.502 Rdelta: 0.00360
#> 148 Number of features: 53 Max AUC: 0.862 AUC: 0.828 Z: 0.565 Rdelta: 0.00288
#> 149 Number of features: 53 Max AUC: 0.862 AUC: 0.842 Z: 0.439 Rdelta: 0.00230
#> 150 Number of features: 53 Max AUC: 0.862 AUC: 0.826 Z: 0.499 Rdelta: 0.00184
#> 151 Number of features: 53 Max AUC: 0.862 AUC: 0.821 Z: 0.473 Rdelta: 0.00147
#> 152 Number of features: 53 Max AUC: 0.862 AUC: 0.837 Z: 0.451 Rdelta: 0.00118
#> 153 Number of features: 54 Max AUC: 0.862 AUC: 0.859 Z: 0.480 Rdelta: 0.01106
#> 154 Number of features: 54 Max AUC: 0.862 AUC: 0.817 Z: 0.474 Rdelta: 0.00885
#> 155 Number of features: 54 Max AUC: 0.862 AUC: 0.846 Z: 0.519 Rdelta: 0.00708
#> 156 Number of features: 54 Max AUC: 0.862 AUC: 0.823 Z: 0.479 Rdelta: 0.00566
#> 157 Number of features: 54 Max AUC: 0.862 AUC: 0.829 Z: 0.391 Rdelta: 0.00453
#> 158 Number of features: 54 Max AUC: 0.862 AUC: 0.844 Z: 0.523 Rdelta: 0.00362
#> 159 Number of features: 54 Max AUC: 0.862 AUC: 0.842 Z: 0.438 Rdelta: 0.00290
#> 160 Number of features: 54 Max AUC: 0.862 AUC: 0.828 Z: 0.451 Rdelta: 0.00232
#> 161 Number of features: 54 Max AUC: 0.862 AUC: 0.828 Z: 0.412 Rdelta: 0.00186
#> 162 Number of features: 54 Max AUC: 0.862 AUC: 0.824 Z: 0.442 Rdelta: 0.00148
#> 163 Number of features: 55 Max AUC: 0.862 AUC: 0.854 Z: 0.402 Rdelta: 0.01134
#> 164 Number of features: 56 Max AUC: 0.862 AUC: 0.854 Z: 0.374 Rdelta: 0.02020
#> 165 Number of features: 56 Max AUC: 0.862 AUC: 0.846 Z: 0.451 Rdelta: 0.01616
#> 166 Number of features: 56 Max AUC: 0.862 AUC: 0.848 Z: 0.509 Rdelta: 0.01293
#> 167 Number of features: 56 Max AUC: 0.862 AUC: 0.844 Z: 0.465 Rdelta: 0.01034
#> 168 Number of features: 56 Max AUC: 0.862 AUC: 0.835 Z: 0.413 Rdelta: 0.00827
#> 169 Number of features: 57 Max AUC: 0.862 AUC: 0.859 Z: 0.384 Rdelta: 0.01745
#> 170 Number of features: 57 Max AUC: 0.862 AUC: 0.827 Z: 0.522 Rdelta: 0.01396
#> 171 Number of features: 57 Max AUC: 0.862 AUC: 0.849 Z: 0.486 Rdelta: 0.01117
#> 172 Number of features: 57 Max AUC: 0.862 AUC: 0.836 Z: 0.431 Rdelta: 0.00893
#> 173 Number of features: 57 Max AUC: 0.862 AUC: 0.839 Z: 0.446 Rdelta: 0.00715
#> 174 Number of features: 57 Max AUC: 0.862 AUC: 0.814 Z: 0.467 Rdelta: 0.00572
#> 175 Number of features: 57 Max AUC: 0.862 AUC: 0.834 Z: 0.381 Rdelta: 0.00457
#> 176 Number of features: 57 Max AUC: 0.862 AUC: 0.840 Z: 0.388 Rdelta: 0.00366
#> 177 Number of features: 57 Max AUC: 0.862 AUC: 0.849 Z: 0.394 Rdelta: 0.00293
#> 178 Number of features: 57 Max AUC: 0.862 AUC: 0.834 Z: 0.402 Rdelta: 0.00234
#> 179 Number of features: 57 Max AUC: 0.862 AUC: 0.844 Z: 0.465 Rdelta: 0.00187
#> 180 Number of features: 58 Max AUC: 0.862 AUC: 0.860 Z: 0.365 Rdelta: 0.01169
#> 181 Number of features: 58 Max AUC: 0.862 AUC: 0.830 Z: 0.421 Rdelta: 0.00935
#> 182 Number of features: 58 Max AUC: 0.862 AUC: 0.826 Z: 0.506 Rdelta: 0.00748
#> 183 Number of features: 58 Max AUC: 0.862 AUC: 0.824 Z: 0.460 Rdelta: 0.00598
#> 184 Number of features: 59 Max AUC: 0.862 AUC: 0.858 Z: 0.509 Rdelta: 0.01538
#> 185 Number of features: 59 Max AUC: 0.862 AUC: 0.850 Z: 0.485 Rdelta: 0.01231
#> 186 Number of features: 59 Max AUC: 0.862 AUC: 0.825 Z: 0.405 Rdelta: 0.00985
#> 187 Number of features: 59 Max AUC: 0.862 AUC: 0.836 Z: 0.436 Rdelta: 0.00788
#> 188 Number of features: 59 Max AUC: 0.862 AUC: 0.823 Z: 0.441 Rdelta: 0.00630
#> 189 Number of features: 59 Max AUC: 0.862 AUC: 0.838 Z: 0.423 Rdelta: 0.00504
#> 190 Number of features: 60 Max AUC: 0.862 AUC: 0.855 Z: 0.282 Rdelta: 0.01454
#> 191 Number of features: 60 Max AUC: 0.862 AUC: 0.843 Z: 0.286 Rdelta: 0.01163
#> 192 Number of features: 60 Max AUC: 0.862 AUC: 0.834 Z: 0.339 Rdelta: 0.00930
#> 193 Number of features: 61 Max AUC: 0.862 AUC: 0.855 Z: 0.292 Rdelta: 0.01837
#> 194 Number of features: 61 Max AUC: 0.862 AUC: 0.817 Z: 0.332 Rdelta: 0.01470
#> 195 Number of features: 61 Max AUC: 0.862 AUC: 0.847 Z: 0.335 Rdelta: 0.01176
#> 196 Number of features: 61 Max AUC: 0.862 AUC: 0.810 Z: 0.345 Rdelta: 0.00941
#> 197 Number of features: 61 Max AUC: 0.862 AUC: 0.838 Z: 0.314 Rdelta: 0.00753
#> 198 Number of features: 61 Max AUC: 0.862 AUC: 0.829 Z: 0.373 Rdelta: 0.00602
#> 199 Number of features: 61 Max AUC: 0.862 AUC: 0.836 Z: 0.298 Rdelta: 0.00482
#> 200 Number of features: 61 Max AUC: 0.862 AUC: 0.848 Z: 0.291 Rdelta: 0.00385
#> 201 Number of features: 61 Max AUC: 0.862 AUC: 0.829 Z: 0.307 Rdelta: 0.00308
#> 202 Number of features: 61 Max AUC: 0.862 AUC: 0.845 Z: 0.272 Rdelta: 0.00247
#> 203 Number of features: 61 Max AUC: 0.862 AUC: 0.823 Z: 0.300 Rdelta: 0.00197
#> 204 Number of features: 61 Max AUC: 0.862 AUC: 0.843 Z: 0.316 Rdelta: 0.00158
#> 205 Number of features: 61 Max AUC: 0.862 AUC: 0.823 Z: 0.387 Rdelta: 0.00126
#> 206 Number of features: 61 Max AUC: 0.862 AUC: 0.835 Z: 0.345 Rdelta: 0.00101
#> 207 Number of features: 61 Max AUC: 0.862 AUC: 0.841 Z: 0.380 Rdelta: 0.00081
#> 208 Number of features: 61 Max AUC: 0.862 AUC: 0.842 Z: 0.436 Rdelta: 0.00065
#> 209 Number of features: 61 Max AUC: 0.862 AUC: 0.821 Z: 0.335 Rdelta: 0.00052
#> 210 Number of features: 61 Max AUC: 0.862 AUC: 0.847 Z: 0.411 Rdelta: 0.00041
#> 211 Number of features: 62 Max AUC: 0.862 AUC: 0.858 Z: 0.336 Rdelta: 0.01037
#> 212 Number of features: 62 Max AUC: 0.862 AUC: 0.845 Z: 0.385 Rdelta: 0.00830
#> 213 Number of features: 62 Max AUC: 0.862 AUC: 0.849 Z: 0.318 Rdelta: 0.00664
#> 214 Number of features: 62 Max AUC: 0.862 AUC: 0.826 Z: 0.375 Rdelta: 0.00531
#> 215 Number of features: 62 Max AUC: 0.862 AUC: 0.835 Z: 0.312 Rdelta: 0.00425
#> 216 Number of features: 62 Max AUC: 0.862 AUC: 0.841 Z: 0.318 Rdelta: 0.00340
#> 217 Number of features: 62 Max AUC: 0.862 AUC: 0.841 Z: 0.319 Rdelta: 0.00272
#> 218 Number of features: 62 Max AUC: 0.862 AUC: 0.838 Z: 0.263 Rdelta: 0.00218
#> 219 Number of features: 62 Max AUC: 0.862 AUC: 0.822 Z: 0.316 Rdelta: 0.00174
#> 220 Number of features: 62 Max AUC: 0.862 AUC: 0.837 Z: 0.447 Rdelta: 0.00139
#> 221 Number of features: 62 Max AUC: 0.862 AUC: 0.825 Z: 0.323 Rdelta: 0.00111
#> 222 Number of features: 62 Max AUC: 0.862 AUC: 0.842 Z: 0.318 Rdelta: 0.00089
#> 223 Number of features: 62 Max AUC: 0.862 AUC: 0.829 Z: 0.253 Rdelta: 0.00071
#> 224 Number of features: 62 Max AUC: 0.862 AUC: 0.827 Z: 0.369 Rdelta: 0.00057
#> 225 Number of features: 62 Max AUC: 0.862 AUC: 0.835 Z: 0.314 Rdelta: 0.00046
#> 226 Number of features: 62 Max AUC: 0.862 AUC: 0.828 Z: 0.410 Rdelta: 0.00036
#> 227 Number of features: 62 Max AUC: 0.862 AUC: 0.850 Z: 0.331 Rdelta: 0.00029
#> 228 Number of features: 62 Max AUC: 0.862 AUC: 0.842 Z: 0.310 Rdelta: 0.00023
#> 229 Number of features: 62 Max AUC: 0.862 AUC: 0.846 Z: 0.374 Rdelta: 0.00019
#> 230 Number of features: 62 Max AUC: 0.862 AUC: 0.819 Z: 0.398 Rdelta: 0.00015
#> 231 Number of features: 62 Max AUC: 0.862 AUC: 0.833 Z: 0.397 Rdelta: 0.00012
#> 232 Number of features: 62 Max AUC: 0.862 AUC: 0.825 Z: 0.285 Rdelta: 0.00010
#> user system elapsed
#> 136.02 0.00 136.12
testDistance <- -signatureDistance(signature$caseTamplate,validLabeled,"MAN")+signatureDistance(signature$controlTemplate,validLabeled,"MAN")
pm<-plotModels.ROC(cbind(as.vector(validLabeled$Labels),testDistance))
ci <- epi.tests(pm$predictionTable)
sig_ACCtable <- rbind(sig_ACCtable,ci$elements$diag.acc)
sig_errorcitable <- rbind(sig_errorcitable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
sizesig <- append(sizesig,ncol(signature$caseTamplate))
#############################################################################################
sig_thesets <- c("Pearson","RMS","RMS_2","Manhatan")
bp <- barPlotCiError(as.matrix(sig_ACCtable),metricname="Accuracy",thesets=sig_thesets,themethod=c("Z>3","V5","V10","V10b"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(sig_errorcitable),metricname="Balanced Error",thesets=sig_thesets,themethod=c("Z>3","V5","V10","V10b"),main="Balanced Error",args.legend = list(x = "topright"))
arceneCV <- FRESA.Model(formula = Labels ~ 1,data = arcene.norm,filter.p.value = 0.01,CVfolds = 3,repeats = 5,bswimsCycles=10,usrFitFun=svm)
#> Unadjusted size: 3373 Adjusted Size: 3535
#> ..
#> Vars: 10000 Size: 100 , Fraction= 0.168, Average random size = 20.00, Size:400.00
#>
#> Z: 2.575829 , Var Max: 3373 , s1: 10001 , s2: 4118 , Independent Size: 800
#> [1] "V3365 + V7748 + V2556 + V1936 + V7513 + V4200 + V7212 + V2804 + V2256 + V52 + V271 + V9818 + V2264 + V4198 + V4352 + V1046 + V4295 + V6584 + V6567 + V2973 + V9402 + V2358 + V3592 + V729 + V6164 + V8026 + V6440 + V5936 + V762 + V22 + V4055 + V5321 + V3161 + V6692 + V7189 + V2533 + V8586 + V754 + V1184 + V6520"
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8899272 : Labels ~ 1 + V698 + V6163 + V4960 + V256 + V4185 + V8247 + V9287 + V8245 + V6408
#> 1 : 3364 : 0.86582 : Labels ~ 1 + V5005 + V1476 + V9275 + V3708 + V9158 + V4584 + V6685 + V9213 + V5321
#> 2 : 3355 : 0.8439689 : Labels ~ 1 + V4290 + V3365 + V9215 + V9395 + V4011 + V7891 + V3675
#> 3 : 3348 : 0.8424779 : Labels ~ 1 + V7748 + V8368 + V5680 + V4406 + V1344 + V4352 + V8969 + V584
#> 4 : 3340 : 0.8463496 : Labels ~ 1 + V2556 + V9818 + V312 + V3161 + V729 + V762
#> 5 : 3334 : 0.8192451 : Labels ~ 1 + V8502 + V7272 + V5761 + V1248 + V7195 + V6472
#> 6 : 3328 : 0.8345967 : Labels ~ 1 + V6584 + V414 + V4973 + V4301 + V5454 + V723 + V5194
#> 7 : 3321 : 0.8124393 : Labels ~ 1 + V2309 + V436 + V8156 + V4326 + V9965 + V9919 + V711
#> 8 : 3314 : 0.7881142 : Labels ~ 1 + V1975 + V9617 + V1184 + V6688 + V1831
#> 9 : 3309 : 0.8124757 : Labels ~ 1 + V9027 + V8055 + V86 + V2515 + V3170
#>
#> Num. Models: 10 To Test: 69 TopFreq: 1 Thrf: 0 Removed: 0
#> .*CV pvalue : 0.05
#> Update : Labels ~ 1 + V5005 + V698 + V6163 + V4960 + V256 + V4185 + V8247 + V9287 + V8245 + V6408
#> At Accuray: Labels ~ 1 + V698 + V6163 + V4960 + V256 + V4185 + V8247 + V9287 + V8245 + V6408
#> B:SWiMS : Labels ~ 1 + V698 + V6163 + V4960 + V256 + V4185 + V8247 + V9287 + V8245 + V6408
#> Loop : 1 Input Cases = 88 Input Control = 112
#> Loop : 1 Train Cases = 58 Train Control = 74
#> Loop : 1 Blind Cases = 30 Blind Control = 38
#> K : 11 KNN T Cases = 58 KNN T Control = 58
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8745541 : Labels ~ 1 + V3628 + V9818 + V3161 + V1664 + V2597 + V7928
#> 1 : 3367 : 0.8216551 : Labels ~ 1 + V3365 + V8156 + V6198 + V3592
#> 2 : 3363 : 0.7936555 : Labels ~ 1 + V312 + V6584 + V729
#> 3 : 3360 : 0.8451632 : Labels ~ 1 + V5005 + V455 + V4584 + V52 + V9275
#> 4 : 3355 : 0.7735905 : Labels ~ 1 + V8368 + V1097
#> 5 : 3353 : 0.7857353 : Labels ~ 1 + V4960 + V9215 + V1128
#> 6 : 3350 : 0.8322935 : Labels ~ 1 + V414 + V9965 + V8981 + V723
#> 7 : 3346 : 0.7671598 : Labels ~ 1 + V376 + V256 + V7849
#> 8 : 3343 : 0.794387 : Labels ~ 1 + V1936 + V9027 + V6164
#> 9 : 3340 : 0.8561705 : Labels ~ 1 + V2309 + V1831 + V9395 + V9932 + V1476
#>
#> Num. Models: 10 To Test: 38 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V3628 + V5005 + V9818 + V3161 + V1664 + V2597 + V7928
#> At Accuracy: Labels ~ 1 + V3628 + V9818 + V3161 + V1664 + V2597 + V7928
#> B:SWiMS : Labels ~ 1 + V3628 + V9818 + V3161 + V1664 + V2597 + V7928
#>
#> Num. Models: 320 To Test: 1115 TopFreq: 11.97959 Thrf: 0 Removed: 0
#> ................................*Loop : 1 Blind Cases = 30 Blind Control = 38 Total = 68 Size Cases = 30 Size Control = 38
#> Accumulated Models CV Accuracy = 0.75 Sensitivity = 0.7333333 Specificity = 0.7631579 Forw. Ensemble Accuracy= 0.75
#> Initial Model Accumulated CV Accuracy = 0.8529412 Sensitivity = 0.8 Specificity = 0.8947368
#> Initial Model Bootstrapped Accuracy = 0.8708198 Sensitivity = 0.8635691 Specificity = 0.8780704
#> Current Model Bootstrapped Accuracy = 0.8745541 Sensitivity = 0.8998216 Specificity = 0.8492866
#> Current KNN Accuracy = 0.7941176 Sensitivity = 0.9333333 Specificity = 0.6842105
#> Initial KNN Accuracy = 0.8088235 Sensitivity = 0.9 Specificity = 0.7368421
#> Train Correlation: 0.7704895 Blind Correlation : 0.6820247
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 26 2
#> TRUE 11 29
#> Loop : 2 Input Cases = 88 Input Control = 112
#> Loop : 2 Train Cases = 59 Train Control = 75
#> Loop : 2 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8468732 : Labels ~ 1 + V2556 + V6584 + V4183 + V1184 + V7629
#> 1 : 3368 : 0.8424216 : Labels ~ 1 + V7891 + V1975 + V4352 + V4301 + V9215 + V7062 + V5321
#> 2 : 3361 : 0.8264052 : Labels ~ 1 + V9213 + V9818 + V5982
#> 3 : 3358 : 0.8649282 : Labels ~ 1 + V8623 + V5 + V9275 + V1602 + V629
#> 4 : 3353 : 0.8542654 : Labels ~ 1 + V4290 + V469 + V3170 + V8055 + V5487 + V9276 + V2408
#> 5 : 3346 : 0.8124814 : Labels ~ 1 + V872 + V1046 + V5680 + V6685
#> 6 : 3342 : 0.875781 : Labels ~ 1 + V782 + V5417 + V4018 + V8245 + V6114 + V4564
#> 7 : 3336 : 0.7950089 : Labels ~ 1 + V4557 + V5761 + V9525
#> 8 : 3333 : 0.8331371 : Labels ~ 1 + V698 + V898 + V4685 + V9346 + V9805
#> 9 : 3328 : 0.8004709 : Labels ~ 1 + V1831 + V4200 + V3947
#>
#> Num. Models: 10 To Test: 48 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V6584 + V4183 + V1184 + V7629
#> At Accuracy: Labels ~ 1 + V2556 + V6584 + V4183 + V1184 + V7629
#> B:SWiMS : Labels ~ 1 + V2556 + V6584 + V4183 + V1184 + V7629
#>
#> Num. Models: 320 To Test: 1038 TopFreq: 9.342105 Thrf: 0 Removed: 0
#> ................................*Loop : 2 Blind Cases = 29 Blind Control = 37 Total = 134 Size Cases = 59 Size Control = 75
#> Accumulated Models CV Accuracy = 0.7462687 Sensitivity = 0.7627119 Specificity = 0.7333333 Forw. Ensemble Accuracy= 0.7686567
#> Initial Model Accumulated CV Accuracy = 0.8731343 Sensitivity = 0.8135593 Specificity = 0.92
#> Initial Model Bootstrapped Accuracy = 0.8780165 Sensitivity = 0.8893467 Specificity = 0.8666863
#> Current Model Bootstrapped Accuracy = 0.8468732 Sensitivity = 0.8822326 Specificity = 0.8115137
#> Current KNN Accuracy = 0.761194 Sensitivity = 0.9152542 Specificity = 0.64
#> Initial KNN Accuracy = 0.8134328 Sensitivity = 0.9322034 Specificity = 0.72
#> Train Correlation: 0.7267996 Blind Correlation : 0.7669972
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 49 4
#> TRUE 20 61
#> Loop : 3 Input Cases = 88 Input Control = 112
#> Loop : 3 Train Cases = 59 Train Control = 75
#> Loop : 3 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8440828 : Labels ~ 1 + V478 + V5005 + V7513 + V2256 + V2638 + V1617 + V6597
#> 1 : 3366 : 0.8376258 : Labels ~ 1 + V6163 + V312 + V662 + V1046 + V4352
#> 2 : 3361 : 0.7667949 : Labels ~ 1 + V4215 + V3365 + V4069
#> 3 : 3358 : 0.8481179 : Labels ~ 1 + V2556 + V9833 + V6083 + V4564 + V4960 + V2818 + V2549
#> 4 : 3351 : 0.7803738 : Labels ~ 1 + V2052 + V8368 + V7212 + V9818
#> 5 : 3347 : 0.792561 : Labels ~ 1 + V2309 + V34 + V1865
#> 6 : 3344 : 0.7642288 : Labels ~ 1 + V7891 + V1762 + V9275
#> 7 : 3341 : 0.8048128 : Labels ~ 1 + V9617 + V4584 + V4290 + V867
#> 8 : 3337 : 0.8217262 : Labels ~ 1 + V414 + V5680 + V4295 + V3675
#> 9 : 3333 : 0.7527023 : Labels ~ 1 + V376 + V4326
#>
#> Num. Models: 10 To Test: 42 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V478 + V5005 + V7513 + V2256 + V2638 + V1617 + V6597
#> At Accuracy: Labels ~ 1 + V478 + V5005 + V7513 + V2256 + V2638 + V1617 + V6597
#> B:SWiMS : Labels ~ 1 + V478 + V5005 + V7513 + V2256 + V2638 + V1617 + V6597
#>
#> Num. Models: 320 To Test: 1184 TopFreq: 12.82353 Thrf: 0 Removed: 0
#> ................................*Loop : 3 Blind Cases = 29 Blind Control = 37 Total = 200 Size Cases = 88 Size Control = 112
#> Accumulated Models CV Accuracy = 0.775 Sensitivity = 0.7613636 Specificity = 0.7857143 Forw. Ensemble Accuracy= 0.785
#> Initial Model Accumulated CV Accuracy = 0.875 Sensitivity = 0.8522727 Specificity = 0.8928571
#> Initial Model Bootstrapped Accuracy = 0.8813858 Sensitivity = 0.8857898 Specificity = 0.8769818
#> Current Model Bootstrapped Accuracy = 0.8440828 Sensitivity = 0.8508876 Specificity = 0.8372781
#> Current KNN Accuracy = 0.755 Sensitivity = 0.9204545 Specificity = 0.625
#> Initial KNN Accuracy = 0.795 Sensitivity = 0.8863636 Specificity = 0.7232143
#> Train Correlation: 0.7619706 Blind Correlation : 0.8150089
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 73 4
#> TRUE 36 87
#> Loop : 4 Input Cases = 88 Input Control = 112
#> Loop : 4 Train Cases = 58 Train Control = 74
#> Loop : 4 Blind Cases = 30 Blind Control = 38
#> K : 11 KNN T Cases = 58 KNN T Control = 58
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8010386 : Labels ~ 1 + V533 + V9275 + V3629 + V2529
#> 1 : 3369 : 0.7609202 : Labels ~ 1 + V5457 + V7378 + V9215
#> 2 : 3366 : 0.7684854 : Labels ~ 1 + V8089 + V1400 + V1831
#> 3 : 3363 : 0.769265 : Labels ~ 1 + V5796 + V872 + V9818
#> 4 : 3360 : 0.8074304 : Labels ~ 1 + V1516 + V4733 + V6584 + V2556
#> 5 : 3356 : 0.7646269 : Labels ~ 1 + V1510
#> 6 : 3355 : 0.7598366 : Labels ~ 1 + V7124
#> 7 : 3354 : 0.8157042 : Labels ~ 1 + V3952 + V9735 + V9965 + V4352
#> 8 : 3350 : 0.779628 : Labels ~ 1 + V256 + V4973 + V5801
#> 9 : 3347 : 0.8365639 : Labels ~ 1 + V6473 + V5005 + V1072 + V8156 + V7716 + V4522
#>
#> Num. Models: 10 To Test: 32 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V533 + V9275 + V3629 + V2529
#> At Accuracy: Labels ~ 1 + V533 + V9275 + V3629 + V2529
#> B:SWiMS : Labels ~ 1 + V533 + V9275 + V3629 + V2529
#>
#> Num. Models: 320 To Test: 1115 TopFreq: 15.97826 Thrf: 0 Removed: 0
#> ................................*Loop : 4 Blind Cases = 30 Blind Control = 38 Total = 268 Size Cases = 118 Size Control = 150
#> Accumulated Models CV Accuracy = 0.7649254 Sensitivity = 0.7457627 Specificity = 0.78 Forw. Ensemble Accuracy= 0.7835821
#> Initial Model Accumulated CV Accuracy = 0.880597 Sensitivity = 0.8728814 Specificity = 0.8866667
#> Initial Model Bootstrapped Accuracy = 0.8573534 Sensitivity = 0.8603006 Specificity = 0.8544061
#> Current Model Bootstrapped Accuracy = 0.8010386 Sensitivity = 0.8181009 Specificity = 0.7839763
#> Current KNN Accuracy = 0.7350746 Sensitivity = 0.8389831 Specificity = 0.6533333
#> Initial KNN Accuracy = 0.7910448 Sensitivity = 0.8728814 Specificity = 0.7266667
#> Train Correlation: 0.7686809 Blind Correlation : 0.6777112
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 102 15
#> TRUE 45 106
#> Loop : 5 Input Cases = 88 Input Control = 112
#> Loop : 5 Train Cases = 59 Train Control = 75
#> Loop : 5 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7629046 : Labels ~ 1 + V5005 + V6440 + V5400 + V9818
#> 1 : 3369 : 0.8326397 : Labels ~ 1 + V7513 + V7891 + V1831 + V2868 + V3161 + V2665 + V5321
#> 2 : 3362 : 0.7637555 : Labels ~ 1 + V723 + V2556 + V6408 + V9275
#> 3 : 3358 : 0.8220191 : Labels ~ 1 + V1936 + V4290 + V9215 + V8038 + V4564 + V7641
#> 4 : 3352 : 0.7593355 : Labels ~ 1 + V5473 + V8378 + V867
#> 5 : 3349 : 0.7896674 : Labels ~ 1 + V8502 + V7745 + V8156 + V1512 + V2358
#> 6 : 3344 : 0.7560976 : Labels ~ 1 + V819 + V764 + V9970
#> 7 : 3341 : 0.7678046 : Labels ~ 1 + V698 + V9395 + V6584
#> 8 : 3338 : 0.7589418 : Labels ~ 1 + V9485 + V2462 + V6163
#> 9 : 3335 : 0.7833679 : Labels ~ 1 + V7748 + V5761 + V3857
#>
#> Num. Models: 10 To Test: 41 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V6440 + V5400 + V9818
#> At Accuracy: Labels ~ 1 + V5005 + V6440 + V5400 + V9818
#> B:SWiMS : Labels ~ 1 + V5005 + V6440 + V5400 + V9818
#>
#> Num. Models: 320 To Test: 1113 TopFreq: 11.97059 Thrf: 0 Removed: 0
#> ................................*Loop : 5 Blind Cases = 29 Blind Control = 37 Total = 334 Size Cases = 147 Size Control = 187
#> Accumulated Models CV Accuracy = 0.7814371 Sensitivity = 0.7823129 Specificity = 0.7807487 Forw. Ensemble Accuracy= 0.8053892
#> Initial Model Accumulated CV Accuracy = 0.8772455 Sensitivity = 0.8571429 Specificity = 0.8930481
#> Initial Model Bootstrapped Accuracy = 0.8823874 Sensitivity = 0.8750731 Specificity = 0.8897016
#> Current Model Bootstrapped Accuracy = 0.7629046 Sensitivity = 0.7970254 Specificity = 0.7287839
#> Current KNN Accuracy = 0.7335329 Sensitivity = 0.8367347 Specificity = 0.6524064
#> Initial KNN Accuracy = 0.8053892 Sensitivity = 0.8911565 Specificity = 0.7379679
#> Train Correlation: 0.6782263 Blind Correlation : 0.7907108
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 127 19
#> TRUE 51 137
#> Loop : 6 Input Cases = 88 Input Control = 112
#> Loop : 6 Train Cases = 59 Train Control = 75
#> Loop : 6 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8891947 : Labels ~ 1 + V7857 + V1831 + V6163 + V3987 + V8222 + V312 + V4564
#> 1 : 3366 : 0.8425382 : Labels ~ 1 + V8806 + V9884 + V7319 + V5107 + V2556
#> 2 : 3361 : 0.8743799 : Labels ~ 1 + V8368 + V7220 + V4584 + V9213 + V1184
#> 3 : 3356 : 0.8182351 : Labels ~ 1 + V86 + V1529 + V729
#> 4 : 3353 : 0.8553922 : Labels ~ 1 + V6428 + V6959 + V3365 + V376 + V3170
#> 5 : 3348 : 0.8150805 : Labels ~ 1 + V436 + V3014 + V4666
#> 6 : 3345 : 0.8098653 : Labels ~ 1 + V1476 + V4326 + V62 + V5346
#> 7 : 3341 : 0.8319093 : Labels ~ 1 + V7748 + V3272 + V5683 + V9276 + V6572
#> 8 : 3336 : 0.8614406 : Labels ~ 1 + V698 + V2309 + V1097 + V4301 + V4352 + V1397 + V6573
#> 9 : 3329 : 0.8144764 : Labels ~ 1 + V8048 + V4960 + V5761 + V1344
#>
#> Num. Models: 10 To Test: 48 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V7857 + V9884 + V1831 + V6163 + V3987 + V8222 + V312 + V6440 + V4564 + V630
#> At Accuracy: Labels ~ 1 + V7857 + V1831 + V6163 + V3987 + V8222 + V312 + V4564
#> B:SWiMS : Labels ~ 1 + V7857 + V1831 + V6163 + V3987 + V8222 + V312 + V4564
#>
#> Num. Models: 320 To Test: 1030 TopFreq: 13.69388 Thrf: 0 Removed: 0
#> ................................*Loop : 6 Blind Cases = 29 Blind Control = 37 Total = 400 Size Cases = 176 Size Control = 224
#> Accumulated Models CV Accuracy = 0.7675 Sensitivity = 0.7670455 Specificity = 0.7678571 Forw. Ensemble Accuracy= 0.7975
#> Initial Model Accumulated CV Accuracy = 0.88 Sensitivity = 0.8636364 Specificity = 0.8928571
#> Initial Model Bootstrapped Accuracy = 0.887456 Sensitivity = 0.8889215 Specificity = 0.8859906
#> Current Model Bootstrapped Accuracy = 0.8891947 Sensitivity = 0.8878911 Specificity = 0.8904983
#> Current KNN Accuracy = 0.7325 Sensitivity = 0.8465909 Specificity = 0.6428571
#> Initial KNN Accuracy = 0.805 Sensitivity = 0.8920455 Specificity = 0.7366071
#> Train Correlation: 0.8266013 Blind Correlation : 0.780691
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 152 19
#> TRUE 61 168
#> Loop : 7 Input Cases = 88 Input Control = 112
#> Loop : 7 Train Cases = 58 Train Control = 74
#> Loop : 7 Blind Cases = 30 Blind Control = 38
#> K : 11 KNN T Cases = 58 KNN T Control = 58
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7811671 : Labels ~ 1 + V1762 + V5005 + V2256
#> 1 : 3370 : 0.8465171 : Labels ~ 1 + V6403 + V3629 + V698 + V1831 + V729 + V6520 + V4738
#> 2 : 3363 : 0.8510289 : Labels ~ 1 + V4290 + V5796 + V8156 + V9070 + V5982 + V5147 + V1289
#> 3 : 3356 : 0.8292358 : Labels ~ 1 + V1400 + V7857 + V3014 + V3545 + V2438
#> 4 : 3351 : 0.8390839 : Labels ~ 1 + V1516 + V7748 + V9965 + V4908 + V3547 + V4708
#> 5 : 3345 : 0.8704196 : Labels ~ 1 + V1476 + V9275 + V723 + V1072 + V8720 + V2597 + V6567 + V9672
#> 6 : 3337 : 0.8848098 : Labels ~ 1 + V436 + V5400 + V9818 + V1584 + V2309 + V6594 + V716
#> 7 : 3330 : 0.7756278 : Labels ~ 1 + V7513 + V8806 + V9215
#> 8 : 3327 : 0.8186054 : Labels ~ 1 + V86 + V5761 + V6597 + V6688
#> 9 : 3323 : 0.8217496 : Labels ~ 1 + V7220 + V3952 + V3170 + V5680 + V3866
#>
#> Num. Models: 10 To Test: 55 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V1762 + V5005 + V2256
#> At Accuracy: Labels ~ 1 + V1762 + V5005 + V2256
#> B:SWiMS : Labels ~ 1 + V1762 + V5005 + V2256
#>
#> Num. Models: 320 To Test: 1060 TopFreq: 16.5 Thrf: 0 Removed: 0
#> ................................*Loop : 7 Blind Cases = 30 Blind Control = 38 Total = 468 Size Cases = 206 Size Control = 262
#> Accumulated Models CV Accuracy = 0.7692308 Sensitivity = 0.7524272 Specificity = 0.7824427 Forw. Ensemble Accuracy= 0.7970085
#> Initial Model Accumulated CV Accuracy = 0.8824786 Sensitivity = 0.8543689 Specificity = 0.9045802
#> Initial Model Bootstrapped Accuracy = 0.873252 Sensitivity = 0.8809878 Specificity = 0.8655162
#> Current Model Bootstrapped Accuracy = 0.7811671 Sensitivity = 0.8414383 Specificity = 0.720896
#> Current KNN Accuracy = 0.7350427 Sensitivity = 0.8349515 Specificity = 0.6564885
#> Initial KNN Accuracy = 0.8119658 Sensitivity = 0.9029126 Specificity = 0.740458
#> Train Correlation: 0.6099607 Blind Correlation : 0.7212658
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 183 23
#> TRUE 73 189
#> Loop : 8 Input Cases = 88 Input Control = 112
#> Loop : 8 Train Cases = 59 Train Control = 75
#> Loop : 8 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8556473 : Labels ~ 1 + V2556 + V22 + V312 + V9275 + V4406 + V4352
#> 1 : 3367 : 0.7690033 : Labels ~ 1 + V7891 + V4326 + V6083
#> 2 : 3364 : 0.8061373 : Labels ~ 1 + V6594 + V3014 + V3218 + V4301 + V2358
#> 3 : 3359 : 0.7749488 : Labels ~ 1 + V5808 + V9215 + V4900
#> 4 : 3356 : 0.8195544 : Labels ~ 1 + V3365 + V7319 + V8402 + V762 + V1046
#> 5 : 3351 : 0.8335308 : Labels ~ 1 + V8368 + V898 + V8117 + V3170 + V5774 + V5881 + V8640
#> 6 : 3344 : 0.7900875 : Labels ~ 1 + V4960 + V5761 + V5995
#> 7 : 3341 : 0.7248244 : Labels ~ 1 + V414 + V8055
#> 8 : 3339 : 0.7323041 : Labels ~ 1 + V2309 + V5680
#> 9 : 3337 : 0.7914844 : Labels ~ 1 + V5005 + V1831 + V754 + V2804 + V3778
#>
#> Num. Models: 10 To Test: 41 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V22 + V312 + V9275 + V4406 + V4352
#> At Accuracy: Labels ~ 1 + V2556 + V22 + V312 + V9275 + V4406 + V4352
#> B:SWiMS : Labels ~ 1 + V2556 + V22 + V312 + V9275 + V4406 + V4352
#>
#> Num. Models: 320 To Test: 1115 TopFreq: 10.31579 Thrf: 0 Removed: 0
#> ................................*Loop : 8 Blind Cases = 29 Blind Control = 37 Total = 534 Size Cases = 235 Size Control = 299
#> Accumulated Models CV Accuracy = 0.7771536 Sensitivity = 0.7659574 Specificity = 0.7859532 Forw. Ensemble Accuracy= 0.8089888
#> Initial Model Accumulated CV Accuracy = 0.8895131 Sensitivity = 0.8723404 Specificity = 0.90301
#> Initial Model Bootstrapped Accuracy = 0.8608901 Sensitivity = 0.867374 Specificity = 0.8544061
#> Current Model Bootstrapped Accuracy = 0.8556473 Sensitivity = 0.846358 Specificity = 0.8649366
#> Current KNN Accuracy = 0.7453184 Sensitivity = 0.8382979 Specificity = 0.6722408
#> Initial KNN Accuracy = 0.8146067 Sensitivity = 0.8978723 Specificity = 0.7491639
#> Train Correlation: 0.8315291 Blind Correlation : 0.8026093
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 211 28
#> TRUE 79 216
#> Loop : 9 Input Cases = 88 Input Control = 112
#> Loop : 9 Train Cases = 59 Train Control = 75
#> Loop : 9 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8306878 : Labels ~ 1 + V2358 + V4194 + V5680 + V4973 + V8193
#> 1 : 3368 : 0.8420124 : Labels ~ 1 + V3628 + V8156 + V1476 + V3987
#> 2 : 3364 : 0.8930771 : Labels ~ 1 + V1055 + V3170 + V9659 + V6163 + V2880 + V7161
#> 3 : 3358 : 0.8169487 : Labels ~ 1 + V5457 + V3339 + V9818
#> 4 : 3355 : 0.8121909 : Labels ~ 1 + V7748 + V533 + V4414 + V9275
#> 5 : 3351 : 0.8064469 : Labels ~ 1 + V436 + V3206 + V723 + V9215
#> 6 : 3347 : 0.798153 : Labels ~ 1 + V6986 + V4733 + V1831
#> 7 : 3344 : 0.7861305 : Labels ~ 1 + V2085 + V872 + V9965
#> 8 : 3341 : 0.8612001 : Labels ~ 1 + V4040 + V2637 + V8849 + V4301 + V2597
#> 9 : 3336 : 0.8018785 : Labels ~ 1 + V6523 + V893 + V819
#>
#> Num. Models: 10 To Test: 40 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2358 + V4194 + V5680 + V4973 + V8193
#> At Accuracy: Labels ~ 1 + V2358 + V4194 + V5680 + V4973 + V8193
#> B:SWiMS : Labels ~ 1 + V2358 + V4194 + V5680 + V4973 + V8193
#>
#> Num. Models: 320 To Test: 1148 TopFreq: 11.40625 Thrf: 0 Removed: 0
#> ................................*Loop : 9 Blind Cases = 29 Blind Control = 37 Total = 600 Size Cases = 264 Size Control = 336
#> Accumulated Models CV Accuracy = 0.77 Sensitivity = 0.7689394 Specificity = 0.7708333 Forw. Ensemble Accuracy= 0.7966667
#> Initial Model Accumulated CV Accuracy = 0.885 Sensitivity = 0.8712121 Specificity = 0.8958333
#> Initial Model Bootstrapped Accuracy = 0.8694371 Sensitivity = 0.8614795 Specificity = 0.8773946
#> Current Model Bootstrapped Accuracy = 0.8306878 Sensitivity = 0.8677249 Specificity = 0.7936508
#> Current KNN Accuracy = 0.735 Sensitivity = 0.844697 Specificity = 0.6488095
#> Initial KNN Accuracy = 0.7933333 Sensitivity = 0.8901515 Specificity = 0.7172619
#> Train Correlation: 0.7336195 Blind Correlation : 0.6606617
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 231 28
#> TRUE 89 252
#> Loop : 10 Input Cases = 88 Input Control = 112
#> Loop : 10 Train Cases = 58 Train Control = 74
#> Loop : 10 Blind Cases = 30 Blind Control = 38
#> K : 11 KNN T Cases = 58 KNN T Control = 58
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8320962 : Labels ~ 1 + V5005 + V9215 + V5 + V1184
#> 1 : 3369 : 0.8314793 : Labels ~ 1 + V9818 + V312 + V7849 + V4301 + V1289
#> 2 : 3364 : 0.784416 : Labels ~ 1 + V8156 + V8368 + V1936
#> 3 : 3361 : 0.8240444 : Labels ~ 1 + V5761 + V8806 + V9213 + V6688
#> 4 : 3357 : 0.8162722 : Labels ~ 1 + V1046 + V3365 + V5417 + V311
#> 5 : 3353 : 0.8167777 : Labels ~ 1 + V4960 + V9275 + V1248 + V7857
#> 6 : 3349 : 0.8350118 : Labels ~ 1 + V1831 + V2309 + V6111 + V698 + V666
#> 7 : 3344 : 0.8424855 : Labels ~ 1 + V7062 + V9617 + V4580 + V2515 + V7748
#> 8 : 3339 : 0.7733076 : Labels ~ 1 + V819 + V414 + V2785
#> 9 : 3336 : 0.8511799 : Labels ~ 1 + V4070 + V7584 + V86 + V9965 + V2973 + V241
#>
#> Num. Models: 10 To Test: 43 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V9215 + V5 + V1184
#> At Accuracy: Labels ~ 1 + V5005 + V9215 + V5 + V1184
#> B:SWiMS : Labels ~ 1 + V5005 + V9215 + V5 + V1184
#>
#> Num. Models: 320 To Test: 1037 TopFreq: 17.53333 Thrf: 0 Removed: 0
#> ................................*Loop : 10 Blind Cases = 30 Blind Control = 38 Total = 668 Size Cases = 294 Size Control = 374
#> Accumulated Models CV Accuracy = 0.7724551 Sensitivity = 0.7619048 Specificity = 0.7807487 Forw. Ensemble Accuracy= 0.7964072
#> Initial Model Accumulated CV Accuracy = 0.8907186 Sensitivity = 0.877551 Specificity = 0.9010695
#> Initial Model Bootstrapped Accuracy = 0.8719821 Sensitivity = 0.8769001 Specificity = 0.8670641
#> Current Model Bootstrapped Accuracy = 0.8320962 Sensitivity = 0.8660926 Specificity = 0.7980998
#> Current KNN Accuracy = 0.741018 Sensitivity = 0.8435374 Specificity = 0.6604278
#> Initial KNN Accuracy = 0.7904192 Sensitivity = 0.8741497 Specificity = 0.7245989
#> Train Correlation: 0.77465 Blind Correlation : 0.8332634
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 262 31
#> TRUE 100 275
#> Loop : 11 Input Cases = 88 Input Control = 112
#> Loop : 11 Train Cases = 59 Train Control = 75
#> Loop : 11 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8369016 : Labels ~ 1 + V6115 + V5761 + V312 + V4564 + V630 + V2064
#> 1 : 3367 : 0.7863938 : Labels ~ 1 + V469 + V4584 + V9027
#> 2 : 3364 : 0.8506647 : Labels ~ 1 + V7928 + V4352 + V9884 + V8055 + V9234 + V4549 + V5466
#> 3 : 3357 : 0.8146413 : Labels ~ 1 + V723 + V4960 + V9275 + V6163 + V1865
#> 4 : 3352 : 0.8071915 : Labels ~ 1 + V698 + V8720 + V1097 + V6999 + V3257
#> 5 : 3347 : 0.7612269 : Labels ~ 1 + V1975 + V2168 + V414 + V22
#> 6 : 3343 : 0.7723318 : Labels ~ 1 + V4183 + V5680 + V5077
#> 7 : 3340 : 0.7936716 : Labels ~ 1 + V872 + V5378 + V9213 + V6009
#> 8 : 3336 : 0.8002603 : Labels ~ 1 + V5 + V9818 + V7402 + V3152
#> 9 : 3332 : 0.771849 : Labels ~ 1 + V782 + V4326 + V5487 + V2145
#>
#> Num. Models: 10 To Test: 45 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V6115 + V5761 + V312 + V4564 + V630 + V2064
#> At Accuracy: Labels ~ 1 + V6115 + V5761 + V312 + V4564 + V630 + V2064
#> B:SWiMS : Labels ~ 1 + V6115 + V5761 + V312 + V4564 + V630 + V2064
#>
#> Num. Models: 320 To Test: 1164 TopFreq: 9.418605 Thrf: 0 Removed: 0
#> ................................*Loop : 11 Blind Cases = 29 Blind Control = 37 Total = 734 Size Cases = 323 Size Control = 411
#> Accumulated Models CV Accuracy = 0.7752044 Sensitivity = 0.7616099 Specificity = 0.7858881 Forw. Ensemble Accuracy= 0.7997275
#> Initial Model Accumulated CV Accuracy = 0.886921 Sensitivity = 0.8854489 Specificity = 0.8880779
#> Initial Model Bootstrapped Accuracy = 0.8783824 Sensitivity = 0.8786798 Specificity = 0.878085
#> Current Model Bootstrapped Accuracy = 0.8369016 Sensitivity = 0.8614332 Specificity = 0.8123699
#> Current KNN Accuracy = 0.739782 Sensitivity = 0.8544892 Specificity = 0.649635
#> Initial KNN Accuracy = 0.7901907 Sensitivity = 0.879257 Specificity = 0.7201946
#> Train Correlation: 0.7429644 Blind Correlation : 0.7657865
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 283 31
#> TRUE 117 303
#> Loop : 12 Input Cases = 88 Input Control = 112
#> Loop : 12 Train Cases = 59 Train Control = 75
#> Loop : 12 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8321534 : Labels ~ 1 + V2556 + V256 + V1055 + V6198
#> 1 : 3369 : 0.8557185 : Labels ~ 1 + V7891 + V9275 + V4158 + V9338 + V1198 + V2700
#> 2 : 3363 : 0.8465332 : Labels ~ 1 + V4290 + V9818 + V7272 + V4900 + V2294 + V8026
#> 3 : 3357 : 0.8461425 : Labels ~ 1 + V7748 + V729 + V4194 + V6511 + V6440
#> 4 : 3352 : 0.8076923 : Labels ~ 1 + V698 + V4069 + V7976 + V9215
#> 5 : 3348 : 0.8350607 : Labels ~ 1 + V8502 + V2256 + V4654 + V6584 + V7598
#> 6 : 3343 : 0.8011995 : Labels ~ 1 + V1476 + V3952 + V1831 + V2747
#> 7 : 3339 : 0.852194 : Labels ~ 1 + V3206 + V9965 + V6181 + V8156 + V7195 + V6180
#> 8 : 3333 : 0.7685564 : Labels ~ 1 + V5473 + V5761 + V4056
#> 9 : 3330 : 0.7673253 : Labels ~ 1 + V436 + V5680 + V3365
#>
#> Num. Models: 10 To Test: 46 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V256 + V1055 + V6198
#> At Accuracy: Labels ~ 1 + V2556 + V256 + V1055 + V6198
#> B:SWiMS : Labels ~ 1 + V2556 + V256 + V1055 + V6198
#>
#> Num. Models: 320 To Test: 1184 TopFreq: 12.39286 Thrf: 0 Removed: 0
#> ................................*Loop : 12 Blind Cases = 29 Blind Control = 37 Total = 800 Size Cases = 352 Size Control = 448
#> Accumulated Models CV Accuracy = 0.7775 Sensitivity = 0.7727273 Specificity = 0.78125 Forw. Ensemble Accuracy= 0.805
#> Initial Model Accumulated CV Accuracy = 0.88625 Sensitivity = 0.8863636 Specificity = 0.8861607
#> Initial Model Bootstrapped Accuracy = 0.8730417 Sensitivity = 0.8785102 Specificity = 0.8675732
#> Current Model Bootstrapped Accuracy = 0.8321534 Sensitivity = 0.8427729 Specificity = 0.8215339
#> Current KNN Accuracy = 0.73625 Sensitivity = 0.8551136 Specificity = 0.6428571
#> Initial KNN Accuracy = 0.78625 Sensitivity = 0.8863636 Specificity = 0.7075893
#> Train Correlation: 0.7915136 Blind Correlation : 0.7823192
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 304 35
#> TRUE 126 335
#> Loop : 13 Input Cases = 88 Input Control = 112
#> Loop : 13 Train Cases = 58 Train Control = 74
#> Loop : 13 Blind Cases = 30 Blind Control = 38
#> K : 11 KNN T Cases = 58 KNN T Control = 58
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8643585 : Labels ~ 1 + V5005 + V9884 + V1184 + V1831 + V86 + V1128
#> 1 : 3367 : 0.8523031 : Labels ~ 1 + V1529 + V4352 + V7857 + V9395 + V1046 + V629 + V7003
#> 2 : 3360 : 0.8735955 : Labels ~ 1 + V5825 + V5761 + V1476 + V2515 + V9213
#> 3 : 3355 : 0.8834027 : Labels ~ 1 + V3628 + V8806 + V4326 + V1248 + V3170 + V7220
#> 4 : 3349 : 0.8729642 : Labels ~ 1 + V436 + V3152 + V5417 + V7446 + V5139 + V6688
#> 5 : 3343 : 0.8757476 : Labels ~ 1 + V7748 + V5680 + V9104 + V6111 + V4301
#> 6 : 3338 : 0.8751486 : Labels ~ 1 + V698 + V9050 + V9818 + V6780 + V1289 + V3592
#> 7 : 3332 : 0.845735 : Labels ~ 1 + V4290 + V312 + V9063 + V7062 + V1667
#> 8 : 3327 : 0.8229322 : Labels ~ 1 + V8502 + V7899 + V9215 + V662
#> 9 : 3323 : 0.8577778 : Labels ~ 1 + V6428 + V3365 + V7849 + V8156 + V4580
#>
#> Num. Models: 10 To Test: 55 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V9884 + V1184 + V1831 + V86 + V1128
#> At Accuracy: Labels ~ 1 + V5005 + V9884 + V1184 + V1831 + V86 + V1128
#> B:SWiMS : Labels ~ 1 + V5005 + V9884 + V1184 + V1831 + V86 + V1128
#>
#> Num. Models: 320 To Test: 1098 TopFreq: 14.8125 Thrf: 0 Removed: 0
#> ................................*Loop : 13 Blind Cases = 30 Blind Control = 38 Total = 868 Size Cases = 382 Size Control = 486
#> Accumulated Models CV Accuracy = 0.7753456 Sensitivity = 0.7670157 Specificity = 0.781893 Forw. Ensemble Accuracy= 0.8029954
#> Initial Model Accumulated CV Accuracy = 0.8894009 Sensitivity = 0.8874346 Specificity = 0.8909465
#> Initial Model Bootstrapped Accuracy = 0.8701356 Sensitivity = 0.8711675 Specificity = 0.8691038
#> Current Model Bootstrapped Accuracy = 0.8643585 Sensitivity = 0.8755995 Specificity = 0.8531175
#> Current KNN Accuracy = 0.7327189 Sensitivity = 0.8507853 Specificity = 0.6399177
#> Initial KNN Accuracy = 0.7834101 Sensitivity = 0.8874346 Specificity = 0.7016461
#> Train Correlation: 0.8638164 Blind Correlation : 0.6517158
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 331 37
#> TRUE 138 362
#> Loop : 14 Input Cases = 88 Input Control = 112
#> Loop : 14 Train Cases = 59 Train Control = 75
#> Loop : 14 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7909224 : Labels ~ 1 + V9215 + V7513 + V3365
#> 1 : 3370 : 0.8374964 : Labels ~ 1 + V1831 + V5400 + V8368 + V7831
#> 2 : 3366 : 0.8321189 : Labels ~ 1 + V8156 + V7212 + V414 + V5321 + V7883
#> 3 : 3361 : 0.8468732 : Labels ~ 1 + V723 + V2556 + V9818 + V7928 + V7402
#> 4 : 3356 : 0.7827748 : Labels ~ 1 + V7891 + V9965 + V1975
#> 5 : 3353 : 0.8193018 : Labels ~ 1 + V9275 + V469 + V312 + V9567
#> 6 : 3349 : 0.8390351 : Labels ~ 1 + V4960 + V6584 + V9585 + V6163 + V5698
#> 7 : 3344 : 0.8036078 : Labels ~ 1 + V9027 + V1097 + V7032 + V2638
#> 8 : 3340 : 0.8199467 : Labels ~ 1 + V7994 + V898 + V8411 + V711 + V5914
#> 9 : 3335 : 0.8182624 : Labels ~ 1 + V7272 + V4973 + V4326 + V2438 + V5995
#>
#> Num. Models: 10 To Test: 43 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V9215 + V7513 + V3365
#> At Accuracy: Labels ~ 1 + V9215 + V7513 + V3365
#> B:SWiMS : Labels ~ 1 + V9215 + V7513 + V3365
#>
#> Num. Models: 320 To Test: 1115 TopFreq: 24.81081 Thrf: 1 Removed: 608
#> ................................*Loop : 14 Blind Cases = 29 Blind Control = 37 Total = 934 Size Cases = 411 Size Control = 523
#> Accumulated Models CV Accuracy = 0.7719486 Sensitivity = 0.7639903 Specificity = 0.7782027 Forw. Ensemble Accuracy= 0.7987152
#> Initial Model Accumulated CV Accuracy = 0.8897216 Sensitivity = 0.892944 Specificity = 0.8871893
#> Initial Model Bootstrapped Accuracy = 0.8686602 Sensitivity = 0.8765758 Specificity = 0.8607446
#> Current Model Bootstrapped Accuracy = 0.7909224 Sensitivity = 0.8333821 Specificity = 0.7484627
#> Current KNN Accuracy = 0.7291221 Sensitivity = 0.8515815 Specificity = 0.6328872
#> Initial KNN Accuracy = 0.7815846 Sensitivity = 0.8905109 Specificity = 0.6959847
#> Train Correlation: 0.8035627 Blind Correlation : 0.7980587
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 354 38
#> TRUE 150 392
#> Loop : 15 Input Cases = 88 Input Control = 112
#> Loop : 15 Train Cases = 59 Train Control = 75
#> Loop : 15 Blind Cases = 29 Blind Control = 37
#> K : 11 KNN T Cases = 59 KNN T Control = 59
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7390607 : Labels ~ 1 + V1936 + V9215 + V7272
#> 1 : 3370 : 0.7126723 : Labels ~ 1 + V5005
#> 2 : 3369 : 0.7598665 : Labels ~ 1 + V5457 + V9818 + V4666
#> 3 : 3366 : 0.794356 : Labels ~ 1 + V533 + V6584 + V3206 + V4301
#> 4 : 3362 : 0.7783909 : Labels ~ 1 + V7891 + V9275 + V5400 + V3170
#> 5 : 3358 : 0.7655565 : Labels ~ 1 + V4290 + V6239 + V1097 + V2747
#> 6 : 3354 : 0.8349572 : Labels ~ 1 + V3352 + V4158 + V4738 + V218 + V1477 + V3983 + V9965
#> 7 : 3347 : 0.8501742 : Labels ~ 1 + V3365 + V4584 + V256 + V4564 + V3675 + V4198
#> 8 : 3341 : 0.7691398 : Labels ~ 1 + V1831 + V2242 + V2256
#> 9 : 3338 : 0.7835447 : Labels ~ 1 + V698 + V8744 + V5774 + V729
#>
#> Num. Models: 10 To Test: 39 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V1936 + V9215 + V7272
#> At Accuracy: Labels ~ 1 + V1936 + V9215 + V7272
#> B:SWiMS : Labels ~ 1 + V1936 + V9215 + V7272
#>
#> Num. Models: 320 To Test: 1120 TopFreq: 9.909091 Thrf: 0 Removed: 0
#> ................................*Loop : 15 Blind Cases = 29 Blind Control = 37 Total = 1000 Size Cases = 440 Size Control = 560
#> Accumulated Models CV Accuracy = 0.779 Sensitivity = 0.7659091 Specificity = 0.7892857 Forw. Ensemble Accuracy= 0.806
#> Initial Model Accumulated CV Accuracy = 0.887 Sensitivity = 0.8818182 Specificity = 0.8910714
#> Initial Model Bootstrapped Accuracy = 0.876545 Sensitivity = 0.8878752 Specificity = 0.8652148
#> Current Model Bootstrapped Accuracy = 0.7390607 Sensitivity = 0.7922987 Specificity = 0.6858226
#> Current KNN Accuracy = 0.737 Sensitivity = 0.8613636 Specificity = 0.6392857
#> Initial KNN Accuracy = 0.784 Sensitivity = 0.8886364 Specificity = 0.7017857
#> Train Correlation: 0.6784956 Blind Correlation : 0.7718401
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 381 38
#> TRUE 164 417
#>
#> Num. Models: 320 To Test: 1334 TopFreq: 13.58621 Thrf: 0 Removed: 0
#> ................................*:.....#.....#:.....#.....#:.....#.....#:.....#.....#:.....#.....#:.....#.....#:.....#.....#:.....#.....#:.....#.....#
#> Num. Models: 101 To Test: 109 TopFreq: 99 Thrf: 0 Removed: 0
#> ..........*
save(arceneCV,file="ArceneV_001_3_5_wsvn.RDATA")
CVACCTable <- NULL
CVBETable <- NULL
arceneCV$cvObject$Models.testPrediction$usrFitFunction_Sel <- arceneCV$cvObject$Models.testPrediction$usrFitFunction_Sel -0.5
arceneCV$cvObject$Models.testPrediction$usrFitFunction <- arceneCV$cvObject$Models.testPrediction$usrFitFunction -0.5
pm <- plotModels.ROC(arceneCV$cvObject$LASSO.testPredictions,theCVfolds=3,main="CV LASSO",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV$cvObject$Models.testPrediction,theCVfolds=3,predictor="Prediction",main="BB:SWiMS",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV$cvObject$Models.testPrediction,theCVfolds=3,predictor="Ensemble.Forward",main="Forward Median",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV$cvObject$Models.testPrediction,theCVfolds=3,predictor="Forward.Selection.Bagged",main="Forward Bagged",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV$cvObject$Models.testPrediction,theCVfolds=3,predictor="usrFitFunction",main="SVM",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV$cvObject$Models.testPrediction,theCVfolds=3,predictor="usrFitFunction_Sel",main="SVM",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
CVthesets <- c("LASSO","BSWIMS","Ensemble","Forward Bagging","SVM:Filterd","SVM:BSWIMS")
bp <- barPlotCiError(as.matrix(CVACCTable),metricname="Accuracy",thesets=CVthesets,themethod=c("CV"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(CVBETable),metricname="Balanced Error",thesets=CVthesets,themethod=c("CV"),main="Balanced Error",args.legend = list(x = "topright"))
arceneCV10 <- FRESA.Model(formula = Labels ~ 1,data = arcene.norm,filter.p.value = 0.01,CVfolds = 10,repeats = 5,bswimsCycles=10,usrFitFun=svm)
#> Unadjusted size: 3373 Adjusted Size: 3535
#> ..
#> Vars: 10000 Size: 100 , Fraction= 0.136, Average random size = 15.60, Size:312.00
#>
#> Z: 2.575829 , Var Max: 3373 , s1: 10001 , s2: 4118 , Independent Size: 624
#> [1] "V3365 + V7748 + V2556 + V1936 + V7513 + V4200 + V7212 + V2804 + V2256 + V52 + V271 + V9818 + V2264 + V4198 + V4352 + V9275 + V1046 + V4295 + V6584 + V6567 + V9402 + V2358 + V3592 + V729 + V6164 + V8026 + V6440 + V5936 + V762 + V22 + V4055 + V5321 + V3161 + V6692 + V7189 + V2533 + V8586 + V754 + V1184 + V6520"
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8570153 : Labels ~ 1 + V5005 + V5680 + V130 + V2145 + V4290 + V1943 + V8245
#> 1 : 3366 : 0.8208178 : Labels ~ 1 + V698 + V3365 + V6163 + V6688 + V6584 + V1344
#> 2 : 3360 : 0.9036778 : Labels ~ 1 + V312 + V9275 + V4584 + V3161 + V9287 + V5321 + V2700 + V4198 + V3725
#> 3 : 3351 : 0.8088065 : Labels ~ 1 + V7891 + V9818 + V1975 + V6780 + V4352
#> 4 : 3346 : 0.8592629 : Labels ~ 1 + V8368 + V7748 + V9215 + V5400 + V729 + V8726 + V6948 + V5140
#> 5 : 3338 : 0.8155512 : Labels ~ 1 + V1476 + V2309 + V1831 + V9395 + V9268
#> 6 : 3333 : 0.8557692 : Labels ~ 1 + V436 + V4960 + V256 + V2515 + V3778 + V3592 + V1173 + V8835
#> 7 : 3325 : 0.8320328 : Labels ~ 1 + V2556 + V3014 + V414 + V5995 + V6083 + V4301
#> 8 : 3319 : 0.8103516 : Labels ~ 1 + V9617 + V5378 + V4973 + V22 + V3170 + V4900
#> 9 : 3313 : 0.8310624 : Labels ~ 1 + V8502 + V4326 + V8411 + V9027 + V1248 + V1046 + V6746
#>
#> Num. Models: 10 To Test: 67 TopFreq: 1 Thrf: 0 Removed: 0
#> .*CV pvalue : 0.05
#> Update : Labels ~ 1 + V5005 + V1975 + V5680 + V130 + V2145 + V4290 + V1943 + V8245
#> At Accuray: Labels ~ 1 + V5005 + V5680 + V130 + V2145 + V4290 + V1943 + V8245
#> B:SWiMS : Labels ~ 1 + V5005 + V5680 + V130 + V2145 + V4290 + V1943 + V8245
#> Loop : 1 Input Cases = 88 Input Control = 112
#> Loop : 1 Train Cases = 79 Train Control = 100
#> Loop : 1 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8535801 : Labels ~ 1 + V5005 + V4352 + V8368 + V762 + V9818 + V7584 + V3778
#> 1 : 3366 : 0.8450751 : Labels ~ 1 + V2556 + V9275 + V6111 + V5321 + V6163 + V3206 + V3170
#> 2 : 3359 : 0.8385073 : Labels ~ 1 + V7891 + V4584 + V698 + V7272 + V6584 + V880 + V1248
#> 3 : 3352 : 0.8434631 : Labels ~ 1 + V4290 + V3365 + V4326 + V9395 + V1344 + V22
#> 4 : 3346 : 0.8225099 : Labels ~ 1 + V7748 + V312 + V9215 + V1943 + V819 + V9086
#> 5 : 3340 : 0.8453722 : Labels ~ 1 + V2309 + V1831 + V5774 + V4406 + V2294 + V9965 + V6487
#> 6 : 3333 : 0.7953992 : Labels ~ 1 + V4960 + V729 + V436
#> 7 : 3330 : 0.8057769 : Labels ~ 1 + V6594 + V9213 + V5761 + V3015 + V9743 + V2358
#> 8 : 3324 : 0.788609 : Labels ~ 1 + V9617 + V4069 + V1320 + V4301 + V2064
#> 9 : 3319 : 0.7946881 : Labels ~ 1 + V1476 + V414 + V3708 + V5680
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V4352 + V8368 + V762 + V9818 + V7584 + V3778
#> At Accuracy: Labels ~ 1 + V5005 + V4352 + V8368 + V762 + V9818 + V7584 + V3778
#> B:SWiMS : Labels ~ 1 + V5005 + V4352 + V8368 + V762 + V9818 + V7584 + V3778
#>
#> Num. Models: 320 To Test: 1299 TopFreq: 13.97561 Thrf: 0 Removed: 0
#> ................................*Loop : 1 Blind Cases = 9 Blind Control = 12 Total = 21 Size Cases = 9 Size Control = 12
#> Accumulated Models CV Accuracy = 0.8571429 Sensitivity = 0.8888889 Specificity = 0.8333333 Forw. Ensemble Accuracy= 0.8095238
#> Initial Model Accumulated CV Accuracy = 0.8095238 Sensitivity = 0.8888889 Specificity = 0.75
#> Initial Model Bootstrapped Accuracy = 0.8636564 Sensitivity = 0.8797357 Specificity = 0.8475771
#> Current Model Bootstrapped Accuracy = 0.8535801 Sensitivity = 0.859898 Specificity = 0.8472622
#> Current KNN Accuracy = 0.6666667 Sensitivity = 1 Specificity = 0.4166667
#> Initial KNN Accuracy = 0.7142857 Sensitivity = 0.7777778 Specificity = 0.6666667
#> Train Correlation: 0.7407675 Blind Correlation : 0.9012987
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 5 0
#> TRUE 6 10
#> Loop : 2 Input Cases = 88 Input Control = 112
#> Loop : 2 Train Cases = 79 Train Control = 100
#> Loop : 2 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8870609 : Labels ~ 1 + V4194 + V9818 + V7857 + V4584 + V3161 + V4564 + V5489 + V6196
#> 1 : 3365 : 0.8524156 : Labels ~ 1 + V5400 + V5005 + V819 + V1184 + V7272 + V629 + V8193
#> 2 : 3358 : 0.8203279 : Labels ~ 1 + V7513 + V3170 + V5683 + V7748
#> 3 : 3354 : 0.8306398 : Labels ~ 1 + V1975 + V1936 + V9275 + V4301 + V6570
#> 4 : 3349 : 0.8461624 : Labels ~ 1 + V7212 + V436 + V9215 + V2358 + V9082 + V7284
#> 5 : 3343 : 0.8821255 : Labels ~ 1 + V86 + V723 + V898 + V8368 + V1584 + V5808 + V1198
#> 6 : 3336 : 0.8437638 : Labels ~ 1 + V8806 + V4326 + V3365 + V4406 + V762
#> 7 : 3331 : 0.8449074 : Labels ~ 1 + V1476 + V1097 + V5801 + V1055 + V7584 + V1046
#> 8 : 3325 : 0.8340684 : Labels ~ 1 + V4183 + V256 + V2309 + V4580 + V729 + V6882
#> 9 : 3319 : 0.8049288 : Labels ~ 1 + V698 + V7928 + V8055 + V3206
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V4194 + V9818 + V7857 + V7513 + V4584 + V3161 + V4564 + V5489 + V6196
#> At Accuracy: Labels ~ 1 + V4194 + V9818 + V7857 + V4584 + V3161 + V4564 + V5489 + V6196
#> B:SWiMS : Labels ~ 1 + V4194 + V9818 + V7857 + V4584 + V3161 + V4564 + V5489 + V6196
#>
#> Num. Models: 320 To Test: 1210 TopFreq: 11.86047 Thrf: 0 Removed: 0
#> ................................*Loop : 2 Blind Cases = 9 Blind Control = 12 Total = 42 Size Cases = 18 Size Control = 24
#> Accumulated Models CV Accuracy = 0.6904762 Sensitivity = 0.6111111 Specificity = 0.75 Forw. Ensemble Accuracy= 0.7619048
#> Initial Model Accumulated CV Accuracy = 0.7857143 Sensitivity = 0.7777778 Specificity = 0.7916667
#> Initial Model Bootstrapped Accuracy = 0.8649546 Sensitivity = 0.8817799 Specificity = 0.8481293
#> Current Model Bootstrapped Accuracy = 0.8870609 Sensitivity = 0.8955091 Specificity = 0.8786127
#> Current KNN Accuracy = 0.6904762 Sensitivity = 0.9444444 Specificity = 0.5
#> Initial KNN Accuracy = 0.7380952 Sensitivity = 0.8333333 Specificity = 0.6666667
#> Train Correlation: 0.8028791 Blind Correlation : 0.4597403
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 13 0
#> TRUE 12 17
#> Loop : 3 Input Cases = 88 Input Control = 112
#> Loop : 3 Train Cases = 79 Train Control = 101
#> Loop : 3 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8251189 : Labels ~ 1 + V4685 + V9215 + V4158 + V8604 + V1344
#> 1 : 3368 : 0.8607204 : Labels ~ 1 + V2556 + V698 + V1831 + V5796 + V729 + V3206 + V3161 + V1184
#> 2 : 3360 : 0.8077251 : Labels ~ 1 + V7891 + V1400 + V4301 + V9818 + V6163 + V2747
#> 3 : 3354 : 0.8536985 : Labels ~ 1 + V4352 + V8411 + V3170 + V5761 + V6505 + V5321 + V4077 + V8245 + V3839
#> 4 : 3345 : 0.8194962 : Labels ~ 1 + V7748 + V8055 + V312 + V6688 + V7137
#> 5 : 3340 : 0.8147988 : Labels ~ 1 + V436 + V9275 + V9485 + V7272 + V7435 + V5139
#> 6 : 3334 : 0.8508715 : Labels ~ 1 + V7857 + V1055 + V8156 + V6959 + V3785 + V3347 + V1256 + V258
#> 7 : 3326 : 0.7978723 : Labels ~ 1 + V4290 + V6239 + V256 + V5982 + V4584 + V9525
#> 8 : 3320 : 0.8155244 : Labels ~ 1 + V4194 + V1476 + V1516 + V3014 + V2145
#> 9 : 3315 : 0.8210821 : Labels ~ 1 + V86 + V5680 + V3365 + V1248 + V3592
#>
#> Num. Models: 10 To Test: 63 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V698 + V4685 + V9215 + V4158 + V8604 + V1344
#> At Accuracy: Labels ~ 1 + V4685 + V9215 + V4158 + V8604 + V1344
#> B:SWiMS : Labels ~ 1 + V4685 + V9215 + V4158 + V8604 + V1344
#>
#> Num. Models: 320 To Test: 1320 TopFreq: 10.73913 Thrf: 0 Removed: 0
#> ................................*Loop : 3 Blind Cases = 9 Blind Control = 11 Total = 62 Size Cases = 27 Size Control = 35
#> Accumulated Models CV Accuracy = 0.7419355 Sensitivity = 0.6666667 Specificity = 0.8 Forw. Ensemble Accuracy= 0.8064516
#> Initial Model Accumulated CV Accuracy = 0.8064516 Sensitivity = 0.8518519 Specificity = 0.7714286
#> Initial Model Bootstrapped Accuracy = 0.8515001 Sensitivity = 0.8776171 Specificity = 0.8253831
#> Current Model Bootstrapped Accuracy = 0.8251189 Sensitivity = 0.8785128 Specificity = 0.771725
#> Current KNN Accuracy = 0.7419355 Sensitivity = 0.962963 Specificity = 0.5714286
#> Initial KNN Accuracy = 0.7903226 Sensitivity = 0.8888889 Specificity = 0.7142857
#> Train Correlation: 0.798572 Blind Correlation : 0.8781955
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 21 0
#> TRUE 16 25
#> Loop : 4 Input Cases = 88 Input Control = 112
#> Loop : 4 Train Cases = 79 Train Control = 101
#> Loop : 4 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7993993 : Labels ~ 1 + V5005 + V723 + V1831 + V1184
#> 1 : 3369 : 0.8481834 : Labels ~ 1 + V4070 + V9215 + V241 + V8368 + V698 + V2052
#> 2 : 3363 : 0.8252618 : Labels ~ 1 + V1936 + V3365 + V6163 + V1046 + V8245 + V1943
#> 3 : 3357 : 0.8021622 : Labels ~ 1 + V5400 + V312 + V5761 + V4301
#> 4 : 3353 : 0.805453 : Labels ~ 1 + V2309 + V9275 + V7513 + V3170 + V9276
#> 5 : 3348 : 0.807254 : Labels ~ 1 + V9617 + V4326 + V8806 + V1289
#> 6 : 3344 : 0.8339806 : Labels ~ 1 + V7748 + V4960 + V6584 + V7212 + V729 + V1344 + V3226
#> 7 : 3337 : 0.804264 : Labels ~ 1 + V7928 + V376 + V8156 + V9965 + V4352 + V4295
#> 8 : 3331 : 0.8177468 : Labels ~ 1 + V414 + V4584 + V436 + V9213 + V4198
#> 9 : 3326 : 0.7970607 : Labels ~ 1 + V2866 + V86 + V5680 + V3708
#>
#> Num. Models: 10 To Test: 51 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V723 + V1831 + V1184
#> At Accuracy: Labels ~ 1 + V5005 + V723 + V1831 + V1184
#> B:SWiMS : Labels ~ 1 + V5005 + V723 + V1831 + V1184
#>
#> Num. Models: 320 To Test: 1325 TopFreq: 14.35135 Thrf: 0 Removed: 0
#> ................................*Loop : 4 Blind Cases = 9 Blind Control = 11 Total = 82 Size Cases = 36 Size Control = 46
#> Accumulated Models CV Accuracy = 0.7682927 Sensitivity = 0.6944444 Specificity = 0.826087 Forw. Ensemble Accuracy= 0.8170732
#> Initial Model Accumulated CV Accuracy = 0.8170732 Sensitivity = 0.8333333 Specificity = 0.8043478
#> Initial Model Bootstrapped Accuracy = 0.8394375 Sensitivity = 0.8609113 Specificity = 0.8179638
#> Current Model Bootstrapped Accuracy = 0.7993993 Sensitivity = 0.8395194 Specificity = 0.7592791
#> Current KNN Accuracy = 0.7317073 Sensitivity = 0.9166667 Specificity = 0.5869565
#> Initial KNN Accuracy = 0.7682927 Sensitivity = 0.8611111 Specificity = 0.6956522
#> Train Correlation: 0.7425085 Blind Correlation : 0.9593985
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 30 0
#> TRUE 19 33
#> Loop : 5 Input Cases = 88 Input Control = 112
#> Loop : 5 Train Cases = 79 Train Control = 101
#> Loop : 5 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8138402 : Labels ~ 1 + V5005 + V2358 + V1831 + V9070 + V6999 + V9585
#> 1 : 3367 : 0.8247038 : Labels ~ 1 + V5321 + V782 + V9215 + V4584 + V8368 + V3161 + V8193
#> 2 : 3360 : 0.863806 : Labels ~ 1 + V3365 + V6163 + V4352 + V3082 + V4564 + V3592 + V9275 + V6068
#> 3 : 3352 : 0.8369987 : Labels ~ 1 + V698 + V6584 + V9027 + V729 + V2438 + V311
#> 4 : 3346 : 0.7518462 : Labels ~ 1 + V9617 + V9818 + V6083
#> 5 : 3343 : 0.8128138 : Labels ~ 1 + V7748 + V4301 + V9395 + V5683 + V2556
#> 6 : 3338 : 0.7816117 : Labels ~ 1 + V1476 + V7272 + V5761 + V1762
#> 7 : 3334 : 0.8122555 : Labels ~ 1 + V4290 + V312 + V8156 + V6181 + V9965 + V5378
#> 8 : 3328 : 0.8071413 : Labels ~ 1 + V436 + V3014 + V414 + V8981 + V3170
#> 9 : 3323 : 0.7515575 : Labels ~ 1 + V8055 + V4960 + V1975
#>
#> Num. Models: 10 To Test: 53 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V2358 + V1831 + V9070 + V6999 + V9585
#> At Accuracy: Labels ~ 1 + V5005 + V2358 + V1831 + V9070 + V6999 + V9585
#> B:SWiMS : Labels ~ 1 + V5005 + V2358 + V1831 + V9070 + V6999 + V9585
#>
#> Num. Models: 320 To Test: 1334 TopFreq: 12.37931 Thrf: 0 Removed: 0
#> ................................*Loop : 5 Blind Cases = 9 Blind Control = 11 Total = 102 Size Cases = 45 Size Control = 57
#> Accumulated Models CV Accuracy = 0.7843137 Sensitivity = 0.6888889 Specificity = 0.8596491 Forw. Ensemble Accuracy= 0.8431373
#> Initial Model Accumulated CV Accuracy = 0.8431373 Sensitivity = 0.8666667 Specificity = 0.8245614
#> Initial Model Bootstrapped Accuracy = 0.8385852 Sensitivity = 0.8628081 Specificity = 0.8143623
#> Current Model Bootstrapped Accuracy = 0.8138402 Sensitivity = 0.8507689 Specificity = 0.7769114
#> Current KNN Accuracy = 0.754902 Sensitivity = 0.9111111 Specificity = 0.6315789
#> Initial KNN Accuracy = 0.7745098 Sensitivity = 0.8444444 Specificity = 0.7192982
#> Train Correlation: 0.7841209 Blind Correlation : 0.7203008
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 40 0
#> TRUE 23 39
#> Loop : 6 Input Cases = 88 Input Control = 112
#> Loop : 6 Train Cases = 79 Train Control = 101
#> Loop : 6 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8574523 : Labels ~ 1 + V7272 + V6584 + V2556 + V4352 + V4584 + V8623 + V8726
#> 1 : 3366 : 0.7991741 : Labels ~ 1 + V7891 + V9818 + V1975 + V686
#> 2 : 3362 : 0.8200173 : Labels ~ 1 + V6594 + V1831 + V1248 + V8502 + V8835
#> 3 : 3357 : 0.8251778 : Labels ~ 1 + V4183 + V9215 + V130 + V5808 + V1184 + V6163
#> 4 : 3351 : 0.8480243 : Labels ~ 1 + V3206 + V4290 + V9275 + V1946 + V7195 + V9833 + V1664
#> 5 : 3344 : 0.8278153 : Labels ~ 1 + V698 + V8156 + V4185 + V5827 + V9525
#> 6 : 3339 : 0.8335496 : Labels ~ 1 + V7748 + V5761 + V1943 + V2294 + V2358
#> 7 : 3334 : 0.8530878 : Labels ~ 1 + V1476 + V4158 + V5680 + V5107 + V4751 + V3170 + V4198
#> 8 : 3327 : 0.7774554 : Labels ~ 1 + V5 + V4301 + V5005 + V4326
#> 9 : 3323 : 0.8812298 : Labels ~ 1 + V5473 + V729 + V7584 + V6511 + V6685 + V1681 + V5943 + V1198
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V7272 + V4183 + V6584 + V2556 + V4352 + V4584 + V8623 + V8726
#> At Accuracy: Labels ~ 1 + V7272 + V6584 + V2556 + V4352 + V4584 + V8623 + V8726
#> B:SWiMS : Labels ~ 1 + V7272 + V6584 + V2556 + V4352 + V4584 + V8623 + V8726
#>
#> Num. Models: 320 To Test: 1335 TopFreq: 13.32075 Thrf: 0 Removed: 0
#> ................................*Loop : 6 Blind Cases = 9 Blind Control = 11 Total = 122 Size Cases = 54 Size Control = 68
#> Accumulated Models CV Accuracy = 0.795082 Sensitivity = 0.7037037 Specificity = 0.8676471 Forw. Ensemble Accuracy= 0.8442623
#> Initial Model Accumulated CV Accuracy = 0.8360656 Sensitivity = 0.8518519 Specificity = 0.8235294
#> Initial Model Bootstrapped Accuracy = 0.8704146 Sensitivity = 0.8899501 Specificity = 0.8508791
#> Current Model Bootstrapped Accuracy = 0.8574523 Sensitivity = 0.8602686 Specificity = 0.854636
#> Current KNN Accuracy = 0.7377049 Sensitivity = 0.9074074 Specificity = 0.6029412
#> Initial KNN Accuracy = 0.7868852 Sensitivity = 0.8518519 Specificity = 0.7352941
#> Train Correlation: 0.8374024 Blind Correlation : 0.9007519
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 46 0
#> TRUE 29 47
#> Loop : 7 Input Cases = 88 Input Control = 112
#> Loop : 7 Train Cases = 79 Train Control = 101
#> Loop : 7 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.868489 : Labels ~ 1 + V7748 + V5680 + V1055 + V6688 + V2145 + V2556 + V9275 + V7402
#> 1 : 3365 : 0.824046 : Labels ~ 1 + V1476 + V5005 + V9215 + V1248 + V1529
#> 2 : 3360 : 0.8349967 : Labels ~ 1 + V698 + V9818 + V2309 + V22 + V3082 + V4352
#> 3 : 3354 : 0.8425265 : Labels ~ 1 + V6584 + V4290 + V7272 + V9276 + V1173 + V6958 + V5995
#> 4 : 3347 : 0.804282 : Labels ~ 1 + V436 + V3365 + V4326 + V9395 + V5321
#> 5 : 3342 : 0.8088299 : Labels ~ 1 + V7891 + V86 + V256 + V7536 + V9213
#> 6 : 3337 : 0.8810711 : Labels ~ 1 + V8806 + V8368 + V6163 + V4900 + V8156 + V9965 + V3161 + V8027
#> 7 : 3329 : 0.8142172 : Labels ~ 1 + V7857 + V1831 + V312 + V762 + V2515
#> 8 : 3324 : 0.806613 : Labels ~ 1 + V8502 + V4960 + V8055 + V686
#> 9 : 3320 : 0.7965718 : Labels ~ 1 + V414 + V5774 + V729 + V6597
#>
#> Num. Models: 10 To Test: 57 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V7748 + V5680 + V1055 + V6688 + V2145 + V2556 + V9275 + V7402
#> At Accuracy: Labels ~ 1 + V7748 + V5680 + V1055 + V6688 + V2145 + V2556 + V9275 + V7402
#> B:SWiMS : Labels ~ 1 + V7748 + V5680 + V1055 + V6688 + V2145 + V2556 + V9275 + V7402
#>
#> Num. Models: 320 To Test: 1305 TopFreq: 15.97368 Thrf: 0 Removed: 0
#> ................................*Loop : 7 Blind Cases = 9 Blind Control = 11 Total = 142 Size Cases = 63 Size Control = 79
#> Accumulated Models CV Accuracy = 0.8028169 Sensitivity = 0.7142857 Specificity = 0.8734177 Forw. Ensemble Accuracy= 0.8521127
#> Initial Model Accumulated CV Accuracy = 0.8450704 Sensitivity = 0.8571429 Specificity = 0.835443
#> Initial Model Bootstrapped Accuracy = 0.8570805 Sensitivity = 0.9026838 Specificity = 0.8114772
#> Current Model Bootstrapped Accuracy = 0.868489 Sensitivity = 0.8852777 Specificity = 0.8517004
#> Current KNN Accuracy = 0.7394366 Sensitivity = 0.8730159 Specificity = 0.6329114
#> Initial KNN Accuracy = 0.7887324 Sensitivity = 0.8412698 Specificity = 0.7468354
#> Train Correlation: 0.8552486 Blind Correlation : 0.6421053
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 56 2
#> TRUE 31 53
#> Loop : 8 Input Cases = 88 Input Control = 112
#> Loop : 8 Train Cases = 79 Train Control = 101
#> Loop : 8 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7923226 : Labels ~ 1 + V723 + V9818 + V312 + V6970
#> 1 : 3369 : 0.8100238 : Labels ~ 1 + V5005 + V6584 + V5 + V4301 + V1248
#> 2 : 3364 : 0.825065 : Labels ~ 1 + V1936 + V4960 + V9275 + V1975 + V4584 + V3170
#> 3 : 3358 : 0.7941049 : Labels ~ 1 + V4183 + V3365 + V1831 + V8726
#> 4 : 3354 : 0.8009229 : Labels ~ 1 + V5400 + V4070 + V5680 + V8193 + V5107
#> 5 : 3349 : 0.8044705 : Labels ~ 1 + V469 + V8368 + V9215 + V1874
#> 6 : 3345 : 0.866849 : Labels ~ 1 + V6408 + V6163 + V7513 + V3986 + V2675 + V782 + V3074 + V6487
#> 7 : 3337 : 0.8253484 : Labels ~ 1 + V414 + V872 + V819 + V8156 + V4352 + V729
#> 8 : 3331 : 0.8139009 : Labels ~ 1 + V2309 + V4326 + V533 + V1046 + V3876
#> 9 : 3326 : 0.8593648 : Labels ~ 1 + V9617 + V5457 + V1198 + V4900 + V9213 + V5378 + V7263 + V138
#>
#> Num. Models: 10 To Test: 55 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V723 + V9818 + V312 + V6970
#> At Accuracy: Labels ~ 1 + V723 + V9818 + V312 + V6970
#> B:SWiMS : Labels ~ 1 + V723 + V9818 + V312 + V6970
#>
#> Num. Models: 320 To Test: 1264 TopFreq: 13.73529 Thrf: 0 Removed: 0
#> ................................*Loop : 8 Blind Cases = 9 Blind Control = 11 Total = 162 Size Cases = 72 Size Control = 90
#> Accumulated Models CV Accuracy = 0.7901235 Sensitivity = 0.7222222 Specificity = 0.8444444 Forw. Ensemble Accuracy= 0.8395062
#> Initial Model Accumulated CV Accuracy = 0.8518519 Sensitivity = 0.8611111 Specificity = 0.8444444
#> Initial Model Bootstrapped Accuracy = 0.8602639 Sensitivity = 0.8860048 Specificity = 0.834523
#> Current Model Bootstrapped Accuracy = 0.7923226 Sensitivity = 0.8125943 Specificity = 0.7720509
#> Current KNN Accuracy = 0.7160494 Sensitivity = 0.8611111 Specificity = 0.6
#> Initial KNN Accuracy = 0.7654321 Sensitivity = 0.8333333 Specificity = 0.7111111
#> Train Correlation: 0.7734325 Blind Correlation : 0.8406015
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 62 2
#> TRUE 34 64
#> Loop : 9 Input Cases = 88 Input Control = 112
#> Loop : 9 Train Cases = 80 Train Control = 101
#> Loop : 9 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8482925 : Labels ~ 1 + V4352 + V6584 + V1975 + V7272 + V3170 + V7849
#> 1 : 3367 : 0.7804554 : Labels ~ 1 + V4183 + V9215 + V4158
#> 2 : 3364 : 0.8429798 : Labels ~ 1 + V698 + V1831 + V6239 + V3591 + V3675 + V6214 + V9070
#> 3 : 3357 : 0.8092264 : Labels ~ 1 + V3206 + V5 + V9818 + V1344 + V4406
#> 4 : 3352 : 0.7967105 : Labels ~ 1 + V1055 + V8156 + V469 + V2515 + V9965
#> 5 : 3347 : 0.8369565 : Labels ~ 1 + V4194 + V4290 + V256 + V5936 + V6688 + V2294 + V4584
#> 6 : 3340 : 0.8280771 : Labels ~ 1 + V4666 + V9275 + V7748 + V729 + V1590 + V2556
#> 7 : 3334 : 0.8422671 : Labels ~ 1 + V1476 + V3365 + V4326 + V9395 + V7141 + V1087
#> 8 : 3328 : 0.8050792 : Labels ~ 1 + V5378 + V9213 + V4414 + V4973 + V3612
#> 9 : 3323 : 0.8220153 : Labels ~ 1 + V436 + V2309 + V5417 + V615 + V5400 + V3592
#>
#> Num. Models: 10 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V4352 + V7748 + V6584 + V1975 + V7272 + V3170 + V7849
#> At Accuracy: Labels ~ 1 + V4352 + V6584 + V1975 + V7272 + V3170 + V7849
#> B:SWiMS : Labels ~ 1 + V4352 + V6584 + V1975 + V7272 + V3170 + V7849
#>
#> Num. Models: 320 To Test: 1317 TopFreq: 9.875 Thrf: 0 Removed: 0
#> ................................*Loop : 9 Blind Cases = 8 Blind Control = 11 Total = 181 Size Cases = 80 Size Control = 101
#> Accumulated Models CV Accuracy = 0.7955801 Sensitivity = 0.7375 Specificity = 0.8415842 Forw. Ensemble Accuracy= 0.8453039
#> Initial Model Accumulated CV Accuracy = 0.8618785 Sensitivity = 0.875 Specificity = 0.8514851
#> Initial Model Bootstrapped Accuracy = 0.8349398 Sensitivity = 0.8663746 Specificity = 0.8035049
#> Current Model Bootstrapped Accuracy = 0.8482925 Sensitivity = 0.8690893 Specificity = 0.8274956
#> Current KNN Accuracy = 0.7237569 Sensitivity = 0.875 Specificity = 0.6039604
#> Initial KNN Accuracy = 0.7734807 Sensitivity = 0.85 Specificity = 0.7128713
#> Train Correlation: 0.7958313 Blind Correlation : 0.7684211
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 69 2
#> TRUE 37 73
#> Loop : 10 Input Cases = 88 Input Control = 112
#> Loop : 10 Train Cases = 80 Train Control = 101
#> Loop : 10 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8443179 : Labels ~ 1 + V7748 + V9818 + V3365 + V729 + V3161 + V7141
#> 1 : 3367 : 0.8320652 : Labels ~ 1 + V312 + V9275 + V723 + V630 + V3778 + V8726
#> 2 : 3361 : 0.8398506 : Labels ~ 1 + V4352 + V8368 + V256 + V4584 + V6068 + V1344 + V9234 + V629
#> 3 : 3353 : 0.8108634 : Labels ~ 1 + V4960 + V5680 + V1476 + V9395 + V5107
#> 4 : 3348 : 0.8010741 : Labels ~ 1 + V2309 + V698 + V8156 + V5761 + V1184 + V4185
#> 5 : 3342 : 0.7951623 : Labels ~ 1 + V9617 + V436 + V6163 + V6584 + V8247
#> 6 : 3337 : 0.8272926 : Labels ~ 1 + V5005 + V1831 + V4973 + V6780 + V1338 + V4290 + V5321
#> 7 : 3330 : 0.7995376 : Labels ~ 1 + V8502 + V414 + V8055 + V1248
#> 8 : 3326 : 0.8124864 : Labels ~ 1 + V86 + V4194 + V2530 + V2356 + V3395 + V2515
#> 9 : 3320 : 0.8118855 : Labels ~ 1 + V9027 + V1046 + V4326 + V2597 + V762
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V7748 + V9818 + V3365 + V729 + V3161 + V7141
#> At Accuracy: Labels ~ 1 + V7748 + V9818 + V3365 + V729 + V3161 + V7141
#> B:SWiMS : Labels ~ 1 + V7748 + V9818 + V3365 + V729 + V3161 + V7141
#>
#> Num. Models: 320 To Test: 1254 TopFreq: 11.97143 Thrf: 0 Removed: 0
#> ................................*Loop : 10 Blind Cases = 8 Blind Control = 11 Total = 200 Size Cases = 88 Size Control = 112
#> Accumulated Models CV Accuracy = 0.79 Sensitivity = 0.7272727 Specificity = 0.8392857 Forw. Ensemble Accuracy= 0.85
#> Initial Model Accumulated CV Accuracy = 0.875 Sensitivity = 0.8863636 Specificity = 0.8660714
#> Initial Model Bootstrapped Accuracy = 0.8462886 Sensitivity = 0.8710313 Specificity = 0.8215459
#> Current Model Bootstrapped Accuracy = 0.8443179 Sensitivity = 0.8570177 Specificity = 0.8316181
#> Current KNN Accuracy = 0.73 Sensitivity = 0.875 Specificity = 0.6160714
#> Initial KNN Accuracy = 0.78 Sensitivity = 0.8636364 Specificity = 0.7142857
#> Train Correlation: 0.7855281 Blind Correlation : 0.8614035
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 77 3
#> TRUE 41 79
#> Loop : 11 Input Cases = 88 Input Control = 112
#> Loop : 11 Train Cases = 79 Train Control = 100
#> Loop : 11 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8005278 : Labels ~ 1 + V5005 + V4973 + V1831 + V7828 + V1129
#> 1 : 3368 : 0.8172055 : Labels ~ 1 + V8368 + V9659 + V3170 + V9215 + V6688 + V711
#> 2 : 3362 : 0.8571115 : Labels ~ 1 + V9585 + V4352 + V6584 + V3365 + V9213 + V2646 + V5378
#> 3 : 3355 : 0.835913 : Labels ~ 1 + V9818 + V8411 + V312 + V7849 + V1046 + V6163
#> 4 : 3349 : 0.8550836 : Labels ~ 1 + V4960 + V9275 + V7032 + V9395 + V1087 + V3675 + V4584
#> 5 : 3342 : 0.8033459 : Labels ~ 1 + V819 + V1787 + V414 + V5489 + V130
#> 6 : 3337 : 0.8196044 : Labels ~ 1 + V2309 + V4557 + V2256 + V8156 + V7381 + V4301
#> 7 : 3331 : 0.7930166 : Labels ~ 1 + V9617 + V1975 + V9965 + V6111
#> 8 : 3327 : 0.8016845 : Labels ~ 1 + V6045 + V5680 + V9027 + V1344
#> 9 : 3323 : 0.8137083 : Labels ~ 1 + V2783 + V4879 + V7748 + V9338 + V8586 + V2973
#>
#> Num. Models: 10 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V4973 + V1831 + V7828 + V1129
#> At Accuracy: Labels ~ 1 + V5005 + V4973 + V1831 + V7828 + V1129
#> B:SWiMS : Labels ~ 1 + V5005 + V4973 + V1831 + V7828 + V1129
#>
#> Num. Models: 320 To Test: 1299 TopFreq: 8.526316 Thrf: 0 Removed: 0
#> ................................*Loop : 11 Blind Cases = 9 Blind Control = 12 Total = 221 Size Cases = 97 Size Control = 124
#> Accumulated Models CV Accuracy = 0.7782805 Sensitivity = 0.7216495 Specificity = 0.8225806 Forw. Ensemble Accuracy= 0.841629
#> Initial Model Accumulated CV Accuracy = 0.8687783 Sensitivity = 0.8969072 Specificity = 0.8467742
#> Initial Model Bootstrapped Accuracy = 0.8586764 Sensitivity = 0.8882249 Specificity = 0.8291279
#> Current Model Bootstrapped Accuracy = 0.8005278 Sensitivity = 0.8607873 Specificity = 0.7402683
#> Current KNN Accuracy = 0.7330317 Sensitivity = 0.8865979 Specificity = 0.6129032
#> Initial KNN Accuracy = 0.7782805 Sensitivity = 0.8556701 Specificity = 0.7177419
#> Train Correlation: 0.7661492 Blind Correlation : 0.5818182
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 84 3
#> TRUE 45 89
#> Loop : 12 Input Cases = 88 Input Control = 112
#> Loop : 12 Train Cases = 79 Train Control = 100
#> Loop : 12 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7965001 : Labels ~ 1 + V723 + V698 + V9818 + V2242
#> 1 : 3369 : 0.817 : Labels ~ 1 + V4290 + V9215 + V9395 + V3444
#> 2 : 3365 : 0.8801552 : Labels ~ 1 + V7748 + V7272 + V1831 + V729 + V6214 + V3403 + V3161
#> 3 : 3358 : 0.8394404 : Labels ~ 1 + V2294 + V1476 + V4326 + V4185 + V4564
#> 4 : 3353 : 0.8578464 : Labels ~ 1 + V2556 + V8502 + V256 + V4584 + V4198 + V4158 + V5321 + V3675
#> 5 : 3345 : 0.8165824 : Labels ~ 1 + V7891 + V5473 + V3014 + V6198 + V1184
#> 6 : 3340 : 0.8440053 : Labels ~ 1 + V6594 + V436 + V2256 + V5680 + V2515 + V2907 + V8126
#> 7 : 3333 : 0.8192959 : Labels ~ 1 + V7197 + V7928 + V9213 + V9275 + V5283
#> 8 : 3328 : 0.8303393 : Labels ~ 1 + V6928 + V5005 + V6584 + V533 + V3170 + V6570 + V7641 + V3778
#> 9 : 3320 : 0.8774849 : Labels ~ 1 + V1055 + V8156 + V6163 + V9965 + V662 + V630 + V4352 + V5808
#>
#> Num. Models: 10 To Test: 61 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V723 + V698 + V9818 + V2242
#> At Accuracy: Labels ~ 1 + V723 + V698 + V9818 + V2242
#> B:SWiMS : Labels ~ 1 + V723 + V698 + V9818 + V2242
#>
#> Num. Models: 320 To Test: 1263 TopFreq: 14.97222 Thrf: 0 Removed: 0
#> ................................*Loop : 12 Blind Cases = 9 Blind Control = 12 Total = 242 Size Cases = 106 Size Control = 136
#> Accumulated Models CV Accuracy = 0.7892562 Sensitivity = 0.7358491 Specificity = 0.8308824 Forw. Ensemble Accuracy= 0.8429752
#> Initial Model Accumulated CV Accuracy = 0.8760331 Sensitivity = 0.9056604 Specificity = 0.8529412
#> Initial Model Bootstrapped Accuracy = 0.8502843 Sensitivity = 0.8727034 Specificity = 0.8278653
#> Current Model Bootstrapped Accuracy = 0.7965001 Sensitivity = 0.8393132 Specificity = 0.753687
#> Current KNN Accuracy = 0.731405 Sensitivity = 0.8962264 Specificity = 0.6029412
#> Initial KNN Accuracy = 0.7892562 Sensitivity = 0.8679245 Specificity = 0.7279412
#> Train Correlation: 0.7606209 Blind Correlation : 0.9493506
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 90 3
#> TRUE 51 98
#> Loop : 13 Input Cases = 88 Input Control = 112
#> Loop : 13 Train Cases = 79 Train Control = 101
#> Loop : 13 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.9001311 : Labels ~ 1 + V7748 + V1936 + V9215 + V6111 + V3675 + V729 + V2680 + V4564 + V8245
#> 1 : 3364 : 0.8645474 : Labels ~ 1 + V1476 + V8055 + V7272 + V1344 + V3857 + V4137
#> 2 : 3358 : 0.8546169 : Labels ~ 1 + V698 + V5005 + V1831 + V9965 + V8135 + V8147 + V4352
#> 3 : 3351 : 0.8719697 : Labels ~ 1 + V4290 + V898 + V5400 + V2397 + V5445 + V9158
#> 4 : 3345 : 0.8537217 : Labels ~ 1 + V7857 + V723 + V9275 + V3206 + V3861 + V2556
#> 5 : 3339 : 0.8609185 : Labels ~ 1 + V9818 + V86 + V7513 + V2747 + V2111 + V9082
#> 6 : 3333 : 0.836857 : Labels ~ 1 + V436 + V4158 + V256 + V5283 + V1173 + V4584
#> 7 : 3327 : 0.8231285 : Labels ~ 1 + V8806 + V5683 + V1055 + V7848
#> 8 : 3323 : 0.8695462 : Labels ~ 1 + V8502 + V6163 + V3365 + V9395 + V6584 + V6380 + V4738 + V3170
#> 9 : 3315 : 0.8243156 : Labels ~ 1 + V5473 + V6239 + V4326 + V2298 + V9346
#>
#> Num. Models: 10 To Test: 63 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V7748 + V1936 + V9215 + V6111 + V3675 + V729 + V2680 + V4564 + V8245
#> At Accuracy: Labels ~ 1 + V7748 + V1936 + V9215 + V6111 + V3675 + V729 + V2680 + V4564 + V8245
#> B:SWiMS : Labels ~ 1 + V7748 + V1936 + V9215 + V6111 + V3675 + V729 + V2680 + V4564 + V8245
#>
#> Num. Models: 320 To Test: 1267 TopFreq: 16.98077 Thrf: 0 Removed: 0
#> ................................*Loop : 13 Blind Cases = 9 Blind Control = 11 Total = 262 Size Cases = 115 Size Control = 147
#> Accumulated Models CV Accuracy = 0.7748092 Sensitivity = 0.7130435 Specificity = 0.8231293 Forw. Ensemble Accuracy= 0.8358779
#> Initial Model Accumulated CV Accuracy = 0.8740458 Sensitivity = 0.9043478 Specificity = 0.8503401
#> Initial Model Bootstrapped Accuracy = 0.8705484 Sensitivity = 0.8940354 Specificity = 0.8470614
#> Current Model Bootstrapped Accuracy = 0.9001311 Sensitivity = 0.9121503 Specificity = 0.8881119
#> Current KNN Accuracy = 0.7290076 Sensitivity = 0.8956522 Specificity = 0.5986395
#> Initial KNN Accuracy = 0.7900763 Sensitivity = 0.8695652 Specificity = 0.7278912
#> Train Correlation: 0.8646069 Blind Correlation : 0.6360902
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 97 3
#> TRUE 57 105
#> Loop : 14 Input Cases = 88 Input Control = 112
#> Loop : 14 Train Cases = 79 Train Control = 101
#> Loop : 14 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8462039 : Labels ~ 1 + V6584 + V8368 + V4584 + V1344 + V3161 + V2788
#> 1 : 3367 : 0.8534427 : Labels ~ 1 + V9275 + V312 + V22 + V3592 + V3778 + V3876 + V5359
#> 2 : 3360 : 0.7870714 : Labels ~ 1 + V2556 + V5005 + V9818 + V1943
#> 3 : 3356 : 0.80899 : Labels ~ 1 + V1936 + V3365 + V8156 + V1865 + V1184
#> 4 : 3351 : 0.7808649 : Labels ~ 1 + V7891 + V1831 + V662 + V1289
#> 5 : 3347 : 0.8084458 : Labels ~ 1 + V9617 + V9215 + V4198 + V6163 + V4580
#> 6 : 3342 : 0.8179059 : Labels ~ 1 + V5801 + V4352 + V3170 + V9965 + V723 + V686
#> 7 : 3336 : 0.8156297 : Labels ~ 1 + V2309 + V4326 + V436 + V8981 + V9213
#> 8 : 3331 : 0.8104342 : Labels ~ 1 + V4960 + V1046 + V3014 + V2204 + V4082
#> 9 : 3326 : 0.8314292 : Labels ~ 1 + V414 + V6164 + V4301 + V762 + V4406 + V6180
#>
#> Num. Models: 10 To Test: 53 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V6584 + V8368 + V4584 + V1344 + V3161 + V2788
#> At Accuracy: Labels ~ 1 + V6584 + V8368 + V4584 + V1344 + V3161 + V2788
#> B:SWiMS : Labels ~ 1 + V6584 + V8368 + V4584 + V1344 + V3161 + V2788
#>
#> Num. Models: 320 To Test: 1279 TopFreq: 12.625 Thrf: 0 Removed: 0
#> ................................*Loop : 14 Blind Cases = 9 Blind Control = 11 Total = 282 Size Cases = 124 Size Control = 158
#> Accumulated Models CV Accuracy = 0.7730496 Sensitivity = 0.7177419 Specificity = 0.8164557 Forw. Ensemble Accuracy= 0.8404255
#> Initial Model Accumulated CV Accuracy = 0.8687943 Sensitivity = 0.9032258 Specificity = 0.8417722
#> Initial Model Bootstrapped Accuracy = 0.8485764 Sensitivity = 0.8729508 Specificity = 0.8242019
#> Current Model Bootstrapped Accuracy = 0.8462039 Sensitivity = 0.8767896 Specificity = 0.8156182
#> Current KNN Accuracy = 0.7234043 Sensitivity = 0.8790323 Specificity = 0.6012658
#> Initial KNN Accuracy = 0.7801418 Sensitivity = 0.8548387 Specificity = 0.721519
#> Train Correlation: 0.772174 Blind Correlation : 0.8827068
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 106 4
#> TRUE 58 114
#> Loop : 15 Input Cases = 88 Input Control = 112
#> Loop : 15 Train Cases = 79 Train Control = 101
#> Loop : 15 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8588325 : Labels ~ 1 + V5005 + V4183 + V1831 + V2358 + V720 + V4723 + V9911
#> 1 : 3366 : 0.8780595 : Labels ~ 1 + V1936 + V5 + V3170 + V8156 + V6163 + V2804 + V4352 + V7760
#> 2 : 3358 : 0.8223383 : Labels ~ 1 + V7212 + V1975 + V9818 + V9213 + V2747 + V6999
#> 3 : 3352 : 0.8278582 : Labels ~ 1 + V723 + V698 + V6959 + V312 + V7562 + V7371
#> 4 : 3346 : 0.8411012 : Labels ~ 1 + V7748 + V7260 + V898 + V1344 + V306 + V7090
#> 5 : 3340 : 0.8347836 : Labels ~ 1 + V469 + V4584 + V5801 + V4301 + V9275 + V4185
#> 6 : 3334 : 0.8725405 : Labels ~ 1 + V5683 + V9585 + V1184 + V2266 + V6487 + V5400 + V6703
#> 7 : 3327 : 0.8750271 : Labels ~ 1 + V4070 + V7513 + V9215 + V5140 + V2675 + V8411 + V8618
#> 8 : 3320 : 0.845 : Labels ~ 1 + V4973 + V9395 + V9965 + V1046 + V3987 + V2256 + V8126
#> 9 : 3313 : 0.8847669 : Labels ~ 1 + V7891 + V729 + V6584 + V4960 + V4580 + V4406 + V1584 + V3161 + V2788
#>
#> Num. Models: 10 To Test: 69 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V4183 + V1831 + V2358 + V720 + V4723 + V9911
#> At Accuracy: Labels ~ 1 + V5005 + V4183 + V1831 + V2358 + V720 + V4723 + V9911
#> B:SWiMS : Labels ~ 1 + V5005 + V4183 + V1831 + V2358 + V720 + V4723 + V9911
#>
#> Num. Models: 320 To Test: 1320 TopFreq: 14.30233 Thrf: 0 Removed: 0
#> ................................*Loop : 15 Blind Cases = 9 Blind Control = 11 Total = 302 Size Cases = 133 Size Control = 169
#> Accumulated Models CV Accuracy = 0.7715232 Sensitivity = 0.6992481 Specificity = 0.8284024 Forw. Ensemble Accuracy= 0.8344371
#> Initial Model Accumulated CV Accuracy = 0.8609272 Sensitivity = 0.887218 Specificity = 0.8402367
#> Initial Model Bootstrapped Accuracy = 0.8539861 Sensitivity = 0.8778163 Specificity = 0.830156
#> Current Model Bootstrapped Accuracy = 0.8588325 Sensitivity = 0.8856337 Specificity = 0.8320312
#> Current KNN Accuracy = 0.7251656 Sensitivity = 0.8721805 Specificity = 0.6094675
#> Initial KNN Accuracy = 0.781457 Sensitivity = 0.8571429 Specificity = 0.7218935
#> Train Correlation: 0.7887713 Blind Correlation : 0.7548872
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 116 4
#> TRUE 64 118
#> Loop : 16 Input Cases = 88 Input Control = 112
#> Loop : 16 Train Cases = 79 Train Control = 101
#> Loop : 16 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.847055 : Labels ~ 1 + V2556 + V729 + V312 + V9275 + V4352 + V3161
#> 1 : 3367 : 0.8317828 : Labels ~ 1 + V4584 + V8368 + V2309 + V7891 + V3170 + V1248
#> 2 : 3361 : 0.806525 : Labels ~ 1 + V3365 + V6163 + V6584 + V8117 + V130
#> 3 : 3356 : 0.7877287 : Labels ~ 1 + V723 + V5005 + V8055 + V9213 + V4290
#> 4 : 3351 : 0.7802506 : Labels ~ 1 + V4960 + V1936 + V9215 + V7426
#> 5 : 3347 : 0.8269231 : Labels ~ 1 + V9617 + V1046 + V5683 + V8411 + V6688 + V5321 + V898
#> 6 : 3340 : 0.8865564 : Labels ~ 1 + V414 + V256 + V5400 + V6959 + V7526 + V52 + V1667 + V4564 + V5108 + V8061
#> 7 : 3330 : 0.8255231 : Labels ~ 1 + V4070 + V9818 + V9027 + V9395 + V5672 + V698 + V2862
#> 8 : 3323 : 0.7737708 : Labels ~ 1 + V376 + V8156 + V7513 + V9965 + V6970
#> 9 : 3318 : 0.7895419 : Labels ~ 1 + V7748 + V2866 + V5378 + V2515
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V723 + V2556 + V729 + V312 + V9275 + V4352 + V3161
#> At Accuracy: Labels ~ 1 + V2556 + V729 + V312 + V9275 + V4352 + V3161
#> B:SWiMS : Labels ~ 1 + V2556 + V729 + V312 + V9275 + V4352 + V3161
#>
#> Num. Models: 320 To Test: 1351 TopFreq: 11.94737 Thrf: 0 Removed: 0
#> ................................*Loop : 16 Blind Cases = 9 Blind Control = 11 Total = 322 Size Cases = 142 Size Control = 180
#> Accumulated Models CV Accuracy = 0.7763975 Sensitivity = 0.7183099 Specificity = 0.8222222 Forw. Ensemble Accuracy= 0.8416149
#> Initial Model Accumulated CV Accuracy = 0.8664596 Sensitivity = 0.8943662 Specificity = 0.8444444
#> Initial Model Bootstrapped Accuracy = 0.8334767 Sensitivity = 0.8522898 Specificity = 0.8146635
#> Current Model Bootstrapped Accuracy = 0.847055 Sensitivity = 0.8589884 Specificity = 0.8351215
#> Current KNN Accuracy = 0.7298137 Sensitivity = 0.8802817 Specificity = 0.6111111
#> Initial KNN Accuracy = 0.7888199 Sensitivity = 0.8661972 Specificity = 0.7277778
#> Train Correlation: 0.7808677 Blind Correlation : 0.6496241
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 122 5
#> TRUE 66 129
#> Loop : 17 Input Cases = 88 Input Control = 112
#> Loop : 17 Train Cases = 79 Train Control = 101
#> Loop : 17 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8702183 : Labels ~ 1 + V2556 + V1936 + V9818 + V1584 + V1476 + V723 + V9487
#> 1 : 3366 : 0.8274002 : Labels ~ 1 + V5005 + V130 + V5 + V4584 + V9275 + V4352 + V5943
#> 2 : 3359 : 0.8687514 : Labels ~ 1 + V7891 + V8156 + V7513 + V9965 + V2636 + V436 + V4069
#> 3 : 3352 : 0.8237193 : Labels ~ 1 + V2294 + V9215 + V4200 + V5489 + V2358
#> 4 : 3347 : 0.8450549 : Labels ~ 1 + V6594 + V9213 + V5417 + V6688 + V6584 + V8502 + V4764
#> 5 : 3340 : 0.8317057 : Labels ~ 1 + V4290 + V5761 + V6780 + V4301 + V5321 + V8193
#> 6 : 3334 : 0.8267743 : Labels ~ 1 + V8806 + V1831 + V5801 + V9395 + V5267 + V932
#> 7 : 3328 : 0.8311015 : Labels ~ 1 + V5808 + V86 + V1046 + V4326 + V4077 + V8369
#> 8 : 3322 : 0.8042674 : Labels ~ 1 + V9735 + V3170 + V4183 + V6181 + V218
#> 9 : 3317 : 0.8531073 : Labels ~ 1 + V7748 + V8623 + V729 + V4960 + V7984 + V381
#>
#> Num. Models: 10 To Test: 62 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V1936 + V9818 + V5 + V1584 + V1476 + V723 + V9487
#> At Accuracy: Labels ~ 1 + V2556 + V1936 + V9818 + V1584 + V1476 + V723 + V9487
#> B:SWiMS : Labels ~ 1 + V2556 + V1936 + V9818 + V1584 + V1476 + V723 + V9487
#>
#> Num. Models: 320 To Test: 1246 TopFreq: 17.55102 Thrf: 0 Removed: 0
#> ................................*Loop : 17 Blind Cases = 9 Blind Control = 11 Total = 342 Size Cases = 151 Size Control = 191
#> Accumulated Models CV Accuracy = 0.7777778 Sensitivity = 0.7284768 Specificity = 0.8167539 Forw. Ensemble Accuracy= 0.8391813
#> Initial Model Accumulated CV Accuracy = 0.8654971 Sensitivity = 0.8874172 Specificity = 0.8481675
#> Initial Model Bootstrapped Accuracy = 0.8670872 Sensitivity = 0.88536 Specificity = 0.8488144
#> Current Model Bootstrapped Accuracy = 0.8702183 Sensitivity = 0.8902096 Specificity = 0.8502269
#> Current KNN Accuracy = 0.7222222 Sensitivity = 0.8741722 Specificity = 0.6020942
#> Initial KNN Accuracy = 0.7865497 Sensitivity = 0.8675497 Specificity = 0.7225131
#> Train Correlation: 0.8194759 Blind Correlation : 0.7172932
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 128 6
#> TRUE 69 139
#> Loop : 18 Input Cases = 88 Input Control = 112
#> Loop : 18 Train Cases = 79 Train Control = 101
#> Loop : 18 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8046657 : Labels ~ 1 + V5005 + V9818 + V3524 + V1184 + V4183
#> 1 : 3368 : 0.8940844 : Labels ~ 1 + V9275 + V8368 + V3161 + V6163 + V8996 + V9255 + V5321 + V8604
#> 2 : 3360 : 0.8331162 : Labels ~ 1 + V9965 + V3365 + V8156 + V729 + V8502 + V3082
#> 3 : 3354 : 0.8627387 : Labels ~ 1 + V5683 + V312 + V1476 + V6688 + V4738 + V4352 + V7606
#> 4 : 3347 : 0.80903 : Labels ~ 1 + V7748 + V2309 + V898 + V1248
#> 5 : 3343 : 0.8485272 : Labels ~ 1 + V4960 + V4584 + V7857 + V6584 + V9395 + V8245 + V9127
#> 6 : 3336 : 0.8012179 : Labels ~ 1 + V9617 + V1831 + V5774 + V6083 + V1344
#> 7 : 3331 : 0.8085502 : Labels ~ 1 + V414 + V4326 + V86 + V9070 + V2747
#> 8 : 3326 : 0.8252947 : Labels ~ 1 + V376 + V5680 + V762 + V2438 + V2462 + V3778
#> 9 : 3320 : 0.8339802 : Labels ~ 1 + V5761 + V436 + V9027 + V5945 + V7584 + V6440
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V9818 + V3524 + V1184 + V4183
#> At Accuracy: Labels ~ 1 + V5005 + V9818 + V3524 + V1184 + V4183
#> B:SWiMS : Labels ~ 1 + V5005 + V9818 + V3524 + V1184 + V4183
#>
#> Num. Models: 320 To Test: 1319 TopFreq: 10.425 Thrf: 0 Removed: 0
#> ................................*Loop : 18 Blind Cases = 9 Blind Control = 11 Total = 362 Size Cases = 160 Size Control = 202
#> Accumulated Models CV Accuracy = 0.781768 Sensitivity = 0.725 Specificity = 0.8267327 Forw. Ensemble Accuracy= 0.8453039
#> Initial Model Accumulated CV Accuracy = 0.8674033 Sensitivity = 0.88125 Specificity = 0.8564356
#> Initial Model Bootstrapped Accuracy = 0.837041 Sensitivity = 0.8611231 Specificity = 0.812959
#> Current Model Bootstrapped Accuracy = 0.8046657 Sensitivity = 0.8497097 Specificity = 0.7596216
#> Current KNN Accuracy = 0.7265193 Sensitivity = 0.875 Specificity = 0.6089109
#> Initial KNN Accuracy = 0.7845304 Sensitivity = 0.8625 Specificity = 0.7227723
#> Train Correlation: 0.7369981 Blind Correlation : 0.8827068
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 137 6
#> TRUE 74 145
#> Loop : 19 Input Cases = 88 Input Control = 112
#> Loop : 19 Train Cases = 80 Train Control = 101
#> Loop : 19 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8634043 : Labels ~ 1 + V8368 + V8411 + V3014 + V5321 + V3170 + V9156 + V4352
#> 1 : 3366 : 0.8228433 : Labels ~ 1 + V1831 + V3365 + V4973 + V4584 + V4301 + V8156
#> 2 : 3360 : 0.8539571 : Labels ~ 1 + V2309 + V9585 + V9818 + V8635 + V1289 + V3592
#> 3 : 3354 : 0.8180151 : Labels ~ 1 + V312 + V9659 + V2638 + V9215 + V713
#> 4 : 3349 : 0.7966791 : Labels ~ 1 + V9617 + V5680 + V1476 + V5107
#> 5 : 3345 : 0.7964309 : Labels ~ 1 + V4960 + V8806 + V4326
#> 6 : 3342 : 0.8218785 : Labels ~ 1 + V414 + V5761 + V436 + V1584
#> 7 : 3338 : 0.8207792 : Labels ~ 1 + V376 + V7857 + V9275 + V4289 + V7032
#> 8 : 3333 : 0.83576 : Labels ~ 1 + V7220 + V9027 + V6584 + V6163 + V7141 + V2556
#> 9 : 3327 : 0.79431 : Labels ~ 1 + V86 + V2866 + V9329 + V5514
#>
#> Num. Models: 10 To Test: 50 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V8368 + V8411 + V3014 + V5321 + V3170 + V9156 + V4352
#> At Accuracy: Labels ~ 1 + V8368 + V8411 + V3014 + V5321 + V3170 + V9156 + V4352
#> B:SWiMS : Labels ~ 1 + V8368 + V8411 + V3014 + V5321 + V3170 + V9156 + V4352
#>
#> Num. Models: 320 To Test: 1285 TopFreq: 11.71875 Thrf: 0 Removed: 0
#> ................................*Loop : 19 Blind Cases = 8 Blind Control = 11 Total = 381 Size Cases = 168 Size Control = 213
#> Accumulated Models CV Accuracy = 0.7769029 Sensitivity = 0.7321429 Specificity = 0.8122066 Forw. Ensemble Accuracy= 0.839895
#> Initial Model Accumulated CV Accuracy = 0.8713911 Sensitivity = 0.8869048 Specificity = 0.8591549
#> Initial Model Bootstrapped Accuracy = 0.8472836 Sensitivity = 0.8745583 Specificity = 0.8200088
#> Current Model Bootstrapped Accuracy = 0.8634043 Sensitivity = 0.8726143 Specificity = 0.8541944
#> Current KNN Accuracy = 0.7244094 Sensitivity = 0.875 Specificity = 0.6056338
#> Initial KNN Accuracy = 0.7821522 Sensitivity = 0.8690476 Specificity = 0.713615
#> Train Correlation: 0.8389108 Blind Correlation : 0.6438596
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 143 7
#> TRUE 75 156
#> Loop : 20 Input Cases = 88 Input Control = 112
#> Loop : 20 Train Cases = 80 Train Control = 101
#> Loop : 20 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8375518 : Labels ~ 1 + V7748 + V7272 + V9275 + V3857 + V1173 + V3161 + V898
#> 1 : 3366 : 0.8350131 : Labels ~ 1 + V9215 + V1184 + V4158 + V9156 + V4352 + V7032 + V7402
#> 2 : 3359 : 0.7949946 : Labels ~ 1 + V5005 + V1975 + V9818 + V9395 + V9346 + V5672
#> 3 : 3353 : 0.7833734 : Labels ~ 1 + V1936 + V1476 + V1831 + V723 + V2358
#> 4 : 3348 : 0.8045939 : Labels ~ 1 + V3629 + V3365 + V6163 + V256 + V2597
#> 5 : 3343 : 0.7860577 : Labels ~ 1 + V7378 + V6239 + V6584 + V8414 + V2973
#> 6 : 3338 : 0.7732953 : Labels ~ 1 + V698 + V4194 + V1400 + V4326
#> 7 : 3334 : 0.8050498 : Labels ~ 1 + V4290 + V8368 + V5680 + V6688 + V6948
#> 8 : 3329 : 0.8015192 : Labels ~ 1 + V312 + V4584 + V436 + V2515
#> 9 : 3325 : 0.7824411 : Labels ~ 1 + V4960 + V8055 + V8502 + V8981
#>
#> Num. Models: 10 To Test: 52 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V7748 + V7272 + V9275 + V3857 + V1173 + V3161 + V898
#> At Accuracy: Labels ~ 1 + V7748 + V7272 + V9275 + V3857 + V1173 + V3161 + V898
#> B:SWiMS : Labels ~ 1 + V7748 + V7272 + V9275 + V3857 + V1173 + V3161 + V898
#>
#> Num. Models: 320 To Test: 1424 TopFreq: 16.38 Thrf: 0 Removed: 0
#> ................................*Loop : 20 Blind Cases = 8 Blind Control = 11 Total = 400 Size Cases = 176 Size Control = 224
#> Accumulated Models CV Accuracy = 0.785 Sensitivity = 0.7443182 Specificity = 0.8169643 Forw. Ensemble Accuracy= 0.8425
#> Initial Model Accumulated CV Accuracy = 0.875 Sensitivity = 0.8920455 Specificity = 0.8616071
#> Initial Model Bootstrapped Accuracy = 0.8428167 Sensitivity = 0.8611293 Specificity = 0.824504
#> Current Model Bootstrapped Accuracy = 0.8375518 Sensitivity = 0.8605717 Specificity = 0.814532
#> Current KNN Accuracy = 0.73 Sensitivity = 0.8806818 Specificity = 0.6116071
#> Initial KNN Accuracy = 0.79 Sensitivity = 0.875 Specificity = 0.7232143
#> Train Correlation: 0.7874365 Blind Correlation : 0.6649123
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 151 7
#> TRUE 77 165
#> Loop : 21 Input Cases = 88 Input Control = 112
#> Loop : 21 Train Cases = 79 Train Control = 100
#> Loop : 21 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8583982 : Labels ~ 1 + V2556 + V1476 + V9818 + V1975 + V1791 + V9395 + V4069 + V7860
#> 1 : 3365 : 0.8518437 : Labels ~ 1 + V1936 + V9275 + V3365 + V5 + V3170 + V5698 + V698
#> 2 : 3358 : 0.814758 : Labels ~ 1 + V5005 + V4183 + V130 + V6584 + V7760
#> 3 : 3353 : 0.8348574 : Labels ~ 1 + V7748 + V9215 + V469 + V1055 + V6163 + V9213 + V3395 + V662
#> 4 : 3345 : 0.8829219 : Labels ~ 1 + V4290 + V1831 + V312 + V8247 + V7891 + V1248 + V9329 + V34
#> 5 : 3337 : 0.8346543 : Labels ~ 1 + V8502 + V729 + V8368 + V4301 + V6688
#> 6 : 3332 : 0.8352601 : Labels ~ 1 + V872 + V4960 + V8156 + V4584 + V1046 + V8193
#> 7 : 3326 : 0.815801 : Labels ~ 1 + V782 + V1198 + V5801 + V3974 + V9965 + V5400
#> 8 : 3320 : 0.8328584 : Labels ~ 1 + V4973 + V2309 + V5761 + V1344 + V9704
#> 9 : 3315 : 0.8014509 : Labels ~ 1 + V9617 + V4557 + V2783 + V2515 + V8623
#>
#> Num. Models: 10 To Test: 63 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V1476 + V9818 + V1975 + V1791 + V9395 + V4069 + V7860
#> At Accuracy: Labels ~ 1 + V2556 + V1476 + V9818 + V1975 + V1791 + V9395 + V4069 + V7860
#> B:SWiMS : Labels ~ 1 + V2556 + V1476 + V9818 + V1975 + V1791 + V9395 + V4069 + V7860
#>
#> Num. Models: 320 To Test: 1217 TopFreq: 12.66667 Thrf: 0 Removed: 0
#> ................................*Loop : 21 Blind Cases = 9 Blind Control = 12 Total = 421 Size Cases = 185 Size Control = 236
#> Accumulated Models CV Accuracy = 0.783848 Sensitivity = 0.7459459 Specificity = 0.8135593 Forw. Ensemble Accuracy= 0.8384798
#> Initial Model Accumulated CV Accuracy = 0.8741093 Sensitivity = 0.8918919 Specificity = 0.8601695
#> Initial Model Bootstrapped Accuracy = 0.8673175 Sensitivity = 0.8824051 Specificity = 0.8522299
#> Current Model Bootstrapped Accuracy = 0.8583982 Sensitivity = 0.8776418 Specificity = 0.8391546
#> Current KNN Accuracy = 0.7292162 Sensitivity = 0.8810811 Specificity = 0.6101695
#> Initial KNN Accuracy = 0.7885986 Sensitivity = 0.8756757 Specificity = 0.720339
#> Train Correlation: 0.8176637 Blind Correlation : 0.9376623
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 159 7
#> TRUE 80 175
#> Loop : 22 Input Cases = 88 Input Control = 112
#> Loop : 22 Train Cases = 79 Train Control = 100
#> Loop : 22 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8110746 : Labels ~ 1 + V5005 + V723 + V5761 + V3170 + V5321 + V2264
#> 1 : 3367 : 0.8445842 : Labels ~ 1 + V1936 + V2556 + V9215 + V6403 + V2636 + V5400
#> 2 : 3361 : 0.8287747 : Labels ~ 1 + V7891 + V662 + V4326 + V6146 + V5107
#> 3 : 3356 : 0.8459568 : Labels ~ 1 + V5801 + V7513 + V9818 + V4352 + V4406 + V4301
#> 4 : 3350 : 0.7940915 : Labels ~ 1 + V6594 + V9970 + V8156 + V2700
#> 5 : 3346 : 0.8379364 : Labels ~ 1 + V7748 + V4194 + V5680 + V6111 + V1344 + V3592
#> 6 : 3340 : 0.8422884 : Labels ~ 1 + V6959 + V3365 + V8502 + V3708 + V2294 + V9275
#> 7 : 3334 : 0.8129386 : Labels ~ 1 + V9735 + V819 + V1184 + V7928 + V3082
#> 8 : 3329 : 0.8384598 : Labels ~ 1 + V898 + V4960 + V4580 + V4198 + V8767 + V3070
#> 9 : 3323 : 0.8496558 : Labels ~ 1 + V312 + V6584 + V6163 + V1865 + V2973 + V2788
#>
#> Num. Models: 10 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V723 + V5761 + V3170 + V5321 + V2264
#> At Accuracy: Labels ~ 1 + V5005 + V723 + V5761 + V3170 + V5321 + V2264
#> B:SWiMS : Labels ~ 1 + V5005 + V723 + V5761 + V3170 + V5321 + V2264
#>
#> Num. Models: 320 To Test: 1240 TopFreq: 13.3871 Thrf: 0 Removed: 0
#> ................................*Loop : 22 Blind Cases = 9 Blind Control = 12 Total = 442 Size Cases = 194 Size Control = 248
#> Accumulated Models CV Accuracy = 0.7828054 Sensitivity = 0.7474227 Specificity = 0.8104839 Forw. Ensemble Accuracy= 0.8371041
#> Initial Model Accumulated CV Accuracy = 0.8642534 Sensitivity = 0.8762887 Specificity = 0.8548387
#> Initial Model Bootstrapped Accuracy = 0.8668775 Sensitivity = 0.8859573 Specificity = 0.8477976
#> Current Model Bootstrapped Accuracy = 0.8110746 Sensitivity = 0.8379386 Specificity = 0.7842105
#> Current KNN Accuracy = 0.7307692 Sensitivity = 0.8814433 Specificity = 0.6129032
#> Initial KNN Accuracy = 0.7918552 Sensitivity = 0.8814433 Specificity = 0.7217742
#> Train Correlation: 0.7715209 Blind Correlation : 0.7532468
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 168 7
#> TRUE 82 185
#> Loop : 23 Input Cases = 88 Input Control = 112
#> Loop : 23 Train Cases = 79 Train Control = 101
#> Loop : 23 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7987792 : Labels ~ 1 + V5005 + V5400 + V9215 + V2358 + V1173
#> 1 : 3368 : 0.7922778 : Labels ~ 1 + V4183 + V1936 + V1831 + V3947 + V4185
#> 2 : 3363 : 0.8022217 : Labels ~ 1 + V723 + V4290 + V3014 + V7272 + V3675
#> 3 : 3358 : 0.8499131 : Labels ~ 1 + V698 + V7513 + V2256 + V3591 + V2266 + V7402 + V2556 + V8156
#> 4 : 3350 : 0.7927489 : Labels ~ 1 + V8368 + V7928 + V6584 + V1344 + V306
#> 5 : 3345 : 0.7841727 : Labels ~ 1 + V3365 + V7212 + V8055 + V7748 + V7033
#> 6 : 3340 : 0.8088427 : Labels ~ 1 + V312 + V4352 + V8720 + V9818 + V4406
#> 7 : 3335 : 0.7940541 : Labels ~ 1 + V1476 + V4069 + V3206 + V9156 + V9275
#> 8 : 3330 : 0.8317266 : Labels ~ 1 + V4960 + V1975 + V1198 + V898 + V3170 + V3554
#> 9 : 3324 : 0.8546549 : Labels ~ 1 + V2309 + V34 + V6959 + V3866 + V9617 + V4554 + V9528
#>
#> Num. Models: 10 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V5400 + V9215 + V2358 + V1173
#> At Accuracy: Labels ~ 1 + V5005 + V5400 + V9215 + V2358 + V1173
#> B:SWiMS : Labels ~ 1 + V5005 + V5400 + V9215 + V2358 + V1173
#>
#> Num. Models: 320 To Test: 1274 TopFreq: 10.70968 Thrf: 0 Removed: 0
#> ................................*Loop : 23 Blind Cases = 9 Blind Control = 11 Total = 462 Size Cases = 203 Size Control = 259
#> Accumulated Models CV Accuracy = 0.7922078 Sensitivity = 0.7586207 Specificity = 0.8185328 Forw. Ensemble Accuracy= 0.8376623
#> Initial Model Accumulated CV Accuracy = 0.8658009 Sensitivity = 0.8817734 Specificity = 0.8532819
#> Initial Model Bootstrapped Accuracy = 0.8390943 Sensitivity = 0.8591118 Specificity = 0.8190768
#> Current Model Bootstrapped Accuracy = 0.7987792 Sensitivity = 0.854371 Specificity = 0.7431873
#> Current KNN Accuracy = 0.7359307 Sensitivity = 0.8866995 Specificity = 0.6177606
#> Initial KNN Accuracy = 0.7965368 Sensitivity = 0.8866995 Specificity = 0.7258687
#> Train Correlation: 0.7809336 Blind Correlation : 0.9082707
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 176 7
#> TRUE 85 194
#> Loop : 24 Input Cases = 88 Input Control = 112
#> Loop : 24 Train Cases = 79 Train Control = 101
#> Loop : 24 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8439477 : Labels ~ 1 + V2556 + V9275 + V8775 + V1943 + V3170 + V8618
#> 1 : 3367 : 0.8122673 : Labels ~ 1 + V7891 + V723 + V9215 + V9050 + V4352
#> 2 : 3362 : 0.8598842 : Labels ~ 1 + V6594 + V6584 + V4158 + V4584 + V5721 + V4900 + V3161
#> 3 : 3355 : 0.8508772 : Labels ~ 1 + V5005 + V9818 + V1184 + V3365 + V4973 + V6172
#> 4 : 3349 : 0.8203108 : Labels ~ 1 + V7748 + V256 + V8411 + V312 + V9070 + V2526
#> 5 : 3343 : 0.8458816 : Labels ~ 1 + V1476 + V8368 + V5680 + V8038 + V2145 + V8323
#> 6 : 3337 : 0.8425946 : Labels ~ 1 + V9585 + V4301 + V7319 + V1248 + V1874 + V7026
#> 7 : 3331 : 0.8254593 : Labels ~ 1 + V7032 + V4960 + V4326 + V1584 + V1046
#> 8 : 3326 : 0.7518669 : Labels ~ 1 + V1831 + V9432
#> 9 : 3324 : 0.8225528 : Labels ~ 1 + V698 + V2309 + V5761 + V6111 + V6440
#>
#> Num. Models: 10 To Test: 54 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V9275 + V8775 + V1943 + V3170 + V8618
#> At Accuracy: Labels ~ 1 + V2556 + V9275 + V8775 + V1943 + V3170 + V8618
#> B:SWiMS : Labels ~ 1 + V2556 + V9275 + V8775 + V1943 + V3170 + V8618
#>
#> Num. Models: 320 To Test: 1362 TopFreq: 10.26829 Thrf: 0 Removed: 0
#> ................................*Loop : 24 Blind Cases = 9 Blind Control = 11 Total = 482 Size Cases = 212 Size Control = 270
#> Accumulated Models CV Accuracy = 0.7904564 Sensitivity = 0.759434 Specificity = 0.8148148 Forw. Ensemble Accuracy= 0.8340249
#> Initial Model Accumulated CV Accuracy = 0.8692946 Sensitivity = 0.8867925 Specificity = 0.8555556
#> Initial Model Bootstrapped Accuracy = 0.8447225 Sensitivity = 0.8739935 Specificity = 0.8154516
#> Current Model Bootstrapped Accuracy = 0.8439477 Sensitivity = 0.8752454 Specificity = 0.8126499
#> Current KNN Accuracy = 0.7365145 Sensitivity = 0.8867925 Specificity = 0.6185185
#> Initial KNN Accuracy = 0.7987552 Sensitivity = 0.8915094 Specificity = 0.7259259
#> Train Correlation: 0.8355725 Blind Correlation : 0.7593985
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 183 8
#> TRUE 88 203
#> Loop : 25 Input Cases = 88 Input Control = 112
#> Loop : 25 Train Cases = 79 Train Control = 101
#> Loop : 25 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8311202 : Labels ~ 1 + V2556 + V698 + V9818 + V9395 + V729 + V4564 + V932
#> 1 : 3366 : 0.8274936 : Labels ~ 1 + V7748 + V4352 + V4584 + V312 + V6688
#> 2 : 3361 : 0.8339524 : Labels ~ 1 + V5005 + V4290 + V2515 + V6584 + V7473 + V2665 + V5672
#> 3 : 3354 : 0.7924773 : Labels ~ 1 + V7891 + V2256 + V256 + V436 + V220
#> 4 : 3349 : 0.8236564 : Labels ~ 1 + V1476 + V9275 + V1936 + V4295 + V2973 + V6163
#> 5 : 3343 : 0.8129763 : Labels ~ 1 + V8502 + V4301 + V4069 + V5283 + V3708
#> 6 : 3338 : 0.7827772 : Labels ~ 1 + V5473 + V34 + V4654 + V8156
#> 7 : 3334 : 0.789748 : Labels ~ 1 + V7197 + V4960 + V4185 + V898
#> 8 : 3330 : 0.7923894 : Labels ~ 1 + V86 + V1831 + V62 + V723 + V1344
#> 9 : 3325 : 0.7959206 : Labels ~ 1 + V7857 + V9215 + V8038 + V7196 + V3675
#>
#> Num. Models: 10 To Test: 53 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V698 + V9818 + V9395 + V729 + V4564 + V932
#> At Accuracy: Labels ~ 1 + V2556 + V698 + V9818 + V9395 + V729 + V4564 + V932
#> B:SWiMS : Labels ~ 1 + V2556 + V698 + V9818 + V9395 + V729 + V4564 + V932
#>
#> Num. Models: 320 To Test: 1337 TopFreq: 13.97059 Thrf: 0 Removed: 0
#> ................................*Loop : 25 Blind Cases = 9 Blind Control = 11 Total = 502 Size Cases = 221 Size Control = 281
#> Accumulated Models CV Accuracy = 0.7948207 Sensitivity = 0.7692308 Specificity = 0.8149466 Forw. Ensemble Accuracy= 0.8366534
#> Initial Model Accumulated CV Accuracy = 0.8705179 Sensitivity = 0.8914027 Specificity = 0.8540925
#> Initial Model Bootstrapped Accuracy = 0.8454012 Sensitivity = 0.8604044 Specificity = 0.8303979
#> Current Model Bootstrapped Accuracy = 0.8311202 Sensitivity = 0.8485757 Specificity = 0.8136646
#> Current KNN Accuracy = 0.7390438 Sensitivity = 0.8914027 Specificity = 0.6192171
#> Initial KNN Accuracy = 0.8027888 Sensitivity = 0.8959276 Specificity = 0.7295374
#> Train Correlation: 0.8386827 Blind Correlation : 0.8556391
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 190 8
#> TRUE 90 214
#> Loop : 26 Input Cases = 88 Input Control = 112
#> Loop : 26 Train Cases = 79 Train Control = 101
#> Loop : 26 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8403888 : Labels ~ 1 + V729 + V6045 + V3947 + V7446 + V6888 + V6584
#> 1 : 3367 : 0.8631055 : Labels ~ 1 + V9215 + V8368 + V4584 + V1476 + V8479 + V5321 + V2556 + V3170 + V5212
#> 2 : 3358 : 0.8218989 : Labels ~ 1 + V5005 + V6163 + V7032 + V7090 + V4301 + V2064
#> 3 : 3352 : 0.8488296 : Labels ~ 1 + V5801 + V762 + V9275 + V4406 + V4352 + V3365 + V3161 + V6520
#> 4 : 3344 : 0.8229955 : Labels ~ 1 + V8411 + V312 + V7319 + V4077 + V9213 + V3518 + V3592
#> 5 : 3337 : 0.8195407 : Labels ~ 1 + V9432 + V8055 + V698 + V6111 + V4018
#> 6 : 3332 : 0.8334046 : Labels ~ 1 + V4960 + V5680 + V9919 + V22 + V3746 + V4580
#> 7 : 3326 : 0.8789588 : Labels ~ 1 + V5378 + V9735 + V2804 + V1831 + V3866 + V9133 + V9884 + V7629 + V2597 + V9965
#> 8 : 3316 : 0.8189805 : Labels ~ 1 + V2309 + V6959 + V7857 + V7562 + V3148
#> 9 : 3311 : 0.8136551 : Labels ~ 1 + V9818 + V9617 + V5995 + V9395 + V7748
#>
#> Num. Models: 10 To Test: 67 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V9432 + V729 + V6045 + V4290 + V3947 + V7446 + V6888 + V6584
#> At Accuracy: Labels ~ 1 + V729 + V6045 + V3947 + V7446 + V6888 + V6584
#> B:SWiMS : Labels ~ 1 + V729 + V6045 + V3947 + V7446 + V6888 + V6584
#>
#> Num. Models: 320 To Test: 1358 TopFreq: 11.54054 Thrf: 0 Removed: 0
#> ................................*Loop : 26 Blind Cases = 9 Blind Control = 11 Total = 522 Size Cases = 230 Size Control = 292
#> Accumulated Models CV Accuracy = 0.7969349 Sensitivity = 0.773913 Specificity = 0.8150685 Forw. Ensemble Accuracy= 0.8371648
#> Initial Model Accumulated CV Accuracy = 0.8735632 Sensitivity = 0.8913043 Specificity = 0.859589
#> Initial Model Bootstrapped Accuracy = 0.8452073 Sensitivity = 0.8715458 Specificity = 0.8188687
#> Current Model Bootstrapped Accuracy = 0.8403888 Sensitivity = 0.8617711 Specificity = 0.8190065
#> Current KNN Accuracy = 0.7394636 Sensitivity = 0.8913043 Specificity = 0.619863
#> Initial KNN Accuracy = 0.8007663 Sensitivity = 0.8956522 Specificity = 0.7260274
#> Train Correlation: 0.8272375 Blind Correlation : 0.7007519
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 197 9
#> TRUE 93 223
#> Loop : 27 Input Cases = 88 Input Control = 112
#> Loop : 27 Train Cases = 79 Train Control = 101
#> Loop : 27 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8502151 : Labels ~ 1 + V1936 + V9818 + V7272 + V6597 + V6163 + V4092 + V8245
#> 1 : 3366 : 0.8247613 : Labels ~ 1 + V9215 + V312 + V6948 + V2294 + V60
#> 2 : 3361 : 0.85013 : Labels ~ 1 + V1831 + V8368 + V4183 + V7141 + V2556 + V8996 + V867
#> 3 : 3354 : 0.7779357 : Labels ~ 1 + V5005 + V1975 + V4326 + V4301
#> 4 : 3350 : 0.7848787 : Labels ~ 1 + V3365 + V9275 + V469 + V3170 + V729
#> 5 : 3345 : 0.8087154 : Labels ~ 1 + V4960 + V3014 + V9213 + V2242
#> 6 : 3341 : 0.8377448 : Labels ~ 1 + V8156 + V3987 + V2973 + V8502 + V9395
#> 7 : 3336 : 0.8250436 : Labels ~ 1 + V4584 + V4290 + V414 + V6584 + V1344 + V7381
#> 8 : 3330 : 0.8122564 : Labels ~ 1 + V698 + V4194 + V9965 + V5680 + V8517 + V1173 + V7584
#> 9 : 3323 : 0.8280035 : Labels ~ 1 + V5473 + V872 + V5417 + V1952 + V4564 + V6688 + V7275
#>
#> Num. Models: 10 To Test: 57 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V1936 + V9818 + V7272 + V6597 + V6163 + V4092 + V8245
#> At Accuracy: Labels ~ 1 + V1936 + V9818 + V7272 + V6597 + V6163 + V4092 + V8245
#> B:SWiMS : Labels ~ 1 + V1936 + V9818 + V7272 + V6597 + V6163 + V4092 + V8245
#>
#> Num. Models: 320 To Test: 1277 TopFreq: 13.54286 Thrf: 0 Removed: 0
#> ................................*Loop : 27 Blind Cases = 9 Blind Control = 11 Total = 542 Size Cases = 239 Size Control = 303
#> Accumulated Models CV Accuracy = 0.795203 Sensitivity = 0.7698745 Specificity = 0.8151815 Forw. Ensemble Accuracy= 0.8394834
#> Initial Model Accumulated CV Accuracy = 0.8726937 Sensitivity = 0.8870293 Specificity = 0.8613861
#> Initial Model Bootstrapped Accuracy = 0.8536425 Sensitivity = 0.8761759 Specificity = 0.8311092
#> Current Model Bootstrapped Accuracy = 0.8502151 Sensitivity = 0.8890323 Specificity = 0.8113978
#> Current KNN Accuracy = 0.7435424 Sensitivity = 0.8912134 Specificity = 0.6270627
#> Initial KNN Accuracy = 0.798893 Sensitivity = 0.8912134 Specificity = 0.7260726
#> Train Correlation: 0.7811064 Blind Correlation : 0.8496241
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 206 10
#> TRUE 96 230
#> Loop : 28 Input Cases = 88 Input Control = 112
#> Loop : 28 Train Cases = 79 Train Control = 101
#> Loop : 28 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8683638 : Labels ~ 1 + V4352 + V3365 + V9215 + V686 + V5321 + V2556 + V3170 + V3647
#> 1 : 3365 : 0.8049045 : Labels ~ 1 + V7891 + V9818 + V4183 + V8981 + V130
#> 2 : 3360 : 0.8403509 : Labels ~ 1 + V1975 + V5005 + V8156 + V7760 + V4554 + V6163
#> 3 : 3354 : 0.826484 : Labels ~ 1 + V4564 + V8368 + V5 + V9275 + V2515 + V4978
#> 4 : 3348 : 0.8297117 : Labels ~ 1 + V469 + V256 + V312 + V1046 + V4584 + V9156
#> 5 : 3342 : 0.8315743 : Labels ~ 1 + V3725 + V7748 + V5680 + V5489 + V3778 + V6440
#> 6 : 3336 : 0.7842716 : Labels ~ 1 + V4960 + V872 + V1831 + V729 + V1344
#> 7 : 3331 : 0.8597495 : Labels ~ 1 + V2309 + V6584 + V4973 + V1184 + V1584 + V4580 + V7513 + V9833
#> 8 : 3323 : 0.7977346 : Labels ~ 1 + V9617 + V782 + V2256 + V8055 + V4301 + V1874
#> 9 : 3317 : 0.78737 : Labels ~ 1 + V414 + V9965 + V4557 + V9448 + V4406
#>
#> Num. Models: 10 To Test: 61 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V4352 + V3365 + V9215 + V686 + V5321 + V2556 + V3170 + V3647
#> At Accuracy: Labels ~ 1 + V4352 + V3365 + V9215 + V686 + V5321 + V2556 + V3170 + V3647
#> B:SWiMS : Labels ~ 1 + V4352 + V3365 + V9215 + V686 + V5321 + V2556 + V3170 + V3647
#>
#> Num. Models: 320 To Test: 1281 TopFreq: 12.56667 Thrf: 0 Removed: 0
#> ................................*Loop : 28 Blind Cases = 9 Blind Control = 11 Total = 562 Size Cases = 248 Size Control = 314
#> Accumulated Models CV Accuracy = 0.797153 Sensitivity = 0.7701613 Specificity = 0.8184713 Forw. Ensemble Accuracy= 0.8434164
#> Initial Model Accumulated CV Accuracy = 0.8754448 Sensitivity = 0.891129 Specificity = 0.8630573
#> Initial Model Bootstrapped Accuracy = 0.8473274 Sensitivity = 0.8796797 Specificity = 0.8149751
#> Current Model Bootstrapped Accuracy = 0.8683638 Sensitivity = 0.8833768 Specificity = 0.8533507
#> Current KNN Accuracy = 0.7437722 Sensitivity = 0.891129 Specificity = 0.6273885
#> Initial KNN Accuracy = 0.7989324 Sensitivity = 0.8951613 Specificity = 0.7229299
#> Train Correlation: 0.8070455 Blind Correlation : 0.9052632
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 212 12
#> TRUE 102 236
#> Loop : 29 Input Cases = 88 Input Control = 112
#> Loop : 29 Train Cases = 80 Train Control = 101
#> Loop : 29 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8777216 : Labels ~ 1 + V2556 + V6584 + V4158 + V7513 + V7402 + V4352 + V6163 + V4301
#> 1 : 3365 : 0.8649382 : Labels ~ 1 + V5005 + V7857 + V4584 + V3365 + V9818 + V9070 + V3161 + V6685 + V9528
#> 2 : 3356 : 0.8464419 : Labels ~ 1 + V7748 + V8368 + V3014 + V1762 + V22 + V7834 + V7373
#> 3 : 3349 : 0.8136443 : Labels ~ 1 + V8806 + V312 + V5761 + V762 + V1344
#> 4 : 3344 : 0.8372866 : Labels ~ 1 + V2309 + V86 + V9215 + V220 + V4534 + V3170 + V7195
#> 5 : 3337 : 0.7576553 : Labels ~ 1 + V9275 + V9617 + V5400
#> 6 : 3334 : 0.8363795 : Labels ~ 1 + V4960 + V4326 + V698 + V2438 + V6913 + V7842
#> 7 : 3328 : 0.7897217 : Labels ~ 1 + V414 + V1831 + V4077 + V6692
#> 8 : 3324 : 0.8056475 : Labels ~ 1 + V1476 + V5680 + V9027 + V4900 + V9213
#> 9 : 3319 : 0.8645267 : Labels ~ 1 + V436 + V376 + V7319 + V2804 + V7960 + V2462 + V3592 + V723
#>
#> Num. Models: 10 To Test: 62 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V5005 + V6584 + V4158 + V7513 + V1118 + V7402 + V4352 + V6163 + V4301
#> At Accuracy: Labels ~ 1 + V2556 + V6584 + V4158 + V7513 + V7402 + V4352 + V6163 + V4301
#> B:SWiMS : Labels ~ 1 + V2556 + V6584 + V4158 + V7513 + V7402 + V4352 + V6163 + V4301
#>
#> Num. Models: 320 To Test: 1298 TopFreq: 13.45946 Thrf: 0 Removed: 0
#> ................................*Loop : 29 Blind Cases = 8 Blind Control = 11 Total = 581 Size Cases = 256 Size Control = 325
#> Accumulated Models CV Accuracy = 0.7934596 Sensitivity = 0.765625 Specificity = 0.8153846 Forw. Ensemble Accuracy= 0.8416523
#> Initial Model Accumulated CV Accuracy = 0.8743546 Sensitivity = 0.890625 Specificity = 0.8615385
#> Initial Model Bootstrapped Accuracy = 0.8466387 Sensitivity = 0.8635559 Specificity = 0.8297214
#> Current Model Bootstrapped Accuracy = 0.8777216 Sensitivity = 0.9027931 Specificity = 0.8526501
#> Current KNN Accuracy = 0.7401033 Sensitivity = 0.8828125 Specificity = 0.6276923
#> Initial KNN Accuracy = 0.7934596 Sensitivity = 0.8867188 Specificity = 0.72
#> Train Correlation: 0.7767733 Blind Correlation : 0.7789474
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 222 12
#> TRUE 103 244
#> Loop : 30 Input Cases = 88 Input Control = 112
#> Loop : 30 Train Cases = 80 Train Control = 101
#> Loop : 30 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8696832 : Labels ~ 1 + V3170 + V7748 + V9275 + V9082 + V4960 + V1943 + V5321 + V2526 + V5754
#> 1 : 3364 : 0.8280563 : Labels ~ 1 + V1184 + V7857 + V3365 + V9818 + V729 + V6688
#> 2 : 3358 : 0.8538933 : Labels ~ 1 + V86 + V312 + V7513 + V1831 + V8770 + V8117 + V9965 + V3592
#> 3 : 3350 : 0.8154488 : Labels ~ 1 + V9215 + V2309 + V7928 + V4352 + V3554
#> 4 : 3345 : 0.8625192 : Labels ~ 1 + V698 + V8368 + V723 + V4326 + V307 + V6594 + V2358 + V256
#> 5 : 3337 : 0.8366812 : Labels ~ 1 + V414 + V5680 + V5400 + V4301 + V52 + V3165
#> 6 : 3331 : 0.8200574 : Labels ~ 1 + V4973 + V4584 + V9617 + V130 + V6584 + V2646
#> 7 : 3325 : 0.7931525 : Labels ~ 1 + V4290 + V3014 + V9027 + V7584
#> 8 : 3321 : 0.8174776 : Labels ~ 1 + V7994 + V6163 + V3676 + V9213
#> 9 : 3317 : 0.8090316 : Labels ~ 1 + V1476 + V7212 + V5761 + V4289 + V3986
#>
#> Num. Models: 10 To Test: 61 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V3170 + V7748 + V9275 + V9082 + V4960 + V1943 + V5321 + V2526 + V5754
#> At Accuracy: Labels ~ 1 + V3170 + V7748 + V9275 + V9082 + V4960 + V1943 + V5321 + V2526 + V5754
#> B:SWiMS : Labels ~ 1 + V3170 + V7748 + V9275 + V9082 + V4960 + V1943 + V5321 + V2526 + V5754
#>
#> Num. Models: 320 To Test: 1384 TopFreq: 12.97297 Thrf: 0 Removed: 0
#> ................................*Loop : 30 Blind Cases = 8 Blind Control = 11 Total = 600 Size Cases = 264 Size Control = 336
#> Accumulated Models CV Accuracy = 0.7933333 Sensitivity = 0.7613636 Specificity = 0.8184524 Forw. Ensemble Accuracy= 0.845
#> Initial Model Accumulated CV Accuracy = 0.875 Sensitivity = 0.8939394 Specificity = 0.860119
#> Initial Model Bootstrapped Accuracy = 0.8473266 Sensitivity = 0.8745029 Specificity = 0.8201502
#> Current Model Bootstrapped Accuracy = 0.8696832 Sensitivity = 0.8886719 Specificity = 0.8506944
#> Current KNN Accuracy = 0.7416667 Sensitivity = 0.8825758 Specificity = 0.6309524
#> Initial KNN Accuracy = 0.7966667 Sensitivity = 0.8863636 Specificity = 0.7261905
#> Train Correlation: 0.8079169 Blind Correlation : 0.6614035
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 231 12
#> TRUE 107 250
#> Loop : 31 Input Cases = 88 Input Control = 112
#> Loop : 31 Train Cases = 79 Train Control = 100
#> Loop : 31 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8823852 : Labels ~ 1 + V2556 + V5005 + V9275 + V1055 + V4584 + V3161 + V720
#> 1 : 3366 : 0.8501539 : Labels ~ 1 + V7891 + V9215 + V7272 + V6163 + V3170 + V7381
#> 2 : 3360 : 0.8100919 : Labels ~ 1 + V6594 + V9818 + V2206 + V4301
#> 3 : 3356 : 0.8237039 : Labels ~ 1 + V5808 + V1831 + V1248 + V4352 + V1046 + V5378
#> 4 : 3350 : 0.8392032 : Labels ~ 1 + V729 + V7748 + V3206 + V2294 + V6584 + V3612
#> 5 : 3344 : 0.7969088 : Labels ~ 1 + V1936 + V9213 + V4183 + V9965 + V5837
#> 6 : 3339 : 0.8509488 : Labels ~ 1 + V9735 + V5761 + V1476 + V9395 + V4564 + V8323 + V3860
#> 7 : 3332 : 0.7720798 : Labels ~ 1 + V5801 + V8156 + V4406
#> 8 : 3329 : 0.8593887 : Labels ~ 1 + V9432 + V4580 + V3014 + V1087 + V8038 + V5107 + V7857
#> 9 : 3322 : 0.8414261 : Labels ~ 1 + V4070 + V5680 + V1975 + V7834 + V8484 + V8972 + V2438
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V5005 + V9275 + V1055 + V4584 + V3161 + V720
#> At Accuracy: Labels ~ 1 + V2556 + V5005 + V9275 + V1055 + V4584 + V3161 + V720
#> B:SWiMS : Labels ~ 1 + V2556 + V5005 + V9275 + V1055 + V4584 + V3161 + V720
#>
#> Num. Models: 320 To Test: 1246 TopFreq: 12.23404 Thrf: 0 Removed: 0
#> ................................*Loop : 31 Blind Cases = 9 Blind Control = 12 Total = 621 Size Cases = 273 Size Control = 348
#> Accumulated Models CV Accuracy = 0.7890499 Sensitivity = 0.7582418 Specificity = 0.8132184 Forw. Ensemble Accuracy= 0.84219
#> Initial Model Accumulated CV Accuracy = 0.8760064 Sensitivity = 0.8937729 Specificity = 0.862069
#> Initial Model Bootstrapped Accuracy = 0.8538479 Sensitivity = 0.8762475 Specificity = 0.8314482
#> Current Model Bootstrapped Accuracy = 0.8823852 Sensitivity = 0.8943554 Specificity = 0.8704151
#> Current KNN Accuracy = 0.7391304 Sensitivity = 0.8791209 Specificity = 0.6293103
#> Initial KNN Accuracy = 0.7954911 Sensitivity = 0.8864469 Specificity = 0.7241379
#> Train Correlation: 0.8066286 Blind Correlation : 0.9207792
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 240 12
#> TRUE 109 260
#> Loop : 32 Input Cases = 88 Input Control = 112
#> Loop : 32 Train Cases = 79 Train Control = 100
#> Loop : 32 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8320796 : Labels ~ 1 + V5005 + V723 + V9275 + V4301 + V8650 + V2597
#> 1 : 3367 : 0.8210549 : Labels ~ 1 + V698 + V1936 + V9818 + V4158 + V9156 + V3675 + V3119
#> 2 : 3360 : 0.8133245 : Labels ~ 1 + V7748 + V3365 + V6584 + V4584 + V8726
#> 3 : 3355 : 0.864083 : Labels ~ 1 + V1476 + V8368 + V1831 + V5400 + V2636 + V7891 + V9965 + V2462
#> 4 : 3347 : 0.8488218 : Labels ~ 1 + V4290 + V312 + V9215 + V5995 + V5514 + V4352 + V819 + V1344 + V3592
#> 5 : 3338 : 0.8018535 : Labels ~ 1 + V436 + V2309 + V256 + V729 + V9213
#> 6 : 3333 : 0.7738042 : Labels ~ 1 + V86 + V4960 + V5761
#> 7 : 3330 : 0.818996 : Labels ~ 1 + V414 + V7513 + V4326 + V3170 + V8193 + V5321
#> 8 : 3324 : 0.8212216 : Labels ~ 1 + V2358 + V9027 + V2256 + V4406 + V762 + V130 + V6408
#> 9 : 3317 : 0.8136673 : Labels ~ 1 + V9617 + V6163 + V22 + V3161 + V2644 + V8247
#>
#> Num. Models: 10 To Test: 62 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V723 + V9275 + V4301 + V8650 + V2597
#> At Accuracy: Labels ~ 1 + V5005 + V723 + V9275 + V4301 + V8650 + V2597
#> B:SWiMS : Labels ~ 1 + V5005 + V723 + V9275 + V4301 + V8650 + V2597
#>
#> Num. Models: 320 To Test: 1369 TopFreq: 13.375 Thrf: 0 Removed: 0
#> ................................*Loop : 32 Blind Cases = 9 Blind Control = 12 Total = 642 Size Cases = 282 Size Control = 360
#> Accumulated Models CV Accuracy = 0.7928349 Sensitivity = 0.7624113 Specificity = 0.8166667 Forw. Ensemble Accuracy= 0.8457944
#> Initial Model Accumulated CV Accuracy = 0.876947 Sensitivity = 0.893617 Specificity = 0.8638889
#> Initial Model Bootstrapped Accuracy = 0.8436118 Sensitivity = 0.8740053 Specificity = 0.8132184
#> Current Model Bootstrapped Accuracy = 0.8320796 Sensitivity = 0.8816372 Specificity = 0.7825221
#> Current KNN Accuracy = 0.7429907 Sensitivity = 0.8794326 Specificity = 0.6361111
#> Initial KNN Accuracy = 0.7975078 Sensitivity = 0.8865248 Specificity = 0.7277778
#> Train Correlation: 0.7620279 Blind Correlation : 0.8363636
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 251 12
#> TRUE 110 269
#> Loop : 33 Input Cases = 88 Input Control = 112
#> Loop : 33 Train Cases = 79 Train Control = 101
#> Loop : 33 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7972381 : Labels ~ 1 + V5005 + V698 + V4584 + V7272
#> 1 : 3369 : 0.8502604 : Labels ~ 1 + V1936 + V4290 + V9275 + V5400 + V9395 + V2991 + V3778
#> 2 : 3362 : 0.8804936 : Labels ~ 1 + V7748 + V9215 + V3206 + V723 + V8239 + V3545 + V3161 + V9965 + V8770
#> 3 : 3353 : 0.8461116 : Labels ~ 1 + V4158 + V7513 + V8502 + V9818 + V6959 + V3826 + V4564
#> 4 : 3346 : 0.8202307 : Labels ~ 1 + V5473 + V1055 + V5680 + V311 + V6688
#> 5 : 3341 : 0.8378144 : Labels ~ 1 + V1476 + V3014 + V6239 + V7584 + V5321 + V3592
#> 6 : 3335 : 0.8151915 : Labels ~ 1 + V436 + V4666 + V1831 + V729 + V6958
#> 7 : 3330 : 0.8253351 : Labels ~ 1 + V7197 + V4194 + V6584 + V6163 + V1512 + V2064
#> 8 : 3324 : 0.8283743 : Labels ~ 1 + V86 + V7928 + V898 + V2309 + V1458 + V4554
#> 9 : 3318 : 0.7813246 : Labels ~ 1 + V7857 + V5761 + V8744 + V7402
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V698 + V4584 + V7272
#> At Accuracy: Labels ~ 1 + V5005 + V698 + V4584 + V7272
#> B:SWiMS : Labels ~ 1 + V5005 + V698 + V4584 + V7272
#>
#> Num. Models: 320 To Test: 1276 TopFreq: 15.52273 Thrf: 0 Removed: 0
#> ................................*Loop : 33 Blind Cases = 9 Blind Control = 11 Total = 662 Size Cases = 291 Size Control = 371
#> Accumulated Models CV Accuracy = 0.7900302 Sensitivity = 0.7560137 Specificity = 0.8167116 Forw. Ensemble Accuracy= 0.8444109
#> Initial Model Accumulated CV Accuracy = 0.8746224 Sensitivity = 0.8900344 Specificity = 0.8625337
#> Initial Model Bootstrapped Accuracy = 0.8549323 Sensitivity = 0.8753492 Specificity = 0.8345154
#> Current Model Bootstrapped Accuracy = 0.7972381 Sensitivity = 0.8331872 Specificity = 0.7612889
#> Current KNN Accuracy = 0.7432024 Sensitivity = 0.8797251 Specificity = 0.6361186
#> Initial KNN Accuracy = 0.8036254 Sensitivity = 0.8900344 Specificity = 0.7358491
#> Train Correlation: 0.8246997 Blind Correlation : 0.8090226
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 258 13
#> TRUE 116 275
#> Loop : 34 Input Cases = 88 Input Control = 112
#> Loop : 34 Train Cases = 79 Train Control = 101
#> Loop : 34 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8359543 : Labels ~ 1 + V5005 + V9215 + V312 + V4352 + V22 + V6570
#> 1 : 3367 : 0.8331186 : Labels ~ 1 + V3365 + V1831 + V3161 + V723 + V9965 + V4584
#> 2 : 3361 : 0.8108138 : Labels ~ 1 + V8368 + V5761 + V86 + V1248
#> 3 : 3357 : 0.837644 : Labels ~ 1 + V4960 + V6584 + V8411 + V6163 + V3170 + V6970
#> 4 : 3351 : 0.8389035 : Labels ~ 1 + V414 + V9818 + V9156 + V7857 + V2418
#> 5 : 3346 : 0.804145 : Labels ~ 1 + V2309 + V7748 + V9275 + V4185
#> 6 : 3342 : 0.8165944 : Labels ~ 1 + V9027 + V4326 + V436 + V6688
#> 7 : 3338 : 0.7986096 : Labels ~ 1 + V9617 + V1476 + V5680 + V3708
#> 8 : 3334 : 0.8254003 : Labels ~ 1 + V8806 + V376 + V5417 + V9395 + V6146
#> 9 : 3329 : 0.8281418 : Labels ~ 1 + V698 + V8156 + V7994 + V686 + V3675
#>
#> Num. Models: 10 To Test: 49 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V9215 + V312 + V2785 + V4352 + V22 + V6570
#> At Accuracy: Labels ~ 1 + V5005 + V9215 + V312 + V4352 + V22 + V6570
#> B:SWiMS : Labels ~ 1 + V5005 + V9215 + V312 + V4352 + V22 + V6570
#>
#> Num. Models: 320 To Test: 1222 TopFreq: 14.97959 Thrf: 0 Removed: 0
#> ................................*Loop : 34 Blind Cases = 9 Blind Control = 11 Total = 682 Size Cases = 300 Size Control = 382
#> Accumulated Models CV Accuracy = 0.7844575 Sensitivity = 0.7533333 Specificity = 0.8089005 Forw. Ensemble Accuracy= 0.8416422
#> Initial Model Accumulated CV Accuracy = 0.8739003 Sensitivity = 0.8933333 Specificity = 0.8586387
#> Initial Model Bootstrapped Accuracy = 0.8517023 Sensitivity = 0.8662156 Specificity = 0.837189
#> Current Model Bootstrapped Accuracy = 0.8359543 Sensitivity = 0.8300302 Specificity = 0.8418785
#> Current KNN Accuracy = 0.7419355 Sensitivity = 0.8833333 Specificity = 0.6308901
#> Initial KNN Accuracy = 0.8020528 Sensitivity = 0.8933333 Specificity = 0.7303665
#> Train Correlation: 0.8133811 Blind Correlation : 0.6571429
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 263 13
#> TRUE 120 286
#> Loop : 35 Input Cases = 88 Input Control = 112
#> Loop : 35 Train Cases = 79 Train Control = 101
#> Loop : 35 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8333695 : Labels ~ 1 + V9215 + V312 + V4352 + V6111 + V7183
#> 1 : 3368 : 0.8019132 : Labels ~ 1 + V3365 + V9275 + V5400 + V8466 + V1184
#> 2 : 3363 : 0.8643554 : Labels ~ 1 + V8368 + V1248 + V9818 + V7748 + V9086 + V2556 + V3170 + V8247
#> 3 : 3355 : 0.8060652 : Labels ~ 1 + V4960 + V4326 + V3708 + V4290
#> 4 : 3351 : 0.8007847 : Labels ~ 1 + V5761 + V2309 + V1046 + V2515
#> 5 : 3347 : 0.8207283 : Labels ~ 1 + V414 + V6584 + V6688 + V698 + V4584
#> 6 : 3342 : 0.7675448 : Labels ~ 1 + V5005 + V1831 + V5 + V9395
#> 7 : 3338 : 0.8234277 : Labels ~ 1 + V9027 + V3189 + V4301 + V6163 + V2780
#> 8 : 3333 : 0.8122318 : Labels ~ 1 + V3014 + V9617 + V3620 + V1606 + V8038
#> 9 : 3328 : 0.8480596 : Labels ~ 1 + V6164 + V5347 + V9031 + V5572 + V9833 + V4520
#>
#> Num. Models: 10 To Test: 51 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V9215 + V312 + V4352 + V6111 + V7183
#> At Accuracy: Labels ~ 1 + V9215 + V312 + V4352 + V6111 + V7183
#> B:SWiMS : Labels ~ 1 + V9215 + V312 + V4352 + V6111 + V7183
#>
#> Num. Models: 320 To Test: 1354 TopFreq: 10.97872 Thrf: 0 Removed: 0
#> ................................*Loop : 35 Blind Cases = 9 Blind Control = 11 Total = 702 Size Cases = 309 Size Control = 393
#> Accumulated Models CV Accuracy = 0.7834758 Sensitivity = 0.7508091 Specificity = 0.8091603 Forw. Ensemble Accuracy= 0.8418803
#> Initial Model Accumulated CV Accuracy = 0.8732194 Sensitivity = 0.8932039 Specificity = 0.8575064
#> Initial Model Bootstrapped Accuracy = 0.8548282 Sensitivity = 0.8690735 Specificity = 0.8405829
#> Current Model Bootstrapped Accuracy = 0.8333695 Sensitivity = 0.8484585 Specificity = 0.8182805
#> Current KNN Accuracy = 0.7407407 Sensitivity = 0.8802589 Specificity = 0.6310433
#> Initial KNN Accuracy = 0.7977208 Sensitivity = 0.8867314 Specificity = 0.7277354
#> Train Correlation: 0.7561591 Blind Correlation : 0.8887218
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 270 15
#> TRUE 125 292
#> Loop : 36 Input Cases = 88 Input Control = 112
#> Loop : 36 Train Cases = 79 Train Control = 101
#> Loop : 36 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8591854 : Labels ~ 1 + V4352 + V1975 + V6163 + V4554 + V9275 + V5321 + V9919 + V1344
#> 1 : 3365 : 0.7960242 : Labels ~ 1 + V5005 + V4183 + V6584 + V130 + V1184
#> 2 : 3360 : 0.8144162 : Labels ~ 1 + V1936 + V7748 + V898 + V3170 + V9395
#> 3 : 3355 : 0.8262966 : Labels ~ 1 + V698 + V9215 + V2804 + V3077 + V5378 + V5881 + V8368 + V819
#> 4 : 3347 : 0.858183 : Labels ~ 1 + V4290 + V1831 + V9965 + V1248 + V1173 + V8055 + V2445
#> 5 : 3340 : 0.8234788 : Labels ~ 1 + V8502 + V4584 + V2309 + V1762 + V8359 + V9818
#> 6 : 3334 : 0.8133772 : Labels ~ 1 + V1476 + V4326 + V312 + V8681 + V9213
#> 7 : 3329 : 0.7897715 : Labels ~ 1 + V436 + V5680 + V3365 + V3082 + V5107
#> 8 : 3324 : 0.8007364 : Labels ~ 1 + V729 + V9617 + V6408 + V4280
#> 9 : 3320 : 0.8185073 : Labels ~ 1 + V7891 + V8156 + V4685 + V9156 + V2358 + V4301
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V4352 + V5005 + V1975 + V6163 + V4554 + V9275 + V5321 + V8623 + V9919 + V1344
#> At Accuracy: Labels ~ 1 + V4352 + V1975 + V6163 + V4554 + V9275 + V5321 + V9919 + V1344
#> B:SWiMS : Labels ~ 1 + V4352 + V1975 + V6163 + V4554 + V9275 + V5321 + V9919 + V1344
#>
#> Num. Models: 320 To Test: 1338 TopFreq: 13.76471 Thrf: 0 Removed: 0
#> ................................*Loop : 36 Blind Cases = 9 Blind Control = 11 Total = 722 Size Cases = 318 Size Control = 404
#> Accumulated Models CV Accuracy = 0.7853186 Sensitivity = 0.7515723 Specificity = 0.8118812 Forw. Ensemble Accuracy= 0.8448753
#> Initial Model Accumulated CV Accuracy = 0.8711911 Sensitivity = 0.8899371 Specificity = 0.8564356
#> Initial Model Bootstrapped Accuracy = 0.8589937 Sensitivity = 0.8797236 Specificity = 0.8382639
#> Current Model Bootstrapped Accuracy = 0.8591854 Sensitivity = 0.8732669 Specificity = 0.845104
#> Current KNN Accuracy = 0.7423823 Sensitivity = 0.8805031 Specificity = 0.6336634
#> Initial KNN Accuracy = 0.799169 Sensitivity = 0.8899371 Specificity = 0.7277228
#> Train Correlation: 0.8260564 Blind Correlation : 0.9278195
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 279 15
#> TRUE 128 300
#> Loop : 37 Input Cases = 88 Input Control = 112
#> Loop : 37 Train Cases = 79 Train Control = 101
#> Loop : 37 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.7925693 : Labels ~ 1 + V2556 + V3952 + V9818 + V9919
#> 1 : 3369 : 0.8646511 : Labels ~ 1 + V7891 + V1831 + V686 + V5321 + V698 + V2973
#> 2 : 3363 : 0.8302462 : Labels ~ 1 + V4290 + V4738 + V3170 + V1248 + V4584
#> 3 : 3358 : 0.8547444 : Labels ~ 1 + V8502 + V5400 + V9275 + V6594 + V6959 + V9213 + V8726
#> 4 : 3351 : 0.8388769 : Labels ~ 1 + V7748 + V9215 + V3365 + V723 + V3554 + V389 + V7426
#> 5 : 3344 : 0.8442783 : Labels ~ 1 + V5796 + V1476 + V6584 + V7272 + V7435 + V8650
#> 6 : 3338 : 0.8293316 : Labels ~ 1 + V5473 + V7928 + V729 + V4194 + V1344 + V256 + V8117 + V1865
#> 7 : 3330 : 0.821796 : Labels ~ 1 + V8156 + V3629 + V436 + V4077 + V6530 + V2294
#> 8 : 3324 : 0.8289814 : Labels ~ 1 + V86 + V1400 + V9965 + V2515 + V3987 + V130
#> 9 : 3318 : 0.7910921 : Labels ~ 1 + V7857 + V7513 + V4326 + V4301 + V2597
#>
#> Num. Models: 10 To Test: 60 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V3952 + V9818 + V9919
#> At Accuracy: Labels ~ 1 + V2556 + V3952 + V9818 + V9919
#> B:SWiMS : Labels ~ 1 + V2556 + V3952 + V9818 + V9919
#>
#> Num. Models: 320 To Test: 1280 TopFreq: 12.36667 Thrf: 0 Removed: 0
#> ................................*Loop : 37 Blind Cases = 9 Blind Control = 11 Total = 742 Size Cases = 327 Size Control = 415
#> Accumulated Models CV Accuracy = 0.7843666 Sensitivity = 0.7461774 Specificity = 0.8144578 Forw. Ensemble Accuracy= 0.8436658
#> Initial Model Accumulated CV Accuracy = 0.8692722 Sensitivity = 0.883792 Specificity = 0.8578313
#> Initial Model Bootstrapped Accuracy = 0.8686014 Sensitivity = 0.9031913 Specificity = 0.8340116
#> Current Model Bootstrapped Accuracy = 0.7925693 Sensitivity = 0.8156412 Specificity = 0.7694974
#> Current KNN Accuracy = 0.7385445 Sensitivity = 0.8776758 Specificity = 0.6289157
#> Initial KNN Accuracy = 0.7991914 Sensitivity = 0.8929664 Specificity = 0.7253012
#> Train Correlation: 0.8046997 Blind Correlation : 0.6406015
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 286 15
#> TRUE 135 306
#> Loop : 38 Input Cases = 88 Input Control = 112
#> Loop : 38 Train Cases = 79 Train Control = 101
#> Loop : 38 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8361957 : Labels ~ 1 + V1476 + V7272 + V9818 + V4584 + V9371 + V9082
#> 1 : 3367 : 0.8265909 : Labels ~ 1 + V7748 + V3365 + V1831 + V5761 + V3964 + V1344
#> 2 : 3361 : 0.8171577 : Labels ~ 1 + V312 + V4326 + V436 + V1248
#> 3 : 3357 : 0.8374701 : Labels ~ 1 + V4960 + V4973 + V9275 + V6163 + V3170 + V4198
#> 4 : 3351 : 0.8536953 : Labels ~ 1 + V414 + V34 + V729 + V2489 + V649 + V9129
#> 5 : 3345 : 0.7717108 : Labels ~ 1 + V9215 + V9617 + V8135
#> 6 : 3342 : 0.8028412 : Labels ~ 1 + V8368 + V86 + V8156 + V1097 + V9395
#> 7 : 3337 : 0.787146 : Labels ~ 1 + V2309 + V7857 + V5683 + V8038
#> 8 : 3333 : 0.8321792 : Labels ~ 1 + V9027 + V898 + V698 + V6688
#> 9 : 3329 : 0.7851562 : Labels ~ 1 + V4290 + V376 + V6584 + V4185
#>
#> Num. Models: 10 To Test: 48 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V1476 + V7272 + V9818 + V4584 + V9371 + V9082
#> At Accuracy: Labels ~ 1 + V1476 + V7272 + V9818 + V4584 + V9371 + V9082
#> B:SWiMS : Labels ~ 1 + V1476 + V7272 + V9818 + V4584 + V9371 + V9082
#>
#> Num. Models: 320 To Test: 1253 TopFreq: 14.93617 Thrf: 0 Removed: 0
#> ................................*Loop : 38 Blind Cases = 9 Blind Control = 11 Total = 762 Size Cases = 336 Size Control = 426
#> Accumulated Models CV Accuracy = 0.7821522 Sensitivity = 0.7470238 Specificity = 0.8098592 Forw. Ensemble Accuracy= 0.8412073
#> Initial Model Accumulated CV Accuracy = 0.8687664 Sensitivity = 0.8869048 Specificity = 0.8544601
#> Initial Model Bootstrapped Accuracy = 0.8518917 Sensitivity = 0.8755374 Specificity = 0.8282459
#> Current Model Bootstrapped Accuracy = 0.8361957 Sensitivity = 0.8884783 Specificity = 0.783913
#> Current KNN Accuracy = 0.7375328 Sensitivity = 0.8809524 Specificity = 0.6244131
#> Initial KNN Accuracy = 0.7965879 Sensitivity = 0.8928571 Specificity = 0.7206573
#> Train Correlation: 0.7543745 Blind Correlation : 0.5413534
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 291 15
#> TRUE 139 317
#> Loop : 39 Input Cases = 88 Input Control = 112
#> Loop : 39 Train Cases = 80 Train Control = 101
#> Loop : 39 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8725212 : Labels ~ 1 + V7857 + V6584 + V2636 + V723 + V3778 + V7891 + V3272
#> 1 : 3366 : 0.8812582 : Labels ~ 1 + V9275 + V6594 + V7272 + V6163 + V3161 + V3547 + V5321 + V7260 + V4352
#> 2 : 3357 : 0.8157664 : Labels ~ 1 + V5005 + V4200 + V9818 + V9213 + V3206 + V7402
#> 3 : 3351 : 0.8008734 : Labels ~ 1 + V3592 + V8368 + V1831 + V4584 + V2747
#> 4 : 3346 : 0.8232334 : Labels ~ 1 + V6440 + V1975 + V3365 + V5680 + V3170 + V7435
#> 5 : 3340 : 0.8289823 : Labels ~ 1 + V86 + V2309 + V4326 + V7367 + V4301
#> 6 : 3335 : 0.8530505 : Labels ~ 1 + V1055 + V7748 + V9215 + V7513 + V729 + V2294 + V2783 + V1025
#> 7 : 3327 : 0.7991141 : Labels ~ 1 + V8806 + V312 + V8156 + V4685 + V3861
#> 8 : 3322 : 0.8027562 : Labels ~ 1 + V2556 + V436 + V5417 + V1248 + V1144
#> 9 : 3317 : 0.8483464 : Labels ~ 1 + V5378 + V9617 + V22 + V649 + V9129 + V686
#>
#> Num. Models: 10 To Test: 62 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V7857 + V3170 + V6584 + V4183 + V2636 + V723 + V3778 + V7891 + V3272 + V4178
#> At Accuracy: Labels ~ 1 + V7857 + V6584 + V2636 + V723 + V3778 + V7891 + V3272
#> B:SWiMS : Labels ~ 1 + V7857 + V6584 + V2636 + V723 + V3778 + V7891 + V3272
#>
#> Num. Models: 320 To Test: 1340 TopFreq: 17.77143 Thrf: 0 Removed: 0
#> ................................*Loop : 39 Blind Cases = 8 Blind Control = 11 Total = 781 Size Cases = 344 Size Control = 437
#> Accumulated Models CV Accuracy = 0.7836108 Sensitivity = 0.747093 Specificity = 0.812357 Forw. Ensemble Accuracy= 0.8412292
#> Initial Model Accumulated CV Accuracy = 0.871959 Sensitivity = 0.8895349 Specificity = 0.8581236
#> Initial Model Bootstrapped Accuracy = 0.8452536 Sensitivity = 0.8597746 Specificity = 0.8307326
#> Current Model Bootstrapped Accuracy = 0.8725212 Sensitivity = 0.8888647 Specificity = 0.8561778
#> Current KNN Accuracy = 0.737516 Sensitivity = 0.877907 Specificity = 0.6270023
#> Initial KNN Accuracy = 0.7951344 Sensitivity = 0.8895349 Specificity = 0.7208238
#> Train Correlation: 0.7195556 Blind Correlation : 0.7175439
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 301 15
#> TRUE 141 324
#> Loop : 40 Input Cases = 88 Input Control = 112
#> Loop : 40 Train Cases = 80 Train Control = 101
#> Loop : 40 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8204629 : Labels ~ 1 + V3365 + V9070 + V729 + V4290 + V9818
#> 1 : 3368 : 0.7993848 : Labels ~ 1 + V5005 + V9215 + V1975 + V1248 + V130
#> 2 : 3363 : 0.8366133 : Labels ~ 1 + V2556 + V4183 + V34 + V9275 + V9156 + V3170
#> 3 : 3357 : 0.8455978 : Labels ~ 1 + V7891 + V6163 + V167 + V469 + V2515 + V2358 + V8193
#> 4 : 3350 : 0.8494718 : Labels ~ 1 + V698 + V312 + V6584 + V4584 + V1762 + V3545 + V3161
#> 5 : 3343 : 0.8183294 : Labels ~ 1 + V7748 + V5417 + V4960 + V2438 + V8586
#> 6 : 3338 : 0.8150308 : Labels ~ 1 + V8368 + V872 + V8156 + V7381 + V9213 + V4352
#> 7 : 3332 : 0.8153745 : Labels ~ 1 + V5 + V414 + V1198 + V4900 + V4580
#> 8 : 3327 : 0.8016901 : Labels ~ 1 + V6594 + V1831 + V8267 + V9395 + V4301
#> 9 : 3322 : 0.7975125 : Labels ~ 1 + V2309 + V782 + V2256 + V256 + V5107
#>
#> Num. Models: 10 To Test: 56 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V3365 + V5005 + V9070 + V729 + V4290 + V9818
#> At Accuracy: Labels ~ 1 + V3365 + V9070 + V729 + V4290 + V9818
#> B:SWiMS : Labels ~ 1 + V3365 + V9070 + V729 + V4290 + V9818
#>
#> Num. Models: 320 To Test: 1318 TopFreq: 16.55263 Thrf: 0 Removed: 0
#> ................................*Loop : 40 Blind Cases = 8 Blind Control = 11 Total = 800 Size Cases = 352 Size Control = 448
#> Accumulated Models CV Accuracy = 0.78125 Sensitivity = 0.7471591 Specificity = 0.8080357 Forw. Ensemble Accuracy= 0.83875
#> Initial Model Accumulated CV Accuracy = 0.8725 Sensitivity = 0.8920455 Specificity = 0.8571429
#> Initial Model Bootstrapped Accuracy = 0.8469588 Sensitivity = 0.8696316 Specificity = 0.824286
#> Current Model Bootstrapped Accuracy = 0.8204629 Sensitivity = 0.8462038 Specificity = 0.794722
#> Current KNN Accuracy = 0.73625 Sensitivity = 0.875 Specificity = 0.6272321
#> Initial KNN Accuracy = 0.79375 Sensitivity = 0.8863636 Specificity = 0.7209821
#> Train Correlation: 0.7985591 Blind Correlation : 0.8614035
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 309 16
#> TRUE 142 333
#> Loop : 41 Input Cases = 88 Input Control = 112
#> Loop : 41 Train Cases = 79 Train Control = 100
#> Loop : 41 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8116782 : Labels ~ 1 + V723 + V5417 + V4194 + V3170 + V2597 + V6584
#> 1 : 3367 : 0.8458555 : Labels ~ 1 + V5400 + V4960 + V9215 + V1184 + V52 + V8618 + V4580
#> 2 : 3360 : 0.8498136 : Labels ~ 1 + V7513 + V7272 + V9275 + V2264 + V1344 + V4352 + V5321 + V4584
#> 3 : 3352 : 0.8528571 : Labels ~ 1 + V7928 + V8193 + V256 + V3365 + V4301 + V729 + V6685
#> 4 : 3345 : 0.8733497 : Labels ~ 1 + V312 + V7212 + V1831 + V6380 + V698 + V7026 + V1173 + V3161 + V8181
#> 5 : 3336 : 0.8030069 : Labels ~ 1 + V8368 + V5683 + V4290 + V4185
#> 6 : 3332 : 0.8241674 : Labels ~ 1 + V7748 + V9818 + V2309 + V6114 + V6959 + V8245
#> 7 : 3326 : 0.7774042 : Labels ~ 1 + V414 + V8720 + V8055 + V1046 + V7085
#> 8 : 3321 : 0.8008879 : Labels ~ 1 + V4158 + V1476 + V9070 + V4326
#> 9 : 3317 : 0.7874343 : Labels ~ 1 + V8156 + V9965 + V9027 + V1602 + V6163
#>
#> Num. Models: 10 To Test: 61 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V723 + V698 + V5417 + V4194 + V3170 + V2597 + V6584
#> At Accuracy: Labels ~ 1 + V723 + V5417 + V4194 + V3170 + V2597 + V6584
#> B:SWiMS : Labels ~ 1 + V723 + V5417 + V4194 + V3170 + V2597 + V6584
#>
#> Num. Models: 320 To Test: 1346 TopFreq: 10.54545 Thrf: 0 Removed: 0
#> ................................*Loop : 41 Blind Cases = 9 Blind Control = 12 Total = 821 Size Cases = 361 Size Control = 460
#> Accumulated Models CV Accuracy = 0.7819732 Sensitivity = 0.7534626 Specificity = 0.8043478 Forw. Ensemble Accuracy= 0.8404385
#> Initial Model Accumulated CV Accuracy = 0.8721072 Sensitivity = 0.8864266 Specificity = 0.8608696
#> Initial Model Bootstrapped Accuracy = 0.8550328 Sensitivity = 0.8980306 Specificity = 0.812035
#> Current Model Bootstrapped Accuracy = 0.8116782 Sensitivity = 0.853577 Specificity = 0.7697794
#> Current KNN Accuracy = 0.7381242 Sensitivity = 0.8781163 Specificity = 0.6282609
#> Initial KNN Accuracy = 0.7978076 Sensitivity = 0.8891967 Specificity = 0.726087
#> Train Correlation: 0.7566178 Blind Correlation : 0.8272727
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 316 17
#> TRUE 143 345
#> Loop : 42 Input Cases = 88 Input Control = 112
#> Loop : 42 Train Cases = 79 Train Control = 100
#> Loop : 42 Blind Cases = 9 Blind Control = 12
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8921047 : Labels ~ 1 + V5005 + V723 + V8156 + V5321 + V2358 + V3675 + V3161 + V5514 + V2264
#> 1 : 3364 : 0.8075211 : Labels ~ 1 + V5400 + V1831 + V4960 + V7831 + V7426
#> 2 : 3359 : 0.8282527 : Labels ~ 1 + V3365 + V9275 + V7928 + V4352 + V630 + V1946
#> 3 : 3353 : 0.8506508 : Labels ~ 1 + V8368 + V9818 + V6688 + V1476 + V9180 + V4564 + V4178
#> 4 : 3346 : 0.7929696 : Labels ~ 1 + V312 + V7513 + V8055 + V3170 + V7748
#> 5 : 3341 : 0.804415 : Labels ~ 1 + V414 + V9965 + V1975 + V7849 + V9213
#> 6 : 3336 : 0.8152415 : Labels ~ 1 + V2309 + V3014 + V5995 + V3725 + V7584
#> 7 : 3331 : 0.7919139 : Labels ~ 1 + V9027 + V9215 + V3189
#> 8 : 3328 : 0.8460339 : Labels ~ 1 + V2644 + V4183 + V819 + V4396 + V1943 + V2862 + V7402
#> 9 : 3321 : 0.8462043 : Labels ~ 1 + V7994 + V5680 + V22 + V6146 + V2438 + V9234 + V9834
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V723 + V8156 + V5321 + V2358 + V3675 + V3161 + V5514 + V2264
#> At Accuracy: Labels ~ 1 + V5005 + V723 + V8156 + V5321 + V2358 + V3675 + V3161 + V5514 + V2264
#> B:SWiMS : Labels ~ 1 + V5005 + V723 + V8156 + V5321 + V2358 + V3675 + V3161 + V5514 + V2264
#>
#> Num. Models: 320 To Test: 1227 TopFreq: 12.61538 Thrf: 0 Removed: 0
#> ................................*Loop : 42 Blind Cases = 9 Blind Control = 12 Total = 842 Size Cases = 370 Size Control = 472
#> Accumulated Models CV Accuracy = 0.7779097 Sensitivity = 0.7459459 Specificity = 0.8029661 Forw. Ensemble Accuracy= 0.8396675
#> Initial Model Accumulated CV Accuracy = 0.8729216 Sensitivity = 0.8891892 Specificity = 0.8601695
#> Initial Model Bootstrapped Accuracy = 0.8467936 Sensitivity = 0.8610199 Specificity = 0.8325673
#> Current Model Bootstrapped Accuracy = 0.8921047 Sensitivity = 0.9035263 Specificity = 0.8806831
#> Current KNN Accuracy = 0.7387173 Sensitivity = 0.8783784 Specificity = 0.6292373
#> Initial KNN Accuracy = 0.7969121 Sensitivity = 0.8891892 Specificity = 0.7245763
#> Train Correlation: 0.802333 Blind Correlation : 0.5571429
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 325 17
#> TRUE 148 352
#> Loop : 43 Input Cases = 88 Input Control = 112
#> Loop : 43 Train Cases = 79 Train Control = 101
#> Loop : 43 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8311351 : Labels ~ 1 + V698 + V312 + V6163 + V256 + V9395
#> 1 : 3368 : 0.8541845 : Labels ~ 1 + V5321 + V8806 + V2309 + V5680 + V3082 + V3170 + V6553
#> 2 : 3361 : 0.824569 : Labels ~ 1 + V7748 + V7062 + V5139 + V1248
#> 3 : 3357 : 0.8485237 : Labels ~ 1 + V3365 + V7857 + V9215 + V8239 + V662 + V8650 + V4352
#> 4 : 3350 : 0.8272629 : Labels ~ 1 + V8368 + V86 + V6584 + V5761 + V8604 + V130
#> 5 : 3344 : 0.8314485 : Labels ~ 1 + V4960 + V4584 + V4973 + V9213 + V5454
#> 6 : 3339 : 0.8481013 : Labels ~ 1 + V8411 + V414 + V5417 + V4301 + V6121
#> 7 : 3334 : 0.806687 : Labels ~ 1 + V1476 + V6959 + V9617 + V7801
#> 8 : 3330 : 0.841596 : Labels ~ 1 + V9585 + V8055 + V7090 + V4580 + V2515
#> 9 : 3325 : 0.8738287 : Labels ~ 1 + V5005 + V3014 + V533 + V1144 + V9319 + V4290 + V4554 + V342 + V9097
#>
#> Num. Models: 10 To Test: 57 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V698 + V312 + V6163 + V256 + V9395
#> At Accuracy: Labels ~ 1 + V698 + V312 + V6163 + V256 + V9395
#> B:SWiMS : Labels ~ 1 + V698 + V312 + V6163 + V256 + V9395
#>
#> Num. Models: 320 To Test: 1335 TopFreq: 9.972222 Thrf: 0 Removed: 0
#> ................................*Loop : 43 Blind Cases = 9 Blind Control = 11 Total = 862 Size Cases = 379 Size Control = 483
#> Accumulated Models CV Accuracy = 0.774942 Sensitivity = 0.7414248 Specificity = 0.8012422 Forw. Ensemble Accuracy= 0.837587
#> Initial Model Accumulated CV Accuracy = 0.8723898 Sensitivity = 0.8891821 Specificity = 0.8592133
#> Initial Model Bootstrapped Accuracy = 0.8634986 Sensitivity = 0.8796276 Specificity = 0.8473696
#> Current Model Bootstrapped Accuracy = 0.8311351 Sensitivity = 0.8575135 Specificity = 0.8047568
#> Current KNN Accuracy = 0.737819 Sensitivity = 0.878628 Specificity = 0.6273292
#> Initial KNN Accuracy = 0.7958237 Sensitivity = 0.8891821 Specificity = 0.7225673
#> Train Correlation: 0.763205 Blind Correlation : 0.7157895
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 332 17
#> TRUE 153 360
#> Loop : 44 Input Cases = 88 Input Control = 112
#> Loop : 44 Train Cases = 79 Train Control = 101
#> Loop : 44 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8230108 : Labels ~ 1 + V3365 + V4352 + V9818 + V4584 + V533 + V1344
#> 1 : 3367 : 0.7899568 : Labels ~ 1 + V5005 + V9215 + V8368 + V762
#> 2 : 3363 : 0.825889 : Labels ~ 1 + V312 + V6163 + V6584 + V22 + V5072
#> 3 : 3358 : 0.842494 : Labels ~ 1 + V4960 + V9275 + V729 + V4290 + V6948 + V6068
#> 4 : 3352 : 0.8801608 : Labels ~ 1 + V414 + V8156 + V4396 + V6083 + V3518 + V9965 + V7891 + V2700
#> 5 : 3344 : 0.8330118 : Labels ~ 1 + V2309 + V3170 + V3222 + V5995 + V8193
#> 6 : 3339 : 0.8099237 : Labels ~ 1 + V9617 + V3014 + V1476 + V6688
#> 7 : 3335 : 0.8465293 : Labels ~ 1 + V376 + V5774 + V1831 + V7831 + V926 + V2783 + V2294
#> 8 : 3328 : 0.842709 : Labels ~ 1 + V9027 + V7319 + V2804 + V3522 + V1087 + V698
#> 9 : 3322 : 0.8173575 : Labels ~ 1 + V7994 + V4326 + V7748 + V2515
#>
#> Num. Models: 10 To Test: 55 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V3365 + V4352 + V9818 + V4584 + V533 + V1344
#> At Accuracy: Labels ~ 1 + V3365 + V4352 + V9818 + V4584 + V533 + V1344
#> B:SWiMS : Labels ~ 1 + V3365 + V4352 + V9818 + V4584 + V533 + V1344
#>
#> Num. Models: 320 To Test: 1324 TopFreq: 11.71795 Thrf: 0 Removed: 0
#> ................................*Loop : 44 Blind Cases = 9 Blind Control = 11 Total = 882 Size Cases = 388 Size Control = 494
#> Accumulated Models CV Accuracy = 0.7709751 Sensitivity = 0.7371134 Specificity = 0.7975709 Forw. Ensemble Accuracy= 0.8367347
#> Initial Model Accumulated CV Accuracy = 0.8707483 Sensitivity = 0.8840206 Specificity = 0.8603239
#> Initial Model Bootstrapped Accuracy = 0.8506269 Sensitivity = 0.8774319 Specificity = 0.8238219
#> Current Model Bootstrapped Accuracy = 0.8230108 Sensitivity = 0.851828 Specificity = 0.7941935
#> Current KNN Accuracy = 0.7335601 Sensitivity = 0.871134 Specificity = 0.6255061
#> Initial KNN Accuracy = 0.792517 Sensitivity = 0.8814433 Specificity = 0.7226721
#> Train Correlation: 0.8210151 Blind Correlation : 0.6992481
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 340 19
#> TRUE 156 367
#> Loop : 45 Input Cases = 88 Input Control = 112
#> Loop : 45 Train Cases = 79 Train Control = 101
#> Loop : 45 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8227903 : Labels ~ 1 + V723 + V5005 + V1184 + V898 + V8806 + V3675
#> 1 : 3367 : 0.8633981 : Labels ~ 1 + V1936 + V6163 + V9275 + V7748 + V4194 + V9395 + V4564 + V6685 + V2397
#> 2 : 3358 : 0.8194354 : Labels ~ 1 + V7857 + V729 + V7891 + V6584 + V1248 + V4301
#> 3 : 3352 : 0.8739149 : Labels ~ 1 + V2556 + V86 + V1831 + V4584 + V7090 + V8726 + V5321 + V3161
#> 4 : 3344 : 0.7899851 : Labels ~ 1 + V1975 + V9215 + V8681 + V414 + V1046
#> 5 : 3339 : 0.7936763 : Labels ~ 1 + V4183 + V4326 + V1344 + V7272
#> 6 : 3335 : 0.7955334 : Labels ~ 1 + V7220 + V5761 + V3206 + V9070 + V3752
#> 7 : 3330 : 0.820382 : Labels ~ 1 + V698 + V8055 + V1055 + V3554 + V5466
#> 8 : 3325 : 0.8269105 : Labels ~ 1 + V4290 + V3170 + V5683 + V1946 + V4352 + V3365
#> 9 : 3319 : 0.7862842 : Labels ~ 1 + V4070 + V5400 + V9818 + V4077 + V7629
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V723 + V5005 + V1184 + V898 + V8806 + V3675
#> At Accuracy: Labels ~ 1 + V723 + V5005 + V1184 + V898 + V8806 + V3675
#> B:SWiMS : Labels ~ 1 + V723 + V5005 + V1184 + V898 + V8806 + V3675
#>
#> Num. Models: 320 To Test: 1292 TopFreq: 13.72727 Thrf: 0 Removed: 0
#> ................................*Loop : 45 Blind Cases = 9 Blind Control = 11 Total = 902 Size Cases = 397 Size Control = 505
#> Accumulated Models CV Accuracy = 0.7738359 Sensitivity = 0.7380353 Specificity = 0.8019802 Forw. Ensemble Accuracy= 0.8392461
#> Initial Model Accumulated CV Accuracy = 0.8691796 Sensitivity = 0.884131 Specificity = 0.8574257
#> Initial Model Bootstrapped Accuracy = 0.855283 Sensitivity = 0.8790618 Specificity = 0.8315042
#> Current Model Bootstrapped Accuracy = 0.8227903 Sensitivity = 0.8687175 Specificity = 0.7768631
#> Current KNN Accuracy = 0.7372506 Sensitivity = 0.8740554 Specificity = 0.629703
#> Initial KNN Accuracy = 0.7949002 Sensitivity = 0.884131 Specificity = 0.7247525
#> Train Correlation: 0.8375176 Blind Correlation : 0.9007519
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 349 19
#> TRUE 160 374
#> Loop : 46 Input Cases = 88 Input Control = 112
#> Loop : 46 Train Cases = 79 Train Control = 101
#> Loop : 46 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8510844 : Labels ~ 1 + V2556 + V5005 + V1831 + V1975 + V5489 + V8983 + V7748 + V2358
#> 1 : 3365 : 0.8414792 : Labels ~ 1 + V7891 + V6584 + V698 + V7899 + V8038 + V729
#> 2 : 3359 : 0.80054 : Labels ~ 1 + V1476 + V1936 + V4301 + V4326 + V6958
#> 3 : 3354 : 0.8198569 : Labels ~ 1 + V4290 + V9275 + V8368 + V2064 + V4584
#> 4 : 3349 : 0.8178244 : Labels ~ 1 + V436 + V5761 + V7272 + V9704
#> 5 : 3345 : 0.8337297 : Labels ~ 1 + V8502 + V256 + V3365 + V6163 + V584 + V3170
#> 6 : 3339 : 0.8312164 : Labels ~ 1 + V86 + V9215 + V312 + V762 + V4077 + V1344 + V4738
#> 7 : 3332 : 0.8263284 : Labels ~ 1 + V7857 + V9818 + V4194 + V3161 + V3708
#> 8 : 3327 : 0.8457711 : Labels ~ 1 + V5473 + V2309 + V5680 + V4406 + V2117 + V22
#> 9 : 3321 : 0.8028794 : Labels ~ 1 + V7197 + V4960 + V3014 + V7500 + V1046 + V4900
#>
#> Num. Models: 10 To Test: 58 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V2556 + V5005 + V1831 + V1975 + V5489 + V8983 + V7748 + V2358
#> At Accuracy: Labels ~ 1 + V2556 + V5005 + V1831 + V1975 + V5489 + V8983 + V7748 + V2358
#> B:SWiMS : Labels ~ 1 + V2556 + V5005 + V1831 + V1975 + V5489 + V8983 + V7748 + V2358
#>
#> Num. Models: 320 To Test: 1252 TopFreq: 14.97619 Thrf: 0 Removed: 0
#> ................................*Loop : 46 Blind Cases = 9 Blind Control = 11 Total = 922 Size Cases = 406 Size Control = 516
#> Accumulated Models CV Accuracy = 0.7754881 Sensitivity = 0.7389163 Specificity = 0.8042636 Forw. Ensemble Accuracy= 0.840564
#> Initial Model Accumulated CV Accuracy = 0.8698482 Sensitivity = 0.8842365 Specificity = 0.8585271
#> Initial Model Bootstrapped Accuracy = 0.8453183 Sensitivity = 0.8716055 Specificity = 0.8190311
#> Current Model Bootstrapped Accuracy = 0.8510844 Sensitivity = 0.8606399 Specificity = 0.8415289
#> Current KNN Accuracy = 0.7375271 Sensitivity = 0.8743842 Specificity = 0.629845
#> Initial KNN Accuracy = 0.7939262 Sensitivity = 0.8817734 Specificity = 0.7248062
#> Train Correlation: 0.8330874 Blind Correlation : 0.8992481
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 356 20
#> TRUE 165 381
#> Loop : 47 Input Cases = 88 Input Control = 112
#> Loop : 47 Train Cases = 79 Train Control = 101
#> Loop : 47 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8255106 : Labels ~ 1 + V1936 + V698 + V3170 + V729 + V3365 + V6285
#> 1 : 3367 : 0.8158582 : Labels ~ 1 + V5005 + V9818 + V7032 + V1184 + V3947
#> 2 : 3362 : 0.816836 : Labels ~ 1 + V4564 + V4290 + V312 + V5761 + V9395
#> 3 : 3357 : 0.870088 : Labels ~ 1 + V7748 + V4326 + V7891 + V6688 + V2991 + V6408 + V5321 + V1748 + V9704
#> 4 : 3348 : 0.8269231 : Labels ~ 1 + V2556 + V1476 + V6584 + V4069 + V1943 + V8484
#> 5 : 3342 : 0.794214 : Labels ~ 1 + V8502 + V8368 + V9924 + V9031
#> 6 : 3338 : 0.8144695 : Labels ~ 1 + V436 + V256 + V7272 + V4584 + V3545 + V5359
#> 7 : 3332 : 0.8055556 : Labels ~ 1 + V4070 + V5680 + V4960 + V762 + V5107
#> 8 : 3327 : 0.8368741 : Labels ~ 1 + V5473 + V9215 + V8411 + V2309 + V8479 + V2397 + V8598 + V4352
#> 9 : 3319 : 0.8041685 : Labels ~ 1 + V414 + V8055 + V4973 + V1344 + V3708
#>
#> Num. Models: 10 To Test: 59 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V1936 + V698 + V3170 + V729 + V3365 + V6285
#> At Accuracy: Labels ~ 1 + V1936 + V698 + V3170 + V729 + V3365 + V6285
#> B:SWiMS : Labels ~ 1 + V1936 + V698 + V3170 + V729 + V3365 + V6285
#>
#> Num. Models: 320 To Test: 1250 TopFreq: 13.9697 Thrf: 0 Removed: 0
#> ................................*Loop : 47 Blind Cases = 9 Blind Control = 11 Total = 942 Size Cases = 415 Size Control = 527
#> Accumulated Models CV Accuracy = 0.7802548 Sensitivity = 0.7445783 Specificity = 0.8083491 Forw. Ensemble Accuracy= 0.8428875
#> Initial Model Accumulated CV Accuracy = 0.8715499 Sensitivity = 0.886747 Specificity = 0.8595825
#> Initial Model Bootstrapped Accuracy = 0.848855 Sensitivity = 0.8708833 Specificity = 0.8268266
#> Current Model Bootstrapped Accuracy = 0.8255106 Sensitivity = 0.8376793 Specificity = 0.813342
#> Current KNN Accuracy = 0.7377919 Sensitivity = 0.8771084 Specificity = 0.6280835
#> Initial KNN Accuracy = 0.7951168 Sensitivity = 0.8843373 Specificity = 0.7248577
#> Train Correlation: 0.8785909 Blind Correlation : 0.8857143
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 362 20
#> TRUE 170 390
#> Loop : 48 Input Cases = 88 Input Control = 112
#> Loop : 48 Train Cases = 79 Train Control = 101
#> Loop : 48 Blind Cases = 9 Blind Control = 11
#> K : 13 KNN T Cases = 79 KNN T Control = 79
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.832788 : Labels ~ 1 + V4290 + V5005 + V9213 + V4326 + V3082 + V5487 + V5107
#> 1 : 3366 : 0.8583778 : Labels ~ 1 + V698 + V312 + V898 + V4295 + V3170 + V4352 + V7605
#> 2 : 3359 : 0.8497521 : Labels ~ 1 + V7748 + V9818 + V3365 + V6163 + V6181 + V5672 + V2862
#> 3 : 3352 : 0.8223997 : Labels ~ 1 + V1476 + V8368 + V8055 + V1943
#> 4 : 3348 : 0.8105161 : Labels ~ 1 + V4960 + V4301 + V5680 + V8089 + V9395
#> 5 : 3343 : 0.8116209 : Labels ~ 1 + V8502 + V2309 + V4584 + V6688
#> 6 : 3339 : 0.8134215 : Labels ~ 1 + V436 + V9617 + V9215 + V22 + V4406 + V1344
#> 7 : 3333 : 0.8071197 : Labels ~ 1 + V86 + V414 + V8156 + V729 + V2515
#> 8 : 3328 : 0.7920932 : Labels ~ 1 + V7891 + V1831 + V3876 + V8650
#> 9 : 3324 : 0.7891933 : Labels ~ 1 + V9027 + V1046 + V9275 + V1975 + V1248
#>
#> Num. Models: 10 To Test: 54 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V4290 + V5005 + V9213 + V4326 + V3082 + V5487 + V5107
#> At Accuracy: Labels ~ 1 + V4290 + V5005 + V9213 + V4326 + V3082 + V5487 + V5107
#> B:SWiMS : Labels ~ 1 + V4290 + V5005 + V9213 + V4326 + V3082 + V5487 + V5107
#>
#> Num. Models: 320 To Test: 1321 TopFreq: 11.36667 Thrf: 0 Removed: 0
#> ................................*Loop : 48 Blind Cases = 9 Blind Control = 11 Total = 962 Size Cases = 424 Size Control = 538
#> Accumulated Models CV Accuracy = 0.7806653 Sensitivity = 0.7476415 Specificity = 0.8066914 Forw. Ensemble Accuracy= 0.8440748
#> Initial Model Accumulated CV Accuracy = 0.8700624 Sensitivity = 0.8867925 Specificity = 0.8568773
#> Initial Model Bootstrapped Accuracy = 0.8555145 Sensitivity = 0.8858011 Specificity = 0.825228
#> Current Model Bootstrapped Accuracy = 0.832788 Sensitivity = 0.8475131 Specificity = 0.8180628
#> Current KNN Accuracy = 0.7359667 Sensitivity = 0.879717 Specificity = 0.6226766
#> Initial KNN Accuracy = 0.7941788 Sensitivity = 0.8867925 Specificity = 0.7211896
#> Train Correlation: 0.9111228 Blind Correlation : 0.7894737
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 365 21
#> TRUE 176 400
#> Loop : 49 Input Cases = 88 Input Control = 112
#> Loop : 49 Train Cases = 80 Train Control = 101
#> Loop : 49 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8881234 : Labels ~ 1 + V5005 + V4352 + V7272 + V7402 + V9215 + V6163 + V7513 + V4751 + V3947 + V4134
#> 1 : 3363 : 0.8856117 : Labels ~ 1 + V698 + V3365 + V6584 + V4584 + V3161 + V649 + V3522 + V7334
#> 2 : 3355 : 0.869613 : Labels ~ 1 + V312 + V4326 + V7748 + V6688 + V3592 + V1944 + V1882
#> 3 : 3348 : 0.8573301 : Labels ~ 1 + V4290 + V4960 + V5761 + V1248 + V9268 + V9648
#> 4 : 3342 : 0.8696739 : Labels ~ 1 + V8368 + V1476 + V9818 + V723 + V2266 + V7891 + V7141
#> 5 : 3335 : 0.7853231 : Labels ~ 1 + V2309 + V8156 + V5774 + V1865
#> 6 : 3331 : 0.8555992 : Labels ~ 1 + V414 + V5417 + V86 + V7584 + V6146 + V1184 + V3265
#> 7 : 3324 : 0.8216463 : Labels ~ 1 + V9617 + V436 + V9275 + V1943 + V3778
#> 8 : 3319 : 0.8569719 : Labels ~ 1 + V7857 + V9027 + V1831 + V7196 + V4185 + V7131 + V6685
#> 9 : 3312 : 0.789738 : Labels ~ 1 + V8806 + V7994 + V3014 + V6780
#>
#> Num. Models: 10 To Test: 65 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V4352 + V7272 + V7402 + V9215 + V6163 + V7513 + V4751 + V6573 + V3947 + V4134
#> At Accuracy: Labels ~ 1 + V5005 + V4352 + V7272 + V7402 + V9215 + V6163 + V7513 + V4751 + V3947 + V4134
#> B:SWiMS : Labels ~ 1 + V5005 + V4352 + V7272 + V7402 + V9215 + V6163 + V7513 + V4751 + V3947 + V4134
#>
#> Num. Models: 320 To Test: 1398 TopFreq: 13.97368 Thrf: 0 Removed: 0
#> ................................*Loop : 49 Blind Cases = 8 Blind Control = 11 Total = 981 Size Cases = 432 Size Control = 549
#> Accumulated Models CV Accuracy = 0.7798165 Sensitivity = 0.7476852 Specificity = 0.8051002 Forw. Ensemble Accuracy= 0.841998
#> Initial Model Accumulated CV Accuracy = 0.8685015 Sensitivity = 0.8865741 Specificity = 0.8542805
#> Initial Model Bootstrapped Accuracy = 0.849989 Sensitivity = 0.8682188 Specificity = 0.8317593
#> Current Model Bootstrapped Accuracy = 0.8881234 Sensitivity = 0.8930446 Specificity = 0.8832021
#> Current KNN Accuracy = 0.7329256 Sensitivity = 0.8773148 Specificity = 0.6193078
#> Initial KNN Accuracy = 0.7900102 Sensitivity = 0.8842593 Specificity = 0.715847
#> Train Correlation: 0.80409 Blind Correlation : 0.8052632
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 372 21
#> TRUE 179 409
#> Loop : 50 Input Cases = 88 Input Control = 112
#> Loop : 50 Train Cases = 80 Train Control = 101
#> Loop : 50 Blind Cases = 8 Blind Control = 11
#> K : 13 KNN T Cases = 80 KNN T Control = 80
#> 4117 : Number of variables to test: 3373
#> 0 : 3373 : 0.8149007 : Labels ~ 1 + V5005 + V4352 + V723 + V4301 + V4738
#> 1 : 3368 : 0.8064307 : Labels ~ 1 + V1936 + V1476 + V9215 + V5400 + V2358
#> 2 : 3363 : 0.8419278 : Labels ~ 1 + V7891 + V9965 + V3591 + V3170 + V1198 + V4295 + V8156
#> 3 : 3356 : 0.7813512 : Labels ~ 1 + V2556 + V9275 + V9617 + V7513
#> 4 : 3352 : 0.8252999 : Labels ~ 1 + V6594 + V9818 + V7260 + V1046 + V5283
#> 5 : 3347 : 0.8352248 : Labels ~ 1 + V698 + V6584 + V3365 + V2515 + V4584 + V4564 + V1289
#> 6 : 3340 : 0.7886701 : Labels ~ 1 + V7748 + V6163 + V312 + V7584
#> 7 : 3336 : 0.7927524 : Labels ~ 1 + V436 + V376 + V5680 + V1248
#> 8 : 3332 : 0.7716457 : Labels ~ 1 + V86 + V5761 + V2866
#> 9 : 3329 : 0.8407359 : Labels ~ 1 + V7857 + V8368 + V1831 + V7212 + V4011 + V5808 + V5587 + V4580
#>
#> Num. Models: 10 To Test: 52 TopFreq: 1 Thrf: 0 Removed: 0
#> .*Update : Labels ~ 1 + V5005 + V4352 + V723 + V4301 + V4738
#> At Accuracy: Labels ~ 1 + V5005 + V4352 + V723 + V4301 + V4738
#> B:SWiMS : Labels ~ 1 + V5005 + V4352 + V723 + V4301 + V4738
#>
#> Num. Models: 320 To Test: 1373 TopFreq: 15.4375 Thrf: 0 Removed: 0
#> ................................*Loop : 50 Blind Cases = 8 Blind Control = 11 Total = 1000 Size Cases = 440 Size Control = 560
#> Accumulated Models CV Accuracy = 0.783 Sensitivity = 0.7522727 Specificity = 0.8071429 Forw. Ensemble Accuracy= 0.844
#> Initial Model Accumulated CV Accuracy = 0.868 Sensitivity = 0.8863636 Specificity = 0.8535714
#> Initial Model Bootstrapped Accuracy = 0.8551007 Sensitivity = 0.8756768 Specificity = 0.8345246
#> Current Model Bootstrapped Accuracy = 0.8149007 Sensitivity = 0.8419426 Specificity = 0.7878587
#> Current KNN Accuracy = 0.734 Sensitivity = 0.875 Specificity = 0.6232143
#> Initial KNN Accuracy = 0.79 Sensitivity = 0.8818182 Specificity = 0.7178571
#> Train Correlation: 0.7807583 Blind Correlation : 0.9438596
#> KNN to Model Confusion Matrix:
#>
#> FALSE TRUE
#> FALSE 381 23
#> TRUE 180 416
#>
#> Num. Models: 320 To Test: 1381 TopFreq: 10.56364 Thrf: 0 Removed: 0
#> ................................*:.....#...:.....#...:.....#...:.....#...:.....#...:.....#...:.....#...
#> Num. Models: 62 To Test: 66 TopFreq: 60 Thrf: 0 Removed: 0
#> ......*
save(arceneCV10,file="ArceneCV_01_10_3_wsvn.RDATA")
arceneCV10$cvObject$Models.testPrediction$usrFitFunction_Sel <- arceneCV10$cvObject$Models.testPrediction$usrFitFunction_Sel -0.5
arceneCV10$cvObject$Models.testPrediction$usrFitFunction <- arceneCV10$cvObject$Models.testPrediction$usrFitFunction -0.5
pm <- plotModels.ROC(arceneCV10$cvObject$LASSO.testPredictions,theCVfolds=10,main="CV LASSO",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV10$cvObject$Models.testPrediction,theCVfolds=10,predictor="Prediction",main="BB:SWiMS",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV10$cvObject$Models.testPrediction,theCVfolds=10,predictor="Ensemble.Forward",main="Forward Median",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV10$cvObject$Models.testPrediction,theCVfolds=10,predictor="Forward.Selection.Bagged",main="Forward Bagged",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV10$cvObject$Models.testPrediction,theCVfolds=10,predictor="usrFitFunction",main="SVM",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
pm <- plotModels.ROC(arceneCV10$cvObject$Models.testPrediction,theCVfolds=10,predictor="usrFitFunction_Sel",main="SVM",cex=0.90)
ci <- epi.tests(pm$predictionTable)
CVACCTable <- rbind(CVACCTable,ci$elements$diag.acc)
CVBETable <- rbind(CVBETable,1-0.5*(ci$elements$sensitivity+ci$elements$specificity))
bp <- barPlotCiError(as.matrix(CVACCTable),metricname="Accuracy",thesets=CVthesets,themethod=c("CV:3","CV:10"),main="Accuracy",args.legend = list(x = "bottomright"))
bp <- barPlotCiError(as.matrix(CVBETable),metricname="Balanced Error",thesets=CVthesets,themethod=c("CV:3","CV:10"),main="Balanced Error",args.legend = list(x = "topright"))