Feature Decorrelation
José Tamez-Peña
March 3, 2020
1 Filters on Raw and Decorrelated data
Here I’ll show the impact of decorrelating high-dimensional data sets.
library("FRESA.CAD")1.1 The ARCENE Data Set
trainLabeled <- read.delim("./trainSet.txt")
validLabeled <- read.delim("./arcene_valid.txt")
trainLabeled$Labels <- 1*(trainLabeled$Labels > 0)
validLabeled$Labels <- 1*(validLabeled$Labels > 0)
trainLabeled[,1:ncol(trainLabeled)] <- sapply(trainLabeled,as.numeric)
validLabeled[,1:ncol(validLabeled)] <- sapply(validLabeled,as.numeric)
table(trainLabeled$Labels)
table(validLabeled$Labels)
1.2 Arcene Train/Test
trainingSet <- trainLabeled
testingSet <- validLabeled
vartoAdjust <- colnames(trainingSet)[!(colnames(trainingSet) %in% c("Labels"))]
noncorrelated <- correlated_Remove(data= trainingSet,fnames= vartoAdjust)
trainingSet <- trainingSet[,c("Labels",noncorrelated)]
testingSet <- testingSet[,c("Labels",noncorrelated)]
rownames(testingSet) <- paste("T",rownames(testingSet),sep="_")
ArceneSet <- rbind(trainingSet,testingSet)
trainIDS <- rownames(trainingSet)
testIDS <- rownames(testingSet)
trainingSetc <- trainingSet
trainingSetc$Labels <- as.factor(trainingSetc$Labels)
TrainDecorrelated <- featureDecorrelation(trainingSet,Outcome="Labels",refdata=trainingSet)
#> 5761 :66 :( 500 ){4906 :}[]( 1328 ){3392 :}[]( 1334 ){2048 :}[]( 957 ){880 :}[]( 469 ){183 :}[]( 299 ){57 :}[]( 238 ){24 :}[]( 212 ){16 :}[]( 200 ){3 :}[]( 198 ){1 :}[]( 197 ){0 :}[]
decorrmatrix <- attr(TrainDecorrelated,"DeCorrmatrix")
attr(TrainDecorrelated,"DeCorrmatrix") <- NULL
dataTestDecorrelated <- testingSet
dataTestDecorrelated[,colnames(decorrmatrix)] <- as.matrix(testingSet[,colnames(decorrmatrix)]) %*% decorrmatrix
ufeat <- attr(TrainDecorrelated, "baseFeatures")
dataTrainDecorrelatedc <- TrainDecorrelated
dataTrainDecorrelatedc$Labels <- as.factor(dataTrainDecorrelatedc$Labels)1.3 Filtered SVM on Arcene
par(mfrow = c(1,2),cex = 0.5);
FilteredSVM_ml <- filteredFit(Labels~.,
trainingSetc,
fitmethod=e1071::svm,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit",
probability = TRUE,
scale=FALSE
)
bsFilteredSVM <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredSVM_ml,testingSet)),"Filtered:SVM",cex = 0.7)
FilteredSVM_ml <- filteredFit(Labels~.,
dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit",
probability = TRUE,
scale=FALSE
)
bsFilteredSVM <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredSVM_ml,dataTestDecorrelated)),"Decorrelated Filtered:SVM",cex = 0.7)
bsFilteredSVM$accc1.4 All FRESA.CAD Filters
par(mfrow = c(2,2),cex = 0.5);
FilteredEnsemble_ml <- filteredFit(Labels~.,
trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_BinEnsemble,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit",
probability = TRUE,
scale=FALSE
)
#fat <- attributes(FilteredEnsemble_ml$filter)
#sf <- names(FilteredEnsemble_ml$filter)[1:120]
#FilteredEnsemble_ml$filter[sf]
#fat$varcount[sf]
#fat$rankVar[sf]
bsFilteredEnsemble <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredEnsemble_ml,testingSet))
,"Ensemble:SVM",cex = 0.7)
FilteredKS_ml <- filteredFit(Labels~.,
trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_KS,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredKS <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredKS_ml,testingSet)),"KS:SVM",cex = 0.7)
#FilteredDTS_ml <- filteredFit(Labels~.,
# trainingSetc,
# fitmethod=e1071::svm,
# filtermethod=univariate_DTS,
# filtermethod.control=list(pvalue=0.05,limit=150),
# Scale="OrderLogit", probability = TRUE, scale=FALSE)
#bsFilteredDTS <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredDTS_ml,testingSet)),"DTS:SVM",cex = 0.7)
FilteredWilcox_ml <- filteredFit(Labels~.,
trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_Wilcoxon,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredWilcox <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredWilcox_ml,testingSet))
,"Wilcox:SVM",cex = 0.7)
Filteredttest_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_tstudent,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredttest <- predictionStats_binary(cbind(testingSet$Labels,predict(Filteredttest_ml,testingSet))
,"t-test:SVM",cex = 0.7)
Filteredkendall_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "kendall"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredkendall <- predictionStats_binary(cbind(testingSet$Labels,predict(Filteredkendall_ml,testingSet))
,"Kendall:SVM",cex = 0.7)
Filteredspearman_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "spearman"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredspearman <- predictionStats_binary(cbind(testingSet$Labels,predict(Filteredspearman_ml,testingSet))
,"Spearman:SVM",cex = 0.7)
Filteredpearson_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "pearson"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredpearson <- predictionStats_binary(cbind(testingSet$Labels,predict(Filteredpearson_ml,testingSet))
,"Pearson:SVM",cex = 0.7)
FilteredFtest_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Ftest",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredFtest <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredFtest_ml,testingSet))
,"Filter_F:SVM",cex = 0.7)
FilteredBinomial_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Binomial",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredBinomial <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredBinomial_ml,testingSet))
,"Filter_Bin:SVM",cex = 0.7)
FilteredWilcox2_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Wilcox",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredWilcox2 <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredWilcox2_ml,testingSet))
,"Filter_Wil:SVM",cex = 0.7)
FilteredtStudent_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "tStudent",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredtStudent <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredtStudent_ml,testingSet))
,"Filter_tt:SVM",cex = 0.7)
FilteredzIDI_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_Logit,
filtermethod.control=list(pvalue=0.1,limit=150,uniTest = "zIDI"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredzIDI <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredzIDI_ml,testingSet))
,"Filter_IDI:SVM",cex = 0.7)
FilteredzNRI_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=univariate_Logit,
filtermethod.control=list(pvalue=0.1,limit=150,uniTest = "zNRI"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredzNRI <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredzNRI_ml,testingSet))
,"Filter_NRI:SVM",cex = 0.7)
FilteredmRMR_ml <- filteredFit(Labels~.,trainingSetc,
fitmethod=e1071::svm,
filtermethod=mRMR.classic_FRESA,
filtermethod.control=list(feature_count=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
bsFilteredmRMR <- predictionStats_binary(cbind(testingSet$Labels,predict(FilteredmRMR_ml,testingSet))
,"Filter_mRMR:SVM",cex = 0.7)
par(mfrow = c(1,1),cex = 0.5)
1.4.1 Analysis of the Extracted Features
par(mfrow = c(1,1),cex = 0.5,pty="m")
rownamesFeat <- c("Wilcox","ttest","spearman","kendall","Pearson","Ftest","Binomial","Wilcox2","tStudent","zIDI","zNRI","mRMR","Ensemble","KS")
featursMatrix <- as.data.frame(matrix(0,ncol=14,nrow=ncol(trainingSet)))
rownames(featursMatrix) <- colnames(trainingSet)
colnames(featursMatrix) <- c(rownamesFeat);
featursMatrix[FilteredWilcox_ml$selectedfeatures,1] <- 1
featursMatrix[Filteredttest_ml$selectedfeatures,2] <- 1
featursMatrix[Filteredspearman_ml$selectedfeatures,3] <- 1
featursMatrix[Filteredkendall_ml$selectedfeatures,4] <- 1
featursMatrix[Filteredpearson_ml$selectedfeatures,5] <- 1
featursMatrix[FilteredFtest_ml$selectedfeatures,6] <- 1
featursMatrix[FilteredBinomial_ml$selectedfeatures,7] <- 1
featursMatrix[FilteredWilcox2_ml$selectedfeatures,8] <- 1
featursMatrix[FilteredtStudent_ml$selectedfeatures,9] <- 1
featursMatrix[FilteredzIDI_ml$selectedfeatures,10] <- 1
featursMatrix[FilteredzNRI_ml$selectedfeatures,11] <- 1
featursMatrix[FilteredmRMR_ml$selectedfeatures,12] <- 1
featursMatrix[FilteredEnsemble_ml$selectedfeatures,13] <- 1
featursMatrix[FilteredKS_ml$selectedfeatures,14] <- 1
sumselected <- apply(featursMatrix,1,sum)
featursMatrix <- featursMatrix[sumselected>0,]
gplots::heatmap.2(as.matrix(featursMatrix),trace = "none",mar = c(7,13),main = "Selected Features", cexRow = 0.5,cexCol = 0.75)
1.5 All FRESA.CAD Filters on the Decorrelated data set
par(mfrow = c(2,2),cex = 0.5);
FilteredEnsemble_ml <- filteredFit(Labels~.,
dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_BinEnsemble,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit",
probability = TRUE,
scale=FALSE
)
deBsFilteredEnsemble <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredEnsemble_ml,dataTestDecorrelated)),"De: Ensemble:SVM",cex = 0.7)
FilteredKS_ml <- filteredFit(Labels~.,
dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_KS,
filtermethod.control=list(pvalue=0.05,limit=1.0),
Scale="OrderLogit",
probability = TRUE,
scale=FALSE
)
deBsFilteredKS <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredKS_ml,dataTestDecorrelated)),"De: KS:SVM",cex = 0.7)
#FilteredDTS_ml <- filteredFit(Labels~.,
# dataTrainDecorrelatedc,
# fitmethod=e1071::svm,
# filtermethod=univariate_DTS,
# filtermethod.control=list(pvalue=0.05,limit=150),
# Scale="OrderLogit", probability = TRUE, scale=FALSE)
#deBsFilteredDTS <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredDTS_ml,dataTestDecorrelated)),"De:DTS:SVM",cex = 0.7)
FilteredWilcox_ml <- filteredFit(Labels~.,
dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_Wilcoxon,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredWilcox <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredWilcox_ml,dataTestDecorrelated)),"De:Wilcox:SVM",cex = 0.7)
Filteredttest_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_tstudent,
filtermethod.control=list(pvalue=0.05,limit=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredttest <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(Filteredttest_ml,dataTestDecorrelated)),"De:t-test:SVM",cex = 0.7)
Filteredkendall_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "kendall"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredkendall <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(Filteredkendall_ml,dataTestDecorrelated)),"Kendall:SVM",cex = 0.7)
Filteredspearman_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "spearman"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredspearman <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(Filteredspearman_ml,dataTestDecorrelated)),"De:Spearman:SVM",cex = 0.7)
Filteredpearson_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_correlation,
filtermethod.control=list(pvalue=0.05,limit=150,method = "pearson"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredpearson <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(Filteredpearson_ml,dataTestDecorrelated)),"De: Pearson:SVM",cex = 0.7)
FilteredFtest_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Ftest",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredFtest <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredFtest_ml,dataTestDecorrelated)),"De:Filter_F:SVM",cex = 0.7)
FilteredBinomial_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Binomial",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredBinomial <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredBinomial_ml,dataTestDecorrelated)),"De:Filter_Bin:SVM",cex = 0.7)
FilteredWilcox2_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "Wilcox",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
#FilteredWilcox2_ml$selectedfeatures
deBsFilteredWilcox2 <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredWilcox2_ml,dataTestDecorrelated)),"De:Filter_Wil:SVM",cex = 0.7)
FilteredtStudent_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_residual,
filtermethod.control=list(pvalue=0.05,limit=150,uniTest = "tStudent",type="LOGIT"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredtStudent <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredtStudent_ml,dataTestDecorrelated)),"De:Filter_tt:SVM",cex = 0.7)
FilteredzIDI_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_Logit,
filtermethod.control=list(pvalue=0.1,limit=150,uniTest = "zIDI"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredzIDI <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredzIDI_ml,dataTestDecorrelated)),"De:Filter_IDI:SVM",cex = 0.7)
FilteredzNRI_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=univariate_Logit,
filtermethod.control=list(pvalue=0.1,limit=150,uniTest = "zNRI"),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredzNRI <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredzNRI_ml,dataTestDecorrelated)),"De:Filter_NRI:SVM",cex = 0.7)
FilteredmRMR_ml <- filteredFit(Labels~.,dataTrainDecorrelatedc,
fitmethod=e1071::svm,
filtermethod=mRMR.classic_FRESA,
filtermethod.control=list(feature_count=150),
Scale="OrderLogit", probability = TRUE, scale=FALSE)
deBsFilteredmRMR <- predictionStats_binary(cbind(dataTestDecorrelated$Labels,predict(FilteredmRMR_ml,dataTestDecorrelated)),"De:Filter_mRMR:SVM",cex = 0.7)
par(mfrow = c(1,1),cex = 0.5)
1.5.1 Plotting the Filters Performance Statistics
berror <- deBsFilteredWilcox$berror
berror <- rbind(berror,deBsFilteredttest$berror)
berror <- rbind(berror,deBsFilteredspearman$berror)
berror <- rbind(berror,deBsFilteredkendall$berror)
berror <- rbind(berror,deBsFilteredpearson$berror)
berror <- rbind(berror,deBsFilteredFtest$berror)
berror <- rbind(berror,deBsFilteredBinomial$berror)
berror <- rbind(berror,deBsFilteredWilcox2$berror)
berror <- rbind(berror,deBsFilteredtStudent$berror)
berror <- rbind(berror,deBsFilteredzIDI$berror)
berror <- rbind(berror,deBsFilteredzNRI$berror)
berror <- rbind(berror,deBsFilteredmRMR$berror)
berror <- rbind(berror,deBsFilteredEnsemble$berror)
berror <- rbind(berror,deBsFilteredKS$berror)
#berror <- rbind(berror,deBsFilteredDTS$berror)
berror <- rbind(berror,bsFilteredWilcox$berror)
berror <- rbind(berror,bsFilteredttest$berror)
berror <- rbind(berror,bsFilteredspearman$berror)
berror <- rbind(berror,bsFilteredkendall$berror)
berror <- rbind(berror,bsFilteredpearson$berror)
berror <- rbind(berror,bsFilteredFtest$berror)
berror <- rbind(berror,bsFilteredBinomial$berror)
berror <- rbind(berror,bsFilteredWilcox2$berror)
berror <- rbind(berror,bsFilteredtStudent$berror)
berror <- rbind(berror,bsFilteredzIDI$berror)
berror <- rbind(berror,bsFilteredzNRI$berror)
berror <- rbind(berror,bsFilteredmRMR$berror)
berror <- rbind(berror,bsFilteredEnsemble$berror)
berror <- rbind(berror,bsFilteredKS$berror)
#berror <- rbind(berror,bsFilteredDTS$berror)
rownames(berror) <- c("De_Wilcox","De_ttest","De_spearman","De_kendall","De_Pearson","De_Ftest","De_Binomial","De_Wilcox2","De_tStudent","De_zIDI","De_zNRI","De_mRMR","De_Ensemble","De_KS","Wilcox","ttest","spearman","kendall","Pearson","Ftest","Binomial","Wilcox2","tStudent","zIDI","zNRI","mRMR","Ensemble","KS")
bpBER <- barPlotCiError(as.matrix(berror),
metricname = "Balanced Error",
thesets = "Hold-Out Error",
themethod = rownames(berror),
main = "Balanced Error",
offsets = c(0.5,1),
scoreDirection = "<",
ho=0.5,
args.legend = list(bg = "white",x="bottomright",inset=c(0.0,0),cex=0.5),
col = terrain.colors(nrow(berror))
)
aucs <- deBsFilteredWilcox$aucs
aucs <- rbind(aucs,deBsFilteredttest$aucs)
aucs <- rbind(aucs,deBsFilteredspearman$aucs)
aucs <- rbind(aucs,deBsFilteredkendall$aucs)
aucs <- rbind(aucs,deBsFilteredpearson$aucs)
aucs <- rbind(aucs,deBsFilteredFtest$aucs)
aucs <- rbind(aucs,deBsFilteredBinomial$aucs)
aucs <- rbind(aucs,deBsFilteredWilcox2$aucs)
aucs <- rbind(aucs,deBsFilteredtStudent$aucs)
aucs <- rbind(aucs,deBsFilteredzIDI$aucs)
aucs <- rbind(aucs,deBsFilteredzNRI$aucs)
aucs <- rbind(aucs,deBsFilteredmRMR$aucs)
aucs <- rbind(aucs,deBsFilteredEnsemble$aucs)
aucs <- rbind(aucs,deBsFilteredKS$aucs)
#aucs <- rbind(aucs,deBsFilteredDTS$aucs)
aucs <- rbind(aucs,bsFilteredWilcox$aucs)
aucs <- rbind(aucs,bsFilteredttest$aucs)
aucs <- rbind(aucs,bsFilteredspearman$aucs)
aucs <- rbind(aucs,bsFilteredkendall$aucs)
aucs <- rbind(aucs,bsFilteredpearson$aucs)
aucs <- rbind(aucs,bsFilteredFtest$aucs)
aucs <- rbind(aucs,bsFilteredBinomial$aucs)
aucs <- rbind(aucs,bsFilteredWilcox2$aucs)
aucs <- rbind(aucs,bsFilteredtStudent$aucs)
aucs <- rbind(aucs,bsFilteredzIDI$aucs)
aucs <- rbind(aucs,bsFilteredzNRI$aucs)
aucs <- rbind(aucs,bsFilteredmRMR$aucs)
aucs <- rbind(aucs,bsFilteredEnsemble$aucs)
aucs <- rbind(aucs,bsFilteredKS$aucs)
#aucs <- rbind(aucs,bsFilteredDTS$aucs)
rownames(aucs) <- rownames(berror)
bpAUC <- barPlotCiError(as.matrix(aucs),
metricname = "ROC AUC",
thesets = "Hold-Out AUC",
themethod = rownames(aucs),
main = "AUC",
offsets = c(-0.5,1),
scoreDirection = ">",
ho=0.5,
args.legend = list(bg = "white",x="bottomleft",inset=c(0.0,0),cex=0.5),
col = terrain.colors(nrow(aucs))
)