Filtered Fit: IDeA and the UPSTM

Filtered ML fit and the UPSTM with FRESA.CAD

Here we make use of the FRESA.CAD::filteredfit() function to train ML models with and without UPSTM on the ARCENE data set.

Isabelle Guyon, Steve R. Gunn, Asa Ben-Hur, Gideon Dror, 2004. Result analysis of the NIPS 2003 feature selection challenge. In: NIPS. \[Web Link\]. from: https://archive.ics.uci.edu/ml/datasets/Arcene

a. Original owners
The data were obtained from two sources: The National Cancer Institute (NCI) and the Eastern Virginia Medical School (EVMS). All the data consist of mass-spectra obtained with the SELDI technique. The samples include patients with cancer (ovarian or prostate cancer), and healthy or control patients.

b. Donor of database
This version of the database was prepared for the NIPS 2003 variable and feature selection benchmark by Isabelle Guyon, 955 Creston Road, Berkeley, CA 94708, USA (isabelle ‘@’ clopinet.com).

This scrip uses FRESA.CAD and mlbench R packages:

knitr::opts_chunk$set(collapse = TRUE, warning = FALSE, message = FALSE,comment = "#>")

library("FRESA.CAD")

## Loading required package: Rcpp

## Loading required package: stringr

## Loading required package: miscTools

## Loading required package: Hmisc

## Loading required package: lattice

## Loading required package: survival

## Loading required package: Formula

## Loading required package: ggplot2

## 
## Attaching package: 'Hmisc'

## The following objects are masked from 'package:base':
## 
##     format.pval, units

## Loading required package: pROC

## Type 'citation("pROC")' for a citation.

## 
## Attaching package: 'pROC'

## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var

library(mlbench)

op <- par(no.readonly = TRUE)

Loading the ARCENE data set

trainLabeled <- read.delim("../Data/trainSet.txt")
validLabeled <- read.delim("../Data/arcene_valid.txt")
wholeArceneSet <- rbind(trainLabeled,validLabeled)


wholeArceneSet$Labels <-  1*(wholeArceneSet$Labels > 0)
wholeArceneSet[,1:ncol(trainLabeled)] <- sapply(wholeArceneSet,as.double)

table(wholeArceneSet$Labels)
#> 
#>   0   1 
#> 112  88

Setting some variables for downstream analysis

studyName = "Arcene"
datasetframe <- wholeArceneSet
Outcome <- "Labels"

trainFraction = 0.5

Setting the Training and Testing sets

tb <- table(datasetframe[,Outcome])
classNames <- unique(datasetframe[,Outcome])

allrowClass <- datasetframe[,Outcome]
names(allrowClass) <- rownames(datasetframe)

trainsize <- trainFraction*min(tb);
trainSamples <- NULL;
for (theClass in classNames)
{
  classSample <- allrowClass[allrowClass == theClass]
  trainSamples <- c(trainSamples,names(classSample[sample(length(classSample),trainsize)]))
}


datasetframe_train <- datasetframe[trainSamples,]
testSamples <- !(rownames(datasetframe) %in% trainSamples)
datasetframe_test <- datasetframe[testSamples,]

outcomes <- datasetframe_train[,Outcome]

pander::pander(table(datasetframe[,Outcome]),caption="All")

All

0	1
112	88

pander::pander(table(datasetframe_train[,Outcome]),caption="Training")

Training

0	1
44	44

pander::pander(table(datasetframe_test[,Outcome]),caption="Testing")

Testing

0	1
68	44

Machine Learning with the filterfit() function

Train a simple Logistic model with LASSO

system.time(mLASSORaw <- filteredFit(paste(Outcome,"~."),
                   datasetframe_train,
                   fitmethod=LASSO_MIN,
                     filtermethod=univariate_Wilcoxon,
                     filtermethod.control=list(pvalue=0.05,limit= -1),
                    family = "binomial"
                   ))

user system elapsed 10.80 1.11 11.91

Now we run filterfit() with decorrelation

system.time(mLASSODecor <- filteredFit(paste(Outcome,"~."),
                   datasetframe_train,
                    fitmethod=LASSO_MIN,
                     filtermethod=univariate_Wilcoxon,
                     filtermethod.control=list(pvalue=0.05,limit= -1),
                     DECOR = TRUE,
                     family = "binomial"
                   ))

user system elapsed 611.18 34.08 207.53

Predicting the Testinig Set

par(mfrow=c(1,2))


psRaw <- predictionStats_binary(cbind(datasetframe_test$Labels,
                                      predict(mLASSORaw,datasetframe_test)),
                                "LASSO Raw",cex=0.75)

LASSO Raw

pander::pander(psRaw$aucs)

est	lower	upper
0.76	0.6697	0.8504

psDecor <- predictionStats_binary(cbind(datasetframe_test$Labels,
                                        predict(mLASSODecor,datasetframe_test)),
                                "LASSO after UPSTM",cex=0.75)

LASSO after UPSTM

pander::pander(psDecor$aucs)

est	lower	upper
0.8031	0.7153	0.891