1 FRESA.CAD Benchmark

1.1 Baseline Tadpole Challenge

library(readr)

TadpoleBL <- NULL
load("TadpoleBL.RDATA")
tadnames <- as.character(t(read_csv("tadnames.csv")[,2]))
TadpoleBL <- TadpoleBL[,tadnames]
TadpoleBL <- TadpoleBL[complete.cases(TadpoleBL),]


sampleTrain <- sample(nrow(TadpoleBL),nrow(TadpoleBL)*0.10)
sampleTadpole <- TadpoleBL[sampleTrain,]

ADAS_Tadpole_FRESA <- FRESA.Model(formula = ADAS13 ~ 1,data = sampleTadpole,repeats = 20)

1.2 Benchmark


cp <- CVRegBenchmark(theData = TadpoleBL, theOutcome = "ADAS13", reps = 50, fraction = 0.15, topincluded = 100 )


elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Results

#The Times
pander::pander(cputimes)
  • Fresa: 26.01
  • LASSO: 0.6572
  • RF: 6.265
  • SVM: 0.2232
pander::pander(featsize)
  • Fresa: 21.84
  • LASSO: 29.92
  • Univ: 312.5

plotMAEEvolution(cp,50,main="Mean Absolute Error (MAE)", location="topright")



bp <- barPlotCiError(as.matrix(cp$CorTable),metricname = "Pearson Correlation",thesets = thesets,themethod = theMethod,main = "Pearson Correlation",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))


pander::pander(bp$barMatrix,caption = "Pearson Correlation",round = 3)
Pearson Correlation
  Default Regresion Method SVM Rendering Filtered
B:SWiMS 0.693 0.719
LASSO 0.706 0.711
RF 0.691 0.7
SVM 0.663 0.701 
pander::pander(bp$ciTable,caption = "Pearson Correlation with 95%CI",round = 3)
Pearson Correlation with 95%CI
  Pearson Correlation lower upper
Default Regresion Method 0.693 0.664 0.72
Default Regresion Method 0.706 0.678 0.732
Default Regresion Method 0.691 0.661 0.718
Default Regresion Method 0.663 0.632 0.693
SVM Rendering Filtered 0.719 0.692 0.744
SVM Rendering Filtered 0.711 0.684 0.737
SVM Rendering Filtered 0.7 0.671 0.726
SVM Rendering Filtered 0.701 0.673 0.728 


bp <- barPlotCiError(as.matrix(cp$RMSETable),metricname = "RMSE",thesets = thesets,themethod = theMethod,main = "RMSE",offsets = c(0.5,5),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "RMSE",round = 3)
RMSE
  Default Regresion Method SVM Rendering Filtered
B:SWiMS 6.932 6.719
LASSO 6.895 6.844
RF 7.01 6.941
SVM 7.412 6.893 
pander::pander(bp$ciTable,caption = "RMSE with 95%CI",round = 3)
RMSE with 95%CI
  RMSE lower upper
Default Regresion Method 6.932 6.678 7.207
Default Regresion Method 6.895 6.641 7.168
Default Regresion Method 7.01 6.752 7.288
Default Regresion Method 7.412 7.139 7.706
SVM Rendering Filtered 6.719 6.472 6.985
SVM Rendering Filtered 6.844 6.592 7.115
SVM Rendering Filtered 6.941 6.686 7.217
SVM Rendering Filtered 6.893 6.64 7.167 


bp <- barPlotCiError(as.matrix(cp$BiasTable),metricname = "BIAS",thesets = thesets,themethod = theMethod,main = "BIAS",offsets = c(0.5,0.5),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "BIAS",round = 3)
BIAS
  Default Regresion Method SVM Rendering Filtered
B:SWiMS 0.039 -0.476
LASSO 0.023 -0.343
RF 0.041 -0.251
SVM -0.477 -0.507 
pander::pander(bp$ciTable,caption = "BIAS with 95%CI",round = 3)
BIAS with 95%CI
  BIAS lower upper
Default Regresion Method 0.039 -0.336 0.413
Default Regresion Method 0.023 -0.35 0.395
Default Regresion Method 0.041 -0.338 0.42
Default Regresion Method -0.477 -0.877 -0.078
SVM Rendering Filtered -0.476 -0.838 -0.114
SVM Rendering Filtered -0.343 -0.712 0.026
SVM Rendering Filtered -0.251 -0.626 0.124
SVM Rendering Filtered -0.507 -0.878 -0.135

1.3 Features Analysis

pander::pander(summary(ADAS_Tadpole_FRESA$BSWiMS.model,caption="Recurency ADAS13 model",round = 3))

  • coefficients:

    Table continues below
      Estimate lower mean upper u.MSE
    ST30SMVO 0.8751 0.7031 0.8751 1.047 64.19
    APOE4 2.128 1.852 2.128 2.403 77.25
    ST29SMVO -1.259 -1.364 -1.259 -1.154 58.38
    ST40CMVO -0.9031 -1.023 -0.9031 -0.7835 66.02
    ST83CMCMP -1.648 -1.734 -1.648 -1.562 56.89
    MaxCVD 1.466 1.135 1.466 1.797 85.72
    ST99CMCMP -0.7865 -1.091 -0.7865 -0.4817 64.1
    ST12SMVO -0.1673 -0.2456 -0.1673 -0.08897 63.05
    Hippocampus -0.0002889 -0.0002958 -0.0002889 -0.0002819 59.41
    ST91CCMP -1.26 -1.672 -1.26 -0.8478 67.32
    ST24TMT -0.5079 -0.7533 -0.5079 -0.2624 57.3
    ST83CCMP -0.1274 -0.1903 -0.1274 -0.06457 60.87
    ST32TMT -0.3217 -0.4842 -0.3217 -0.1592 63.55
    ST24CMVO -0.04734 -0.06917 -0.04734 -0.02552 61.72
    Entorhinal -7.148e-05 -9.623e-05 -7.148e-05 -4.673e-05 63.13
    stdCOMPD 0.9246 0.506 0.9246 1.343 72.44
    Table continues below
      r.MSE model.MSE NeRI F.pvalue t.pvalue
    ST30SMVO 54.6 42.08 0.1618 5.905e-09 0.0002147
    APOE4 48.72 42.08 0.1298 1.199e-05 0.01014
    ST29SMVO 53.33 46.15 0.1809 1.338e-05 0.01778
    ST40CMVO 59.07 45.59 0.1437 6.394e-09 0.004838
    ST83CMCMP 54.69 47.06 0.1824 9.614e-06 0.01493
    MaxCVD 49.06 44.83 0.05802 0.0006201 0.1219
    ST99CMCMP 56.79 46.16 0.1908 1.427e-07 0.005876
    ST12SMVO 54.98 47.2 0.1832 7.066e-06 0.0102
    Hippocampus 56.48 50.65 0.1897 0.0001605 0.03474
    ST91CCMP 61.07 45.77 0.2 1.037e-09 0.0007465
    ST24TMT 58.68 51.2 0.1221 2.487e-05 0.01649
    ST83CCMP 60.85 53.18 0.1959 3.088e-05 0.006942
    ST32TMT 61.63 54.31 0.0687 4.969e-05 0.05799
    ST24CMVO 65.06 56.06 0.2519 1.052e-05 0.000875
    Entorhinal 77.64 58.62 0.2595 3.658e-09 5.535e-05
    stdCOMPD 73.6 61.58 0.1985 4.194e-06 0.002539
      Sign.pvalue Wilcox.pvalue
    ST30SMVO 0.0304 0.0008135
    APOE4 0.07908 0.01249
    ST29SMVO 0.0229 0.009357
    ST40CMVO 0.05674 0.006351
    ST83CMCMP 0.02135 0.01398
    MaxCVD 0.2772 0.1218
    ST99CMCMP 0.01705 0.006505
    ST12SMVO 0.01443 0.007304
    Hippocampus 0.01638 0.01103
    ST91CCMP 0.0127 0.002119
    ST24TMT 0.08269 0.03697
    ST83CCMP 0.01422 0.007642
    ST32TMT 0.1952 0.06407
    ST24CMVO 0.002491 0.0009708
    Entorhinal 0.001191 0.0001453
    stdCOMPD 0.01306 0.001695
  • MSE: 38.6
  • R2: 0.5824

  • bootstrap:

gain <- length(ADAS_Tadpole_FRESA$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*ADAS_Tadpole_FRESA$BSWiMS.models$bagging$formulaNetwork,trace="none",mar=c(10,10),main="B:SWiMS Formula Network")