1 FRESA.CAD Benchmark

1.1 Modeling Body FAT


data("bodyfat", package = "TH.data")
bodyfat_mat <- as.data.frame(model.matrix(DEXfat~.*.,bodyfat))
bodyfat_mat$`(Intercept)` <- NULL
bodyfat_mat$DEXfat <- bodyfat$DEXfat
fnames <- colnames(bodyfat_mat)
fnames <- str_replace_all(fnames," ","_")
fnames <- str_replace_all(fnames,"/","_")
fnames <- str_replace_all(fnames,":",".")
colnames(bodyfat_mat) <- fnames
Body_FAT_FRESA <- FRESA.Model(formula = DEXfat ~ 1,data = bodyfat_mat,repeats = 20)

1.2 Benchmark


cp <- CVRegBenchmark(theData = bodyfat_mat, theOutcome = "DEXfat", reps = 100, fraction = 0.90, topincluded = 40 )


elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]))

featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))

1.2.1 Results

#The Times
pander::pander(cputimes)
  • Fresa: 1.854
  • LASSO: 0.211
  • RF: 0.1008
  • SVM: 0.0121
pander::pander(featsize)
  • Fresa: 24.97
  • LASSO: 8.77
  • Univ: 44.75

plotMAEEvolution(cp,30,main="Mean Absolute Error (MAE)", location="topright")



bp <- barPlotCiError(as.matrix(cp$CorTable),metricname = "Pearson Correlation",thesets = thesets,themethod = theMethod,main = "Pearson Correlation",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))


pander::pander(bp$barMatrix,caption = "Pearson Correlation",round = 3)
Pearson Correlation
  Default Regresion Method SVM Rendering Filtered
B:SWiMS 0.951 0.922
LASSO 0.948 0.929
RF 0.954 0.915
SVM 0.909 0.912 
pander::pander(bp$ciTable,caption = "Pearson Correlation with 95%CI",round = 3)
Pearson Correlation with 95%CI
  Pearson Correlation lower upper
Default Regresion Method 0.951 0.922 0.969
Default Regresion Method 0.948 0.918 0.967
Default Regresion Method 0.954 0.927 0.971
Default Regresion Method 0.909 0.857 0.942
SVM Rendering Filtered 0.922 0.878 0.951
SVM Rendering Filtered 0.929 0.889 0.956
SVM Rendering Filtered 0.915 0.867 0.947
SVM Rendering Filtered 0.912 0.861 0.944 


bp <- barPlotCiError(as.matrix(cp$RMSETable),metricname = "RMSE",thesets = thesets,themethod = theMethod,main = "RMSE",offsets = c(0.5,5),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "RMSE",round = 3)
RMSE
  Default Regresion Method SVM Rendering Filtered
B:SWiMS 3.396 4.38
LASSO 3.493 4.232
RF 3.342 4.599
SVM 4.748 4.681 
pander::pander(bp$ciTable,caption = "RMSE with 95%CI",round = 3)
RMSE with 95%CI
  RMSE lower upper
Default Regresion Method 3.396 2.917 4.063
Default Regresion Method 3.493 3.001 4.179
Default Regresion Method 3.342 2.871 3.999
Default Regresion Method 4.748 4.079 5.681
SVM Rendering Filtered 4.38 3.763 5.241
SVM Rendering Filtered 4.232 3.636 5.064
SVM Rendering Filtered 4.599 3.951 5.502
SVM Rendering Filtered 4.681 4.022 5.601 


bp <- barPlotCiError(as.matrix(cp$BiasTable),metricname = "BIAS",thesets = thesets,themethod = theMethod,main = "BIAS",offsets = c(0.5,0.5),args.legend = list(x = "bottomright"))

pander::pander(bp$barMatrix,caption = "BIAS",round = 3)
BIAS
  Default Regresion Method SVM Rendering Filtered
B:SWiMS -0.028 -0.477
LASSO -0.027 -0.565
RF -0.027 -0.548
SVM -0.598 -0.567 
pander::pander(bp$ciTable,caption = "BIAS with 95%CI",round = 3)
BIAS with 95%CI
  BIAS lower upper
Default Regresion Method -0.028 -0.832 0.775
Default Regresion Method -0.027 -0.854 0.799
Default Regresion Method -0.027 -0.818 0.764
Default Regresion Method -0.598 -1.713 0.516
SVM Rendering Filtered -0.477 -1.508 0.554
SVM Rendering Filtered -0.565 -1.558 0.428
SVM Rendering Filtered -0.548 -1.629 0.533
SVM Rendering Filtered -0.567 -1.666 0.533

1.3 Features Analysis

pander::pander(summary(Body_FAT_FRESA$BSWiMS.model,caption="Recurency ADAS13 model",round = 3))

  • coefficients:

    Table continues below
      Estimate lower mean upper
    waistcirc 0.01637 0.01637 0.01637 0.01637
    hipcirc.anthro3a 0.005835 0.005835 0.005835 0.005835
    hipcirc 0.0245 0.02338 0.0245 0.02561
    hipcirc.kneebreadth 0.001624 0.001438 0.001624 0.00181
    waistcirc.kneebreadth 0.00198 0.00198 0.00198 0.00198
    waistcirc.hipcirc 0.0001676 0.0001335 0.0001676 0.0002017
    hipcirc.anthro3b 0.007269 0.007269 0.007269 0.007269
    waistcirc.anthro3a 0.003909 0.003368 0.003909 0.00445
    kneebreadth.anthro3b 0.03577 0.0291 0.03577 0.04243
    anthro3a.anthro3b 0.1503 -0.002064 0.1503 0.3026
    waistcirc.anthro3b 0.004438 0.002044 0.004438 0.006832
    kneebreadth.anthro3a 0.04367 0.03069 0.04367 0.05664
    hipcirc.anthro4 0.005066 0.003331 0.005066 0.006801
    anthro3a -0.6686 -1.394 -0.6686 0.0567
    kneebreadth.anthro4 0.04304 0.02627 0.04304 0.05981
    hipcirc.anthro3c 0.008515 0.003574 0.008515 0.01346
    waistcirc.anthro4 0.004984 0.00118 0.004984 0.008787
    anthro3c -0.6946 -1.399 -0.6946 0.009866
    kneebreadth 0.3388 0.1235 0.3388 0.5542
    waistcirc.anthro3c 0.007264 0.003278 0.007264 0.01125
    anthro3b 0.02012 0.008769 0.02012 0.03147
    kneebreadth.anthro3c 0.06392 0.01785 0.06392 0.11
    hipcirc.elbowbreadth 0.01305 0.007221 0.01305 0.01888
    elbowbreadth -1.466 -1.873 -1.466 -1.059
    waistcirc.elbowbreadth 0.005371 -0.004781 0.005371 0.01552
    anthro3a.anthro4 0.08828 0.04666 0.08828 0.1299
    anthro3b.anthro3c 0.06721 0.04376 0.06721 0.09066
    anthro3b.anthro4 0.1429 0.1088 0.1429 0.177
    anthro3a.anthro3c 0.2281 0.1846 0.2281 0.2716
    elbowbreadth.kneebreadth 0.07767 0.01798 0.07767 0.1374
    elbowbreadth.anthro3c -0.1313 -0.1964 -0.1313 -0.0661
    anthro4 -0.1107 -0.197 -0.1107 -0.02441
    elbowbreadth.anthro4 -0.07389 -0.1016 -0.07389 -0.04613
    elbowbreadth.anthro3a -0.04018 -0.05906 -0.04018 -0.02129
    Table continues below
      u.MSE r.MSE model.MSE NeRI
    waistcirc 23.13 13.57 11.03 0.2113
    hipcirc.anthro3a 13.57 23.13 11.03 0.3239
    hipcirc 22.37 12.49 9.394 0.262
    hipcirc.kneebreadth 25.05 11.08 9.364 0.3211
    waistcirc.kneebreadth 21.16 13.66 9.124 0.2394
    waistcirc.hipcirc 17.65 11.99 9.438 0.3268
    hipcirc.anthro3b 13.66 21.16 9.124 0.4085
    waistcirc.anthro3a 14.99 11.79 9.6 0.1817
    kneebreadth.anthro3b 20.33 11.67 9.792 0.1028
    anthro3a.anthro3b 31.96 16.99 9.436 0.4366
    waistcirc.anthro3b 14.75 14.26 9.604 0.1845
    kneebreadth.anthro3a 21 11.39 9.676 0.107
    hipcirc.anthro4 13.56 14.39 9.701 0.2732
    anthro3a 35.99 11.24 9.808 0.09296
    kneebreadth.anthro4 20.16 14.22 10.93 0.08451
    hipcirc.anthro3c 16.71 16.9 10.5 0.2535
    waistcirc.anthro4 15.39 21.93 11.02 0.2563
    anthro3c 41.43 12.79 10.58 0.2475
    kneebreadth 49.3 15.82 12.45 0.1704
    waistcirc.anthro3c 17.48 29.58 12.55 0.3693
    anthro3b 41.52 15.81 12.19 0.1142
    kneebreadth.anthro3c 21.89 19.1 12.65 0.1849
    hipcirc.elbowbreadth 37.48 18.3 10.77 0.3437
    elbowbreadth 105.2 15.79 10.91 0.2632
    waistcirc.elbowbreadth 32.8 17.88 12.96 0.1649
    anthro3a.anthro4 32.02 18.43 12.86 0.1706
    anthro3b.anthro3c 33.99 20.79 11.34 0.2535
    anthro3b.anthro4 34.23 24.81 12.77 0.3239
    anthro3a.anthro3c 29.33 36.94 16.29 0.3204
    elbowbreadth.kneebreadth 61.5 22.88 15.78 0.2075
    elbowbreadth.anthro3c 40.03 20.09 15.89 0.2676
    anthro4 38.79 19.42 17.45 -0.01408
    elbowbreadth.anthro4 43.25 19.74 13.29 0.277
    elbowbreadth.anthro3a 42.27 22.49 17.21 0.09859
    Table continues below
      F.pvalue t.pvalue Sign.pvalue
    waistcirc 0.0001297 0.007319 0.04796
    hipcirc.anthro3a 4.843e-13 2.054e-05 0.004277
    hipcirc 7.502e-06 0.009104 0.01762
    hipcirc.kneebreadth 0.0006206 0.1178 0.004568
    waistcirc.kneebreadth 9.596e-08 0.01008 0.02841
    waistcirc.hipcirc 6.12e-05 0.002234 0.003959
    hipcirc.anthro3b 1.332e-14 7.657e-06 0.0003834
    waistcirc.anthro3a 0.0001461 0.01177 0.06591
    kneebreadth.anthro3b 0.0004689 0.2233 0.1819
    anthro3a.anthro3b 1.665e-10 0.0001457 0.0001516
    waistcirc.anthro3b 0 0.005098 0.04893
    kneebreadth.anthro3a 0.0008628 0.2932 0.1774
    hipcirc.anthro4 0 0.008489 0.01054
    anthro3a 0.001858 0.1251 0.2226
    kneebreadth.anthro4 2.865e-05 0.1477 0.2442
    hipcirc.anthro3c 0 0.0007915 0.01516
    waistcirc.anthro4 0 0.0005556 0.008463
    anthro3c 0.0002828 0.0603 0.02119
    kneebreadth 0.0001025 0.09712 0.08015
    waistcirc.anthro3c 0 1.312e-05 0.0005899
    anthro3b 6.955e-05 0.01476 0.06694
    kneebreadth.anthro3c 6.13e-07 0.01681 0.06399
    hipcirc.elbowbreadth 0 0.0009651 0.001786
    elbowbreadth 3.259e-07 0.0004344 0.01463
    waistcirc.elbowbreadth 7.492e-06 0.01228 0.07089
    anthro3a.anthro4 4.777e-06 0.01622 0.06485
    anthro3b.anthro3c 6.279e-10 0.002491 0.009559
    anthro3b.anthro4 0 0.0002747 0.003349
    anthro3a.anthro3c 0 4.206e-05 0.003503
    elbowbreadth.kneebreadth 9.196e-07 0.01491 0.03458
    elbowbreadth.anthro3c 2.773e-05 0.007359 0.01358
    anthro4 0.005961 0.1093 0.5
    elbowbreadth.anthro4 7.047e-08 0.004649 0.01204
    elbowbreadth.anthro3a 1.467e-05 0.03004 0.2185
      Wilcox.pvalue
    waistcirc 0.009925
    hipcirc.anthro3a 7.184e-05
    hipcirc 0.009909
    hipcirc.kneebreadth 0.04874
    waistcirc.kneebreadth 0.01281
    waistcirc.hipcirc 0.001208
    hipcirc.anthro3b 2.581e-05
    waistcirc.anthro3a 0.01852
    kneebreadth.anthro3b 0.2239
    anthro3a.anthro3b 0.0001993
    waistcirc.anthro3b 0.01217
    kneebreadth.anthro3a 0.3202
    hipcirc.anthro4 0.006806
    anthro3a 0.1876
    kneebreadth.anthro4 0.2006
    hipcirc.anthro3c 0.001539
    waistcirc.anthro4 0.00151
    anthro3c 0.06545
    kneebreadth 0.07822
    waistcirc.anthro3c 5.68e-05
    anthro3b 0.03654
    kneebreadth.anthro3c 0.01896
    hipcirc.elbowbreadth 0.0009285
    elbowbreadth 0.0006837
    waistcirc.elbowbreadth 0.02226
    anthro3a.anthro4 0.02113
    anthro3b.anthro3c 0.005764
    anthro3b.anthro4 0.0004527
    anthro3a.anthro3c 0.0001752
    elbowbreadth.kneebreadth 0.0195
    elbowbreadth.anthro3c 0.006379
    anthro4 1
    elbowbreadth.anthro4 0.005305
    elbowbreadth.anthro3a 0.05985
  • MSE: 9.202
  • R2: 0.9245

  • bootstrap:

gain <- length(Body_FAT_FRESA$BSWiMS.models$formula.list)/20
gplots::heatmap.2(gain*Body_FAT_FRESA$BSWiMS.models$bagging$formulaNetwork,trace="none",mar=c(10,10),main="B:SWiMS Formula Network")