Benchmarks for Modeling OAI KL Scores
José Tamez-Peña
April 30, 2018
1 FRESA.CAD Regresion Benchmark
1.1 OAI Joint Space Width KL Modelling Data Set
load(file = "OAIJSW_BASELINE.RDATA")
tsamples <- sample(nrow(baseline),nrow(baseline)*0.2)
OAItrainSet <- baseline[tsamples,]
OAItestSet <- baseline[-tsamples,]
bmodel <- BSWiMS.model(formula = BLKL ~ 1,data = OAItrainSet,NumberofRepeats = 20 )
1.2 Benchmark
cp <- CVOrdBenchmark(theData = baseline, theOutcome = "BLKL", reps = 20, fraction = 0.10, topincluded = 40 )
elapcol <- names(cp$times[[1]]) == "elapsed"
cputimes <- list(Fresa = mean(cp$times$fresatime[ elapcol ]),LASSO = mean(cp$times$LASSOtime[ elapcol ]),RF = mean(cp$times$RFtime[ elapcol ]),SVM = mean(cp$times$SVMtime[ elapcol ]))
featsize <- list(Fresa = mean(cp$featSize$FRESASize),LASSO = mean(cp$featSize$LASSOSize),Univ = mean(cp$featSize$UNIVSize))
1.2.1 Results
prolr <- predict(bmodel$oridinalModels$polr,OAItestSet)
boxplot(as.numeric(as.character(prolr))~OAItestSet$BLKL)
tb <- table(prolr,OAItestSet$BLKL)
pander::pander(tb)| 0 | 1 | 2 | 3 | 4 | |
|---|---|---|---|---|---|
| 0 | 348 | 169 | 337 | 4 | 0 |
| 1 | 0 | 0 | 0 | 0 | 0 |
| 2 | 376 | 437 | 1220 | 251 | 1 |
| 3 | 4 | 15 | 100 | 534 | 51 |
| 4 | 0 | 1 | 2 | 32 | 106 |
kp <- kappa2(cbind(as.numeric(as.character(prolr)),OAItestSet$BLKL),"unweighted")
pander::pander(kp$value)0.3481
prlin <- predict(bmodel$bagging$bagged.model,OAItestSet)
boxplot(prlin~OAItestSet$BLKL)
kp <- kappa2(cbind(as.integer(prlin+0.5),OAItestSet$BLKL),"unweighted")
pander::pander(kp$value)0.1371
pr <- predict(bmodel$oridinalModels,OAItestSet)
boxplot(pr[,1]~OAItestSet$BLKL)
boxplot(pr[,2]~OAItestSet$BLKL)
summary(pr) V1 V2 V3 V4 Min. :0.002008 Min. :0.00 Min. :0.5461 Min. :0.03236
1st Qu.:1.000556 1st Qu.:0.00 1st Qu.:0.6407 1st Qu.:0.64717
Median :1.996206 Median :2.00 Median :0.6716 Median :0.74075
Mean :1.834584 Mean :1.64 Mean :0.6980 Mean :0.74337
3rd Qu.:2.995752 3rd Qu.:3.00 3rd Qu.:0.7250 3rd Qu.:0.84453
Max. :4.000000 Max. :4.00 Max. :0.9996 Max. :0.99999
V5 V6
Min. :-0.9567 Min. :-0.99944
1st Qu.:-0.3864 1st Qu.:-0.32287
Median :-0.1410 Median :-0.02595
Mean :-0.0923 Mean :-0.01489
3rd Qu.: 0.1805 3rd Qu.: 0.31141
Max. : 0.9697 Max. : 0.96973
tb <- table(as.integer(pr[,1] + 0.5),as.integer(OAItestSet$BLKL))
pander::pander(tb)| 0 | 1 | 2 | 3 | 4 | |
|---|---|---|---|---|---|
| 0 | 326 | 151 | 306 | 10 | 0 |
| 1 | 100 | 100 | 257 | 39 | 2 |
| 2 | 271 | 292 | 813 | 41 | 0 |
| 3 | 30 | 77 | 280 | 702 | 47 |
| 4 | 1 | 2 | 3 | 29 | 109 |
tb <- table(as.integer(pr[,2] + 0.5),as.integer(OAItestSet$BLKL))
pander::pander(tb)| 0 | 1 | 2 | 3 | 4 | |
|---|---|---|---|---|---|
| 0 | 393 | 214 | 429 | 21 | 0 |
| 1 | 130 | 140 | 363 | 50 | 2 |
| 2 | 179 | 207 | 616 | 50 | 0 |
| 3 | 24 | 59 | 241 | 656 | 42 |
| 4 | 2 | 2 | 10 | 44 | 114 |
kp <- kappa2(cbind(as.integer(pr[,1] + 0.5),OAItestSet$BLKL),"unweighted")
pander::pander(kp$value)0.34
kp <- kappa2(cbind(as.integer(pr[,2] + 0.5),OAItestSet$BLKL),"unweighted")
pander::pander(kp$value)0.3179
#The Times
pander::pander(cputimes)- Fresa: 17.17
- LASSO: 0.2015
- RF: 0.5525
- SVM: 0.0765
pander::pander(featsize)- Fresa: 14
- LASSO: 31.25
- Univ: 36.1
plotMAEEvolution(cp,30,main="Mean Absolute Error (MAE)", location="topright")
bp <- barPlotCiError(as.matrix(cp$CorTable),metricname="Kendall Correlation",thesets=thesets,themethod=theMethod,main="Kendall Correlation",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))
pander::pander(bp$barMatrix,caption = "Kendall Correlation",round = 3)| Regresion Method | |
|---|---|
| B:SWiMS | 0.439 |
| B:SWiMS(OLR) | 0.398 |
| B:SWiMS(SVM) | 0.424 |
| RF | 0.436 |
| RF(OLR) | 0.389 |
| RF(SVM) | 0.415 |
| LASSO | 0.448 |
| SVM | 0.428 |
| Univ.(SVM) | 0.425 |
| LASSO(SVM) | 0.427 |
pander::pander(bp$ciTable,caption = "Kendall Correlation with 95%CI",round = 3)| Kendall Correlation | lower | upper | |
|---|---|---|---|
| Regresion Method | 0.439 | 0.424 | 0.455 |
| Regresion Method | 0.398 | 0.396 | 0.4 |
| Regresion Method | 0.424 | 0.421 | 0.426 |
| Regresion Method | 0.436 | 0.434 | 0.439 |
| Regresion Method | 0.389 | 0.386 | 0.391 |
| Regresion Method | 0.415 | 0.412 | 0.417 |
| Regresion Method | 0.448 | 0.433 | 0.463 |
| Regresion Method | 0.428 | 0.426 | 0.431 |
| Regresion Method | 0.425 | 0.423 | 0.427 |
| Regresion Method | 0.427 | 0.424 | 0.429 |
bp <- barPlotCiError(as.matrix(cp$KappaTable),metricname="Kappa Agreement",thesets=thesets,themethod=theMethod,main="Kappa Agreement",offsets = c(0.5,0.05),args.legend = list(x = "bottomright"))
pander::pander(bp$barMatrix,caption = "Kappa Agreement",round = 3)| Regresion Method | |
|---|---|
| B:SWiMS | 0.333 |
| B:SWiMS(OLR) | 0.335 |
| B:SWiMS(SVM) | 0.363 |
| RF | 0.356 |
| RF(OLR) | 0.33 |
| RF(SVM) | 0.342 |
| LASSO | 0.203 |
| SVM | 0.375 |
| Univ.(SVM) | 0.368 |
| LASSO(SVM) | 0.375 |
pander::pander(bp$ciTable,caption = "Kappa Agreement with 95%CI",round = 3)| Kappa Agreement | lower | upper | |
|---|---|---|---|
| Regresion Method | 0.333 | 0.317 | 0.348 |
| Regresion Method | 0.335 | 0.319 | 0.351 |
| Regresion Method | 0.363 | 0.347 | 0.379 |
| Regresion Method | 0.356 | 0.34 | 0.371 |
| Regresion Method | 0.33 | 0.315 | 0.346 |
| Regresion Method | 0.342 | 0.328 | 0.357 |
| Regresion Method | 0.203 | 0.189 | 0.217 |
| Regresion Method | 0.375 | 0.36 | 0.391 |
| Regresion Method | 0.368 | 0.352 | 0.384 |
| Regresion Method | 0.375 | 0.359 | 0.39 |
bp <- barPlotCiError(as.matrix(cp$MAETable),metricname="MAE",thesets=thesets,themethod=theMethod,main="MAE",offsets = c(0.5,5),args.legend = list(x = "bottomright"))
pander::pander(bp$barMatrix,caption = "MAE",round = 3)| Regresion Method | |
|---|---|
| B:SWiMS | 0.678 |
| B:SWiMS(OLR) | 0.727 |
| B:SWiMS(SVM) | 0.682 |
| RF | 0.665 |
| RF(OLR) | 0.74 |
| RF(SVM) | 0.755 |
| LASSO | 0.693 |
| SVM | 0.669 |
| Univ.(SVM) | 0.674 |
| LASSO(SVM) | 0.67 |
pander::pander(bp$ciTable,caption = "MAE with 95%CI",round = 3)| MAE | lower | upper | |
|---|---|---|---|
| Regresion Method | 0.678 | 0.665 | 0.692 |
| Regresion Method | 0.727 | 0.713 | 0.742 |
| Regresion Method | 0.682 | 0.669 | 0.696 |
| Regresion Method | 0.665 | 0.652 | 0.679 |
| Regresion Method | 0.74 | 0.726 | 0.755 |
| Regresion Method | 0.755 | 0.74 | 0.77 |
| Regresion Method | 0.693 | 0.679 | 0.707 |
| Regresion Method | 0.669 | 0.656 | 0.682 |
| Regresion Method | 0.674 | 0.661 | 0.688 |
| Regresion Method | 0.67 | 0.657 | 0.684 |
bp <- barPlotCiError(as.matrix(cp$BiasTable),metricname="BIAS",thesets=thesets,themethod=theMethod,main="BIAS",offsets = c(0.5,0.5),args.legend = list(x = "bottomright"))
pander::pander(bp$barMatrix,caption = "BIAS",round = 3)| Regresion Method | |
|---|---|
| B:SWiMS | 0.165 |
| B:SWiMS(OLR) | -0.175 |
| B:SWiMS(SVM) | -0.085 |
| RF | -0.08 |
| RF(OLR) | -0.159 |
| RF(SVM) | -0.265 |
| LASSO | -0.113 |
| SVM | -0.076 |
| Univ.(SVM) | -0.068 |
| LASSO(SVM) | -0.064 |
pander::pander(bp$ciTable,caption = "BIAS with 95%CI",round = 3)| BIAS | lower | upper | |
|---|---|---|---|
| Regresion Method | 0.165 | 0.136 | 0.193 |
| Regresion Method | -0.175 | -0.205 | -0.144 |
| Regresion Method | -0.085 | -0.114 | -0.055 |
| Regresion Method | -0.08 | -0.108 | -0.052 |
| Regresion Method | -0.159 | -0.189 | -0.128 |
| Regresion Method | -0.265 | -0.296 | -0.235 |
| Regresion Method | -0.113 | -0.136 | -0.09 |
| Regresion Method | -0.076 | -0.104 | -0.047 |
| Regresion Method | -0.068 | -0.097 | -0.039 |
| Regresion Method | -0.064 | -0.093 | -0.035 |
1.3 Features
pander::pander(summary(bmodel$bagging$bagged.model,caption="KL model",round = 3))coefficients:
Table continues below Estimate lower mean upper u.MSE MtoLDiff 0.113 0.08918 0.113 0.1369 0.8884 mCV 0.01473 -0.01068 0.01473 0.04013 1.148 CVJSW 0.04983 0.03692 0.04983 0.06274 0.8187 STDJSW 0.07528 0.05183 0.07528 0.09874 0.9116 TPCFDS 0.000819 -0.0009818 0.000819 0.00262 1.335 AGE 0.005747 0.004824 0.005747 0.00667 1.291 MeanJSW -0.04294 -0.06787 -0.04294 -0.01801 1.153 MCMJSW -0.03635 -0.06295 -0.03635 -0.009754 1.073 BMI 0.005745 0.004565 0.005745 0.006926 1.271 LJSW700 -0.03982 -0.07051 -0.03982 -0.009133 1.246 LatSlope 0.01319 0.002736 0.01319 0.02365 1.274 LJSW850 -0.009015 -0.03794 -0.009015 0.01991 1.324 LJSW725 -0.02155 -0.03921 -0.02155 -0.003903 1.267 medialSlope -0.003381 -0.01334 -0.003381 0.00658 1.331 MaxCV 0.002971 -0.002366 0.002971 0.008307 1.056 XMJSW -0.00127 -0.004961 -0.00127 0.002421 1.326 stdLJSW -0.03157 -0.05624 -0.03157 -0.006901 1.304 MaxSlope 0.0006856 -0.000117 0.0006856 0.001488 1.328 lCV 0.01273 0.0101 0.01273 0.01536 1.319 WEIGHT 0.0006611 0.0003932 0.0006611 0.0009289 1.282 JointSlope 0.008147 0.004882 0.008147 0.01141 1.268 LateralSlope 0.002591 0.001684 0.002591 0.003497 1.272 MedCurvature 0.0008128 -0.001324 0.0008128 0.00295 1.33 LJSW775 -0.005019 -0.005801 -0.005019 -0.004237 1.303 meanLJSW -0.01295 -0.0147 -0.01295 -0.0112 1.304 LatCurvature -0.002163 -0.003193 -0.002163 -0.001132 1.312 JSW225 -0.002133 -0.003036 -0.002133 -0.00123 1.057 JSW200 -0.003638 -0.005058 -0.003638 -0.002218 1.044 LJSW750 -0.003617 -0.004295 -0.003617 -0.002939 1.29 Table continues below r.MSE model.MSE NeRI F.pvalue t.pvalue MtoLDiff 1.144 0.8059 0.1762 0 2.047e-15 mCV 0.8374 0.8159 0.03922 0 0.03461 CVJSW 1.072 0.7749 0.1479 0 1.457e-13 STDJSW 1.035 0.7884 0.1355 0 6.623e-12 TPCFDS 1.089 1.089 -0.00295 0.5301 0.4699 AGE 0.7929 0.764 0.08655 1.073e-09 0.004811 MeanJSW 0.8605 0.7861 0.0746 0 0.000223 MCMJSW 0.8004 0.7638 0.109 0 3.465e-05 BMI 1.116 1.067 0.06597 1.595e-11 0.001204 LJSW700 0.8295 0.7612 0.0905 0 0.000247 LatSlope 0.9289 0.8966 0.03005 5.569e-09 0.02999 LJSW850 0.8237 0.7945 0.04794 0 0.01436 LJSW725 0.8319 0.7726 0.0672 0 0.0005829 medialSlope 0.9372 0.9342 -0.0107 0.1132 0.2868 MaxCV 0.8327 0.8022 0.02015 0 0.0392 XMJSW 0.9771 0.9737 0.004833 0.0785 0.2795 stdLJSW 0.8292 0.792 0.07249 0 0.009049 MaxSlope 1.287 1.281 0.05035 0.03979 0.05075 lCV 0.9184 0.8841 0.04614 0 0.002163 WEIGHT 0.9471 0.9212 0.04892 1.761e-07 0.004382 JointSlope 0.7894 0.767 0.05405 1e-07 0.002953 LateralSlope 1.169 1.122 0.04238 5.603e-10 0.02134 MedCurvature 0.9938 0.9917 0.002257 0.185 0.3401 LJSW775 0.8388 0.7958 0.05373 0 0.0009538 meanLJSW 0.8324 0.7592 0.07723 0 5.857e-05 LatCurvature 1.169 1.148 0.0662 1.618e-05 0.02808 JSW225 0.843 0.8162 0.04915 2.311e-08 0.02275 JSW200 0.8119 0.7876 0.05667 3.325e-08 0.001511 LJSW750 0.8321 0.7857 0.06118 0 0.003322 Sign.pvalue Wilcox.pvalue MtoLDiff 4.362e-09 1.091e-12 mCV 0.06996 0.02905 CVJSW 6.242e-07 2.324e-10 STDJSW 2.488e-06 1.262e-09 TPCFDS 0.4555 0.6854 AGE 0.003067 0.005984 MeanJSW 0.006799 0.00118 MCMJSW 0.0002402 4.51e-05 BMI 0.01588 0.002159 LJSW700 0.001235 0.0001873 LatSlope 0.1514 0.04902 LJSW850 0.06277 0.02758 LJSW725 0.009506 0.00265 medialSlope 0.213 0.3752 MaxCV 0.2197 0.07881 XMJSW 0.1903 0.2854 stdLJSW 0.00777 0.003513 MaxSlope 0.03965 0.08317 lCV 0.008572 0.001229 WEIGHT 0.04859 0.009873 JointSlope 0.03666 0.008432 LateralSlope 0.0765 0.03458 MedCurvature 0.3828 0.5182 LJSW775 0.03121 0.004614 meanLJSW 0.005087 0.0001915 LatCurvature 0.01902 0.01303 JSW225 0.05522 0.0436 JSW200 0.03788 0.01702 LJSW750 0.02748 0.009975 - MSE: 0.7333
- R2: 0.4512
bootstrap:
gain <- length(bmodel$oridinalModels$formulas)/(20*4)
gplots::heatmap.2(gain*bmodel$bagging$formulaNetwork,trace="none",mar=c(10,10),main="B:SWiMS Formula Network")