Cross Valid. & Model Selection, Ta-Feng

交叉驗證與參數調校流程

Fig-1: Supervised Learning Process

Fig-2: CV, Model Sel. & Parameter Tuning

---

Preparing Data

Libraries

Sys.setlocale("LC_ALL","C")

[1] "C"

library(dplyr)
library(ggplot2)
library(caTools)
library(Matrix)
library(rpart)
library(rpart.plot)
library(caret)
library(doParallel)

Loading and Spliting

rm(list=ls(all=TRUE))
load("data/tf2.rdata")
A$buy = factor(ifelse(A$buy, "yes", "no"))  # comply to the rule of caret
TR = A[spl, c(2:9,11)]
TS = A[!spl, c(2:9,11)]

Turn on Parallel Processing

clust = makeCluster(detectCores())
registerDoParallel(clust); getDoParWorkers()

[1] 4

CV for Classification Model

CV Control for Classification

ctrl = trainControl(
  method="repeatedcv", number=10,    # 10-fold, Repeated CV
  savePredictions = "final", classProbs=TRUE,
  summaryFunction=twoClassSummary)

CV: rpart(), Classification Tree

ctrl$repeats = 2
t0 = Sys.time(); set.seed(2)
cv.rpart = train(
  buy ~ ., data=TR, method="rpart", 
  trControl=ctrl, metric="ROC",
  tuneGrid = expand.grid(cp = seq(0.0002,0.001,0.0001) ) )
Sys.time() - t0

Time difference of 19.511 secs

plot(cv.rpart)

cv.rpart$results 

      cp     ROC    Sens    Spec    ROCSD   SensSD   SpecSD
1 0.0002 0.70841 0.75985 0.57913 0.010364 0.014478 0.019012
2 0.0003 0.71555 0.78010 0.57423 0.010408 0.029139 0.025506
3 0.0004 0.71786 0.79710 0.56716 0.015507 0.028938 0.026220
4 0.0005 0.71821 0.80497 0.56219 0.017172 0.027989 0.024565
5 0.0006 0.71697 0.81041 0.56058 0.016999 0.021776 0.021361
6 0.0007 0.71382 0.81311 0.56112 0.017613 0.016638 0.016526
7 0.0008 0.70874 0.81628 0.55945 0.013033 0.013177 0.013865
8 0.0009 0.70777 0.81777 0.55923 0.010477 0.014680 0.014994
9 0.0010 0.70758 0.81819 0.55918 0.010578 0.014432 0.015508

Classification Tree, Final Model

rpart1 = rpart(buy ~ ., TR, method="class", cp=0.0005)
predict(rpart1, TS, type="prob")[,2] %>% 
  colAUC(TS$buy)

              [,1]
no vs. yes 0.74018

---

CV: glm(), General Linear Model

ctrl$repeats = 2
t0 = Sys.time(); set.seed(2)
cv.glm = train(
  buy ~ ., data=TR, method="glm", 
  trControl=ctrl, metric="ROC")
Sys.time() - t0

Time difference of 4.4909 secs

cv.glm$results

  parameter     ROC    Sens    Spec    ROCSD   SensSD   SpecSD
1      none 0.74879 0.80576 0.56856 0.010178 0.013218 0.014611

glm(), Final Model

glm1 = b=glm(buy ~ ., TR, family=binomial)
predict(glm1, TS, type="response") %>% colAUC(TS$buy)

             [,1]
no vs. yes 0.7556

---

CV for Regression Model

Spliting Data

A2 = subset(A, A$buy == "yes") %>% mutate_at(c("m","rev","amount"), log10)
TR2 = A2[ spl2, c(2:10)]
TS2 = A2[!spl2, c(2:10)]

CV Control for Regression

ctrl2 = trainControl(
  method="repeatedcv", number=10,    # 10-fold, Repeated CV
  savePredictions = "final")

CV: rpart() Regression Tree

ctrl$repeats = 2
set.seed(2)
cv.rpart2 = train(
  amount ~ ., data=TR2, method="rpart", 
  trControl=ctrl2, metric="Rsquared",
  tuneGrid = expand.grid(cp = seq(0.0008,0.0024,0.0001) ) )
plot(cv.rpart2)

cv.rpart2$results

       cp    RMSE Rsquared     MAE    RMSESD RsquaredSD     MAESD
1  0.0008 0.42675  0.27564 0.32766 0.0090329   0.023591 0.0061173
2  0.0009 0.42477  0.28117 0.32635 0.0103407   0.025773 0.0066172
3  0.0010 0.42359  0.28403 0.32583 0.0102501   0.025828 0.0064855
4  0.0011 0.42309  0.28549 0.32526 0.0100893   0.025807 0.0065420
5  0.0012 0.42297  0.28581 0.32530 0.0096765   0.023913 0.0058852
6  0.0013 0.42313  0.28532 0.32534 0.0097162   0.023931 0.0058034
7  0.0014 0.42320  0.28514 0.32556 0.0100614   0.026334 0.0061210
8  0.0015 0.42279  0.28628 0.32541 0.0104651   0.026525 0.0062114
9  0.0016 0.42252  0.28711 0.32525 0.0105918   0.025878 0.0061396
10 0.0017 0.42256  0.28688 0.32533 0.0107874   0.026540 0.0063368
11 0.0018 0.42275  0.28627 0.32552 0.0111151   0.028421 0.0066105
12 0.0019 0.42335  0.28421 0.32591 0.0110973   0.027216 0.0066057
13 0.0020 0.42369  0.28307 0.32599 0.0110733   0.027393 0.0067058
14 0.0021 0.42339  0.28411 0.32581 0.0110896   0.028081 0.0067905
15 0.0022 0.42349  0.28378 0.32590 0.0113953   0.029014 0.0069534
16 0.0023 0.42335  0.28421 0.32575 0.0111905   0.028332 0.0066832
17 0.0024 0.42342  0.28393 0.32573 0.0108693   0.027563 0.0064283

rpart(), Regression Tree Final Model

rpart2 = rpart(amount ~ ., data=TR2, cp=0.0016)
SST = sum((TS2$amount - mean(TR2$amount))^ 2)
SSE = sum((predict(rpart2, TS2) -  TS2$amount)^2)
(r2.ts.rpart2 = 1 - (SSE/SST))

[1] 0.24672

CV: lm(), Linear Model

ctrl$repeats = 2
set.seed(2)
cv.lm2 = train(
  amount ~ ., data=TR2, method="lm", 
  trControl=ctrl2, metric="Rsquared",
    tuneGrid = expand.grid( intercept = seq(0,5,0.5) ) 
  )
plot(cv.lm2)

cv.lm2$results

   intercept    RMSE Rsquared     MAE  RMSESD RsquaredSD     MAESD
1        0.0 0.43331  0.27303 0.33095 0.01424   0.033815 0.0103186
2        0.5 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
3        1.0 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
4        1.5 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
5        2.0 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
6        2.5 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
7        3.0 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
8        3.5 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
9        4.0 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
10       4.5 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456
11       5.0 0.42046  0.29444 0.32349 0.01090   0.028294 0.0066456

lm() Final Model

lm2 = lm(amount ~ ., TR2)
SST = sum((TS2$amount - mean(TR2$amount))^ 2)
SSE = sum((predict(lm2, TS2) -  TS2$amount)^2)
(r2.ts.lm2 = 1 - (SSE/SST))

[1] 0.26226

Stop Parallel Processing

stopCluster(clust)

---