library(AppliedPredictiveModeling)  
library(mlbench)                    
library(caret)                      
library(earth)                      
library(kernlab)                   
library(nnet)                       
library(glmnet)                     # lasso from last week
library(RANN)                       # needed for knn imputation

7.2 Friedman simulated data

The data comes from a known formula that only uses X1 to X5. X6 to X10 are just noise.

# starter code from the book, 200 training rows and 5000 test rows
set.seed(200)
trainingData <- mlbench.friedman1(200, sd = 1)
trainingData$x <- data.frame(trainingData$x)
featurePlot(trainingData$x, trainingData$y)

testData <- mlbench.friedman1(5000, sd = 1)
testData$x <- data.frame(testData$x)

Tune several models

I tried the four models from the chapter: KNN, MARS, SVM and a neural net.

# 10 fold cross-validation for every model
ctrl <- trainControl(method = "cv", number = 10)

# same seed before each model so they all get the same folds

# KNN whic was the example from the book
set.seed(200)
knnModel <- train(x = trainingData$x, y = trainingData$y, method = "knn",
                  preProc = c("center", "scale"), tuneLength = 10, trControl = ctrl)

# MARS which I  tried with and without interactions and different numbers of terms
set.seed(200)
marsModel <- train(x = trainingData$x, y = trainingData$y, method = "earth",
                   tuneGrid = expand.grid(degree = 1:2, nprune = 2:15), trControl = ctrl)

# SVM with a radial kernel, try 10 values of cost
set.seed(200)
svmModel <- train(x = trainingData$x, y = trainingData$y, method = "svmRadial",
                  preProc = c("center", "scale"), tuneLength = 10, trControl = ctrl)

# neural net with average of 5 small nets, try 1 to 5 hidden units and two decay values
set.seed(200)
nnetModel <- train(x = trainingData$x, y = trainingData$y, method = "avNNet",
                   preProc = c("center", "scale"),
                   tuneGrid = expand.grid(decay = c(0.01, 0.1), size = 1:5, bag = FALSE),
                   linout = TRUE, trace = FALSE, maxit = 500, trControl = ctrl)

Which models give the best performance?

models <- list(KNN = knnModel, MARS = marsModel, SVM = svmModel, NNet = nnetModel)

# cross validation results for the best tune of each model
cv_res <- t(sapply(models, function(m) unlist(getTrainPerf(m)[1:2])))

# test set results
test_res <- t(sapply(models, function(m) postResample(predict(m, testData$x), testData$y)))

results <- cbind(cv_res, test_res[, 1:2])
colnames(results) <- c("CV RMSE", "CV R2", "Test RMSE", "Test R2")
round(results, 2)
##      CV RMSE CV R2 Test RMSE Test R2
## KNN     3.09  0.68      3.12    0.67
## MARS    1.26  0.93      1.17    0.94
## SVM     1.92  0.85      2.05    0.83
## NNet    1.95  0.84      2.19    0.81

MARS wins by a lot. Its test RMSE is 1.17 with an R-squared of 0.94. SVM (2.05) and the neural net (2.19) are in the middle, and KNN is the worst at 3.12 and this makes sense actually, the real formula is a few smooth peice plus an interaction between X1 and X2 and that is the kind of things MARS is good at building. KNN had a hard time because half the predictors are noise for the most part and they threw the distances off.

Does MARS select the informative predictors?

marsModel$bestTune
##    nprune degree
## 27     14      2
varImp(marsModel)
## earth variable importance
## 
##    Overall
## X1  100.00
## X4   75.24
## X2   48.74
## X5   15.53
## X3    0.00

Yes indeed, MARS only used X1 to X5 and left out all the other five noisu variables which were X6 to X10. X1, X4 and X2 matter the most while X3 only shows a 0 because the scores are rescaled so the weakest predictor that was used lands at 0, but it is still in the model.

7.5 Chemical manufacturing process

Same prep as Homework 7

Same steps as last time, the sensor failure zeros are treated as missing, missing values are filled in with the 5 nearest neighbors, then a 75/25 split occurs.

data(ChemicalManufacturingProcess)
cmp <- ChemicalManufacturingProcess
pred_names <- setdiff(names(cmp), "Yield")

# a single 0 in a column where everything else is far from 0 is a sensor failure so were just make it NA
for (v in pred_names) {
  x <- cmp[[v]]
  nz <- x[!is.na(x) & x != 0]
  if (sum(x == 0, na.rm = TRUE) == 1 && length(nz) > 2 && mean(nz) / sd(nz) > 5) cmp[[v]][x == 0] <- NA
}

# fill missing values using the 5 most similar runs
imp <- preProcess(cmp[, pred_names], method = "knnImpute", k = 5)
X <- predict(imp, cmp[, pred_names])
X <- sweep(sweep(X, 2, imp$std, "*"), 2, imp$mean, "+")   # back to original units
cmp <- data.frame(Yield = cmp$Yield, X)

# 75/25 split
set.seed(624)
idx <- createDataPartition(cmp$Yield, p = 0.75, list = FALSE)
train <- cmp[idx, ]
test  <- cmp[-idx, ]

# 10-fold CV repeated 5 times with the  same folds for every model
ctrl <- trainControl(method = "repeatedcv", number = 10, repeats = 5,
                     index = createMultiFolds(train$Yield, k = 10, times = 5))

# drop the near zero variance predictors and  then center, then scale
pp <- c("nzv", "center", "scale")

Train the nonlinear models

# same four models as 7.2
knn5  <- train(Yield ~ ., data = train, method = "knn", tuneLength = 10,
               preProcess = pp, trControl = ctrl)

mars5 <- train(Yield ~ ., data = train, method = "earth",
               tuneGrid = expand.grid(degree = 1:2, nprune = 2:15),
               preProcess = pp, trControl = ctrl)

svm5  <- train(Yield ~ ., data = train, method = "svmRadial", tuneLength = 10,
               preProcess = pp, trControl = ctrl)

nnet5 <- train(Yield ~ ., data = train, method = "avNNet",
               tuneGrid = expand.grid(decay = c(0.01, 0.1), size = c(1, 3, 5), bag = FALSE),
               preProcess = pp, linout = TRUE, trace = FALSE, maxit = 500, trControl = ctrl)

(a) Which model has the best resampling and test set performance?

models5 <- list(KNN = knn5, MARS = mars5, SVM = svm5, NNet = nnet5)

# cross validation and test set results to stand  side by side
cv5   <- t(sapply(models5, function(m) unlist(getTrainPerf(m)[1:2])))
test5 <- t(sapply(models5, function(m) postResample(predict(m, test), test$Yield)))

results5 <- cbind(cv5, test5[, 1:2])
colnames(results5) <- c("CV RMSE", "CV R2", "Test RMSE", "Test R2")
round(results5, 2)
##      CV RMSE CV R2 Test RMSE Test R2
## KNN     1.25  0.52      1.49    0.49
## MARS    1.17  0.60      1.32    0.60
## SVM     1.17  0.57      1.21    0.68
## NNet    1.33  0.51      1.61    0.42

MARS and SVM are basically tied in cross validation with both RMSE beign around 1.17. On the test set SVM does better with an RMSE of 1.21 and an R-squared of 0.68. So for the best nonlinear model I am going with SVM. KNN and the neural net are behind on both.

(b) Most important predictors

# caret has no built in importance for SVM so varImp scores each predictor by how well a smooth curve of just that predictor fits Yield
svm_imp <- varImp(svm5)
plot(svm_imp, top = 10)

#  Top ten nsmrd
svm_top10 <- rownames(svm_imp$importance)[order(-svm_imp$importance$Overall)][1:10]

# how many are biological and how many are process
table(substr(svm_top10, 1, 3))
## 
## Bio Man 
##   4   6

Most important one is ManufacturingProcess32 . The top ten has 6 process variables and 4 biological ones, so process is ahead but it doesn’t really dominate.

Now the top ten from last week’s best linear model (the lasso), fit on the same split.

# same elastic net grid as Homework 7, it landed on the lasso again with alpha being 1
lin5 <- train(Yield ~ ., data = train, method = "glmnet",
              tuneGrid = expand.grid(alpha = seq(0, 1, by = 0.1),
                                     lambda = 10^seq(-3, 0.5, length = 30)),
              preProcess = pp, trControl = ctrl)
lin5$bestTune
##     alpha     lambda
## 313     1 0.02807216
lin_imp <- varImp(lin5)$importance
lin_top10 <- rownames(lin_imp)[order(-lin_imp$Overall)][1:10]
lin_top10
##  [1] "ManufacturingProcess32" "ManufacturingProcess33" "ManufacturingProcess09"
##  [4] "BiologicalMaterial03"   "ManufacturingProcess37" "ManufacturingProcess06"
##  [7] "ManufacturingProcess04" "ManufacturingProcess29" "ManufacturingProcess17"
## [10] "ManufacturingProcess39"
intersect(svm_top10, lin_top10)   # in both lists
## [1] "ManufacturingProcess32" "BiologicalMaterial03"   "ManufacturingProcess09"
setdiff(svm_top10, lin_top10)     # only in the SVM list
## [1] "BiologicalMaterial06"   "ManufacturingProcess31" "ManufacturingProcess13"
## [4] "ManufacturingProcess36" "BiologicalMaterial12"   "BiologicalMaterial02"  
## [7] "ManufacturingProcess11"

Only 3 predictors are in both lists, and ManufacturingProcess32 appear to be number one in both. The lasso list was almost all process variables (9 of 10), meanwhile the SVM list has more of the biological ones. Part of the reason is that the lasso keeps one predictor out of a group that move together and drops the rest, and this ranking looks at each predictor individually.

(c) Predictors that are only in the nonlinear top ten

only_svm <- setdiff(svm_top10, lin_top10)

# each predictor against Yield with a smooth line
featurePlot(x = train[, only_svm], y = train$Yield,
            type = c("p", "smooth"), layout = c(4, 2))

The three biological ones (02, 06 and 12) all go up as yield goes up meaning that better raw material gives better yield. For 06 and 12 the line goes flat at the high end meaning that after a point more of it doesn’t really help. For the process ones, yield tends to go up with ManufacturingProcess11 and goes down with 13 and 36. Process 36 only has a few values so it looks kind of like steps, and Process 31 is hard to read because almost every run is around 70 and one odd run near 60 pulls the line. So with all this, my takeaway is that the biological materials matter but you can’t change them, you can only check them before a run. The process ones like 11, 13 and 36 can be changed therefore, those are the ones to work on to get better yield.