library(mlbench)
## Warning: package 'mlbench' was built under R version 4.4.3
set.seed(200)
simulated <- mlbench.friedman1(200, sd = 1)
simulated <- cbind(simulated$x, simulated$y)
simulated <- as.data.frame(simulated)
colnames(simulated)[ncol(simulated)] <- "y"
library(caret)
## Loading required package: lattice
##
## Attaching package: 'caret'
## The following object is masked from 'package:purrr':
##
## lift
library(randomForest)
## randomForest 4.7-1.2
## Type rfNews() to see new features/changes/bug fixes.
##
## Attaching package: 'randomForest'
## The following object is masked from 'package:dplyr':
##
## combine
## The following object is masked from 'package:ggplot2':
##
## margin
set.seed(200)
model1 <- randomForest(y ~., data = simulated,
importance = TRUE,
ntree = 1000)
rfImp1 <- varImp(model1, scale = FALSE)
rfImp1 |> arrange(desc(Overall))
Inspecting the variables by order of importance score, V6 - V10 have importance scores near 0. Their score values range from 0.119 to -0.163, while the score values for V1 - V5 range from 8.541 to 2.268. This indicates that the forest made little use of the uninformative predictors.
simulated$duplicate1 <- simulated$V1 + rnorm(200) * .1
cor(simulated$duplicate1, simulated$V1)
## [1] 0.9442153
set.seed(200)
model2 <- randomForest(y ~., data = simulated,
importance = TRUE,
ntree = 1000)
rfImp2 <- varImp(model2, scale = FALSE)
rfImp2 |> arrange(desc(Overall))
paste("Model 1, V1:", rfImp1['V1','Overall'])
## [1] "Model 1, V1: 8.54067293369175"
paste("Model 2, V1:", rfImp2['V1','Overall'])
## [1] "Model 2, V1: 6.30192429862483"
When comparing Model 1 and Model 2’s V1 values, Model 2’s V1 values is lower. This is a result of the added value to the dataset that is highly correlated with the V1 value.
simulated$duplicate2 <- simulated$V1 + rnorm(200) * .1
cor(simulated$duplicate2, simulated$V1)
## [1] 0.9393087
set.seed(200)
model3 <- randomForest(y ~., data = simulated,
importance = TRUE,
ntree = 1000)
rfImp3 <- varImp(model3, scale = FALSE)
rfImp3 |> arrange(desc(Overall))
paste("Model 1, V1:", rfImp1['V1','Overall'])
## [1] "Model 1, V1: 8.54067293369175"
paste("Model 2, V1:", rfImp2['V1','Overall'])
## [1] "Model 2, V1: 6.30192429862483"
paste("Model 3, V1:", rfImp3['V1','Overall'])
## [1] "Model 3, V1: 5.26286548604689"
As in Model2, the V1 importance value in Model3 is lower again than it was in Model2 and Model1. The high level of correlation with the simulated variables has resulted in a lower level of importance for the V1 variable.
sim1 <- simulated |> select(-duplicate1, -duplicate2)
sim2 <- simulated |> select(-duplicate2)
sim3 <- simulated
conmodel1 <- cforest(y ~ ., data = sim1)
rfimp4conditional1 <- varimp(conmodel1, conditional = TRUE)
rfimp4unconditional1 <- varimp(conmodel1, conditional = FALSE)
rfimp4conditional1
## V1 V2 V3 V4 V5 V6
## 5.702959683 5.389079723 0.010640020 6.769493611 1.149964102 -0.004099685
## V7 V8 V9 V10
## -0.001884745 -0.020343817 -0.004473036 -0.032689567
paste("Conditional Importance:", rfimp4conditional1["V1"])
## [1] "Conditional Importance: 5.70295968340852"
paste("Unconditional Importance:", rfimp4unconditional1["V1"])
## [1] "Unconditional Importance: 8.98330064132288"
conmodel2 <- cforest(y ~ ., data = sim2)
rfimp4conditional2 <- varimp(conmodel2, conditional = TRUE)
rfimp4unconditional2 <- varimp(conmodel2, conditional = FALSE)
paste("Conditional Importance:", rfimp4conditional2["V1"])
## [1] "Conditional Importance: 2.93069479568143"
paste("Unconditional Importance:", rfimp4unconditional2["V1"])
## [1] "Unconditional Importance: 5.97635175607071"
conmodel3 <- cforest(y ~ ., data = sim3)
rfimp4conditional3 <- varimp(conmodel3, conditional = TRUE)
rfimp4unconditional3 <- varimp(conmodel3, conditional = FALSE)
paste("Conditional Importance:", rfimp4conditional3["V1"])
## [1] "Conditional Importance: 2.05206919500697"
paste("Unconditional Importance:", rfimp4unconditional3["V1"])
## [1] "Unconditional Importance: 5.36950994189056"
Inspecting the results of the conditional argument on the importance scores: - The conditional value begins at a lower value than the unconditional value. - The conditional value then declines as correlated values are repeatedly added, similar to how the unconditional values decline.
The forest could use either correlated copy as the predictor to make similar splits, so it relied less on V1 itself.
library(gbm)
## Warning: package 'gbm' was built under R version 4.4.3
## Loaded gbm 2.2.3
## This version of gbm is no longer under development. Consider transitioning to gbm3, https://github.com/gbm-developers/gbm3
set.seed(200)
sim1 <- simulated |> select(-duplicate1, -duplicate2)
gbmModel1 <- gbm(y ~ ., data = sim1, distribution = "gaussian", n.trees = 1000)
gbImp1 <- varImp(gbmModel1, numTrees = 1000, scale = FALSE)
gbImp1 |> arrange(desc(Overall))
set.seed(200)
sim2 <- simulated |> select(-duplicate2)
gbmModel2 <- gbm(y ~ ., data = sim2, distribution = "gaussian", n.trees = 1000)
gbImp2 <- varImp(gbmModel2, numTrees = 1000, scale = FALSE)
gbImp2 |> arrange(desc(Overall))
set.seed(200)
sim3 <- simulated
gbmModel3 <- gbm(y ~ ., data = sim3, distribution = "gaussian", n.trees = 1000)
gbImp3 <- varImp(gbmModel3, numTrees = 1000, scale = FALSE)
gbImp3 |> arrange(desc(Overall))
paste("Boosted Trees, no correlated values, V1:", gbImp1["V1", "Overall"])
## [1] "Boosted Trees, no correlated values, V1: 4703.55635788008"
paste("Boosted Trees, one correlated value, V1:", gbImp2["V1", "Overall"])
## [1] "Boosted Trees, one correlated value, V1: 3899.51050625266"
paste("Boosted Trees, two correlated values, V1:", gbImp3["V1", "Overall"])
## [1] "Boosted Trees, two correlated values, V1: 2835.18745132612"
Boosted trees displays the same pattern as previous models: as correlated values for V1 are introduced, the V1 importance value declines.
library(Cubist)
## Warning: package 'Cubist' was built under R version 4.4.3
X <- sim1 |> select(-y)
y <- sim1 |> pull(y)
set.seed(200)
cubistMod1 <- cubist(X, y)
cubImp1 <- varImp(cubistMod1)
cubImp1
X <- sim2 |> select(-y)
y <- sim2 |> pull(y)
set.seed(200)
cubistMod2 <- cubist(X, y)
cubImp2 <- varImp(cubistMod2)
cubImp2
X <- sim3 |> select(-y)
y <- sim3 |> pull(y)
set.seed(200)
cubistMod3 <- cubist(X, y)
cubImp3 <- varImp(cubistMod3)
cubImp3
paste("Cubist, no correlated values, V1:", cubImp1["V1", "Overall"])
## [1] "Cubist, no correlated values, V1: 50"
paste("Cubist, one correlated values, V1:", cubImp2["V1", "Overall"])
## [1] "Cubist, one correlated values, V1: 50"
paste("Cubist, two correlated values, V1:", cubImp3["V1", "Overall"])
## [1] "Cubist, two correlated values, V1: 50"
Unlike the other models, the Cubist model did not see a decline in the importance value of V1 as correlated values were introduced.
The following code simulates 1,000 datasets with three predictors of different granularities none of which have an actual meaningful relationship with the simulated yield values. Each dataset is fitted to a tree and it was set to a max_depth of 1. The feature that it was split on is then recorded and they are counted.
library(rpart)
set.seed(200)
root_split <- replicate(1000, {
n <- 200
dat <- data.frame(
yield = rnorm(n),
binary = sample(0:1, n, replace = TRUE),
ten_values = sample(1:10, n, replace = TRUE),
continuous = runif(n)
)
tree <- rpart(
yield ~ .,
data = dat,
method = "anova",
control = rpart.control(maxdepth = 1, cp = 0, xval = 0)
)
as.character(tree$frame$var[1])
})
100 * prop.table(table(root_split))
## root_split
## binary continuous ten_values
## 4.7 69.6 25.7
The results indicate that the regression tree chose the continuous predictor for the first split 69.6% of the time, the predictor with ten distinct values 25.7%, and the binary predictor 4.7%. The tree selected the continuous predictor most often. The large number of possible cutoffs gives more opportunities to reduce training error by chance. This effectively demonstrates split selection bias present in tree models.
The model on the right has a higher bagging fraction and learning rate at 0.9 for each. The chart on the right has a much steeper importance slope. This is related to the learning rate gamma value including 90% of each tree’s correction. Additionally, the higher bagging fraction uses more of the data for each random subsample which will decrease variance, and increase the contribution of the most important features relative to the others. At 90% of each tree’s correction, a few early splits can account for a large share of total reduction in error, making those features high importance.
This is in contrast to the chart with the lower bagging fraction and learning rate, at 0.1 each. Here the importance value decline by feature is less steep. The model will learn more gradually from a smaller subsample of the data which will create more variance and spread the importance score to more features than with the high value model.
The model with lower values would likely be more predictive of other values. As a model it learned more gradually which should reduce the risk of overfitting. The high valued model may have more possible overfitting, meaning it would be less applicable to new data. Comparing the RMSE would ultimately tell if this is the case.
Increasing interaction depth may flatten the importance profile. For the model with the low value for the bagging fraction and the learning rate, increasing interaction depth could lead to more splits contributing to the importance.
For the model with the high value for the bagging fraction and the learning rate, the greater depth would likely decrease the slope of the importance scores as the further depth would allow for splits on more features, potentially increasing variance in the results.
library(AppliedPredictiveModeling)
library(impute)
library(caret)
data(ChemicalManufacturingProcess)
chem <- ChemicalManufacturingProcess
set.seed(200)
split <- createDataPartition(
y = chem$Yield,
p = 0.8,
list = FALSE
)
train_chem <- chem[split,]
test_chem <- chem[-split,]
X_train = train_chem |> select(-Yield)
y_train = train_chem$Yield
X_test = test_chem |> select(-Yield)
y_test = test_chem$Yield
set.seed(200)
folds75 <- createFolds(
y_train,
k = 10,
returnTrain = TRUE
)
ctrl75 <- trainControl(
method = "cv",
number = 10,
index = folds75,
allowParallel = FALSE
)
set.seed(200)
singleTreeComplexity <- train(X_train,
y_train,
method = "rpart",
preProcess = c("zv","knnImpute"),
tuneLength = 9,
trControl = ctrl75)
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo,
## : There were missing values in resampled performance measures.
singleTreePredictComplexity <- predict(singleTreeComplexity, newdata = X_test)
singleTreeResultsComplexity <- postResample(pred = singleTreePredictComplexity, obs = y_test)
singleTreeResultsComplexity
## RMSE Rsquared MAE
## 1.7600424 0.2550851 1.2709342
plot(singleTreeComplexity)
set.seed(200)
singleTreeMaxDepth <- train(X_train,
y_train,
method = "rpart2",
preProcess = c("zv","knnImpute"),
tuneLength = 9,
trControl = ctrl75)
singleTreePredictMaxDepth <- predict(singleTreeMaxDepth, newdata = X_test)
singleTreeResultsMaxDepth <- postResample(pred = singleTreePredictMaxDepth, obs = y_test)
singleTreeResultsMaxDepth
## RMSE Rsquared MAE
## 1.6478912 0.3299598 1.1716700
plot(singleTreeMaxDepth)
set.seed(200)
singleTreeCTree <- train(X_train,
y_train,
method = "ctree",
preProcess = c("zv","knnImpute"),
trControl = ctrl75)
singleTreePredictCTree <- predict(singleTreeCTree, newdata = X_test)
singleTreeResultsCTree <- postResample(pred = singleTreePredictCTree, obs = y_test)
singleTreeResultsCTree
## RMSE Rsquared MAE
## 1.7026547 0.2924444 1.2786154
plot(singleTreeCTree)
set.seed(200)
mTree <- train(X_train,
y_train,
preProcess = c("zv","knnImpute"),
method = "M5",
trControl = ctrl75)
mTreePredict <- predict(mTree, newdata = X_test)
mTreeResults <- postResample(pred = mTreePredict, obs = y_test)
mTreeResults
## RMSE Rsquared MAE
## 1.4854475 0.4923197 1.1698792
plot(mTree)
##### Bagged Trees
set.seed(200)
bagTree <- train(X_train,
y_train,
method = "treebag",
preProcess = c("zv","knnImpute"),
trControl = ctrl75)
bagTreePredict <- predict(bagTree, newdata = X_test)
bagTreeResults <- postResample(pred = bagTreePredict, obs = y_test)
bagTreeResults
## RMSE Rsquared MAE
## 1.4460085 0.4853406 1.0057594
## Warning in randomForest.default(x, y, mtry = param$mtry, ...): invalid mtry:
## reset to within valid range
## RMSE Rsquared MAE
## 1.3546287 0.5847611 0.9655581
plot(rf)
set.seed(200)
BoostGrid <- expand.grid(
n.trees = c(50, 100, 150, 300, 500),
interaction.depth = c(1, 2, 3),
shrinkage = c(0.05, 0.1),
n.minobsinnode = 10
)
BoostT <- train(X_train,
y_train,
method = "gbm",
tuneGrid = BoostGrid,
preProcess = c("zv","knnImpute"),
trControl = ctrl75,
verbose = FALSE)
BoostTPredict <- predict(BoostT, newdata = X_test)
BoostTResults <- postResample(pred = BoostTPredict, obs = y_test)
BoostTResults
## RMSE Rsquared MAE
## 1.1732240 0.6844078 0.8565369
plot(BoostT)
set.seed(200)
CubistGrid <- expand.grid(
committees = c(1, 10, 50),
neighbors = c(0, 5, 9)
)
Cubist <- train(X_train,
y_train,
method = "cubist",
tuneGrid = CubistGrid,
preProcess = c("zv","knnImpute"),
trControl = ctrl75,
verbose = FALSE)
CubistPredict <- predict(Cubist, newdata = X_test)
CubistResults <- postResample(pred = CubistPredict, obs = y_test)
CubistResults
## RMSE Rsquared MAE
## 1.0766473 0.7611986 0.8702304
plot(Cubist)
testResults <- rbind(
"Single tree (cp)" = singleTreeResultsComplexity,
"Single tree (depth)" = singleTreeResultsMaxDepth,
"Single tree (ctree)" = singleTreeResultsCTree,
"Model tree (M5)" = mTreeResults,
"Bagged trees" = bagTreeResults,
"Random forest" = rfResults,
"Boosted trees" = BoostTResults,
"Cubist" = CubistResults
)
round(testResults[order(testResults[, "RMSE"]), , drop = FALSE], 3)
## RMSE Rsquared MAE
## Cubist 1.077 0.761 0.870
## Boosted trees 1.173 0.684 0.857
## Random forest 1.355 0.585 0.966
## Bagged trees 1.446 0.485 1.006
## Model tree (M5) 1.485 0.492 1.170
## Single tree (depth) 1.648 0.330 1.172
## Single tree (ctree) 1.703 0.292 1.279
## Single tree (cp) 1.760 0.255 1.271
models <- list(
"Single tree (depth)" = singleTreeMaxDepth,
"Single tree (cp)" = singleTreeComplexity,
"Single tree (ctree)" = singleTreeCTree,
"Model tree (M5)" = mTree,
"Bagged trees" = bagTree,
"Random forest" = rf,
"Boosted trees" = BoostT,
"Cubist" = Cubist
)
comparison <- purrr::imap_dfr(models, function(fit, model_name) {
cv <- caret::getTrainPerf(fit)
tibble(
Model = model_name,
CV_RMSE = cv$TrainRMSE,
Test_RMSE = testResults[model_name, "RMSE"],
CV_R2 = cv$TrainRsquared,
Test_R2 = testResults[model_name, "Rsquared"]
)
}) |>
arrange(CV_RMSE)
comparison
Inspecting the RMSE results for the assorted tree models. Cubist had the best performing RMSE on both cross validation and the test data.
If we recall from previous assignments, the most important features in the optimal linear model were:
lambdas <- 10^seq(3, -3, length.out = 100)
set.seed(200)
chem_ridge_model <- train(
x = train_chem |> select(-Yield),
y = train_chem$Yield,
method = "glmnet",
trControl = ctrl75,
preProcess = c('zv', 'knnImpute'),
tuneGrid = expand.grid(alpha = 0, lambda = lambdas),
lambda = lambdas
)
chem_pred <- predict(
chem_ridge_model,
newdata = test_chem |> select(-Yield)
)
postResample(pred = as.numeric(chem_pred), obs = test_chem$Yield)
## RMSE Rsquared MAE
## 3.5921787 0.1076416 1.5860509
linear_importance <- varImp(chem_ridge_model)
linear_importance_df <- linear_importance$importance |>
rownames_to_column("Feature") |>
arrange(desc(Overall)) |>
head(10)
linear_importance_df
The most important variables in the non-linear model were:
set.seed(200)
marsGrid <- expand.grid(.degree = 1:2, .nprune = 2:17)
marsModel <- train(
x = X_train,
y = y_train,
preProcess = c("zv","knnImpute", "center", "scale"),
method = "earth",
tuneGrid = marsGrid,
trControl = ctrl75
)
## Loading required package: earth
## Warning: package 'earth' was built under R version 4.4.3
## Loading required package: Formula
## Loading required package: plotmo
## Warning: package 'plotmo' was built under R version 4.4.3
## Loading required package: plotrix
marsPredict <- predict(marsModel, newdata = X_test)
marsResults <- postResample(pred = marsPredict, obs = y_test)
non_linear_importance <- varImp(marsModel)
non_linear_importance_df <- non_linear_importance$importance |>
rownames_to_column("Feature") |> arrange(desc(Overall)) |>
head(10)
non_linear_importance_df
The best tree based model on the test data was the Cubist. As shown below eight of the top ten variables are Manufacturing Processes. This is similar to the optimal linear and nonlinear models, where Manufacturing Processes also dominated.
CubistImp <- varImp(Cubist)
cubistImportance <- CubistImp$importance |> arrange(desc(Overall)) |> head(10)
cubistImportance
importance_comparison <- list(
linear_importance_df |>
select(Feature, Ridge = Overall),
non_linear_importance_df |>
select(Feature, MARS = Overall),
cubistImportance |>
rownames_to_column("Feature") |>
select(Feature, Cubist = Overall)
) |>
purrr::reduce(dplyr::full_join, by = "Feature")
importance_comparison
When comparing all of the most important features, all three models have ManufacturingProcess32 as the most or second most important feature. Additionally, all models also include ManufacturingProcess13 in their ten most important features.
The optimal single tree model was the single tree with max depth. The cross validation selected a max depth = 2.
single_tree <- singleTreeMaxDepth$finalModel
node_ids <- rownames(single_tree$frame)
batch_node <- node_ids[single_tree$where]
is_leaf <- single_tree$frame$var == "<leaf>"
old_par <- par(mfrow = c(2, 1), mar = c(1, 3, 2, 2))
coords <- plot(single_tree, uniform = TRUE, margin = 0.05)
text(single_tree, pretty = 0, digits = 3)
title("Single tree, max depth = 2")
par(mar = c(4, 4, 2, 2))
boxplot(y_train ~ batch_node,
xlab = "Terminal node", ylab = "Observed yield",
col = "lightblue")
par(old_par)
The single tree indicates that the two manufacturing processes, 32 and 06 identify the highest yield groups. While a biological process is an indicator of the lower yield groups. The boxplots show that the typical yields increases across the four groups, although observed yields overlapped, especially for nodes 5 and 6.