Data 624 Homework 07

7.2

Loading the training and testing data

trainingData <- mlbench.friedman1(200, sd = 1)
trainingData$x <- data.frame(trainingData$x)
featurePlot(trainingData$x, trainingData$y)

testData <- mlbench.friedman1(5000, sd = 1)
testData$x <- data.frame(testData$x)

The following code sets up a consistent training control cross validation so that results are directly comparable.

set.seed(200)

folds72 <- createFolds(
  trainingData$y,
  k = 5,
  returnTrain = TRUE
)

ctrl72 <- trainControl(
  method = "cv",
  number = 5,
  index = folds72,
  allowParallel = FALSE
)

knn

Fitting the knn model.

set.seed(200)

knnModel <- train(x = trainingData$x,
                  y = trainingData$y,
                  method = "knn",
                  preProc = c('center', 'scale'),
                  trControl = ctrl72,
                  tuneLength = 10
                  )

knnModel
## k-Nearest Neighbors 
## 
## 200 samples
##  10 predictor
## 
## Pre-processing: centered (10), scaled (10) 
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results across tuning parameters:
## 
##   k   RMSE      Rsquared   MAE     
##    5  3.238910  0.5836930  2.663624
##    7  3.181990  0.6173594  2.596400
##    9  3.118743  0.6567222  2.528975
##   11  3.149714  0.6579169  2.556553
##   13  3.152493  0.6835182  2.534883
##   15  3.168762  0.6917688  2.553509
##   17  3.149480  0.7096402  2.546620
##   19  3.152352  0.7255634  2.547220
##   21  3.198272  0.7199107  2.591646
##   23  3.269315  0.7059526  2.671478
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was k = 9.
knnModel$results[which.min(knnModel$results$RMSE), ]

Predicting the witheld data using the KNN model.

set.seed(200)

knnPred <- predict(knnModel, newdata = testData$x)

knnResults <- postResample(pred = knnPred, obs = testData$y)
knnResults
##      RMSE  Rsquared       MAE 
## 3.1172319 0.6556622 2.4899907

The test and cross-validation results are similar indicating that the resampling did not appear to overfit the model and the model may have relatively good generalization.

Neural Network

The neural network is fit below with a selection of size and decay values.

set.seed(200)

nneModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "nnet",
  preProc = c('center', 'scale'),
  tuneGrid = expand.grid(size = c(25, 50, 75), decay = c(0.001, 0.01, 0.1, 1)),
  trControl = ctrl72,
  linout = TRUE,
  trace = FALSE,
  maxit = 500,
  MaxNWts = 1000
)

nneModel
## Neural Network 
## 
## 200 samples
##  10 predictor
## 
## Pre-processing: centered (10), scaled (10) 
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results across tuning parameters:
## 
##   size  decay  RMSE      Rsquared   MAE     
##   25    0.001  2.417185  0.7692591  1.938978
##   25    0.010  2.557241  0.7566397  2.009410
##   25    0.100  2.174954  0.8144335  1.682910
##   25    1.000  2.048946  0.8346804  1.591834
##   50    0.001  2.594348  0.7562955  2.039374
##   50    0.010  2.158141  0.8191828  1.671643
##   50    0.100  2.082642  0.8199364  1.626178
##   50    1.000  2.011944  0.8419770  1.560870
##   75    0.001  2.299417  0.8004563  1.805010
##   75    0.010  1.975049  0.8407852  1.559329
##   75    0.100  2.109130  0.8219871  1.642919
##   75    1.000  2.034929  0.8358733  1.557973
## 
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were size = 75 and decay = 0.01.

Then the model is applied to the withheld data.

nnePredict <- predict(nneModel, newdata = testData$x)

nneResults <- postResample(pred = nnePredict, obs = testData$y)

nneResults
##      RMSE  Rsquared       MAE 
## 2.2176511 0.8004593 1.7035109

Average Neural Networks

Below and Average Neural Network model is fit.

set.seed(200)

avnneModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "avNNet",
  preProc = c('center', 'scale'),
  tuneGrid = expand.grid(size = c(50, 75), decay = c(0.001, 0.01, 0.1), bag =  TRUE),
  trControl = ctrl72,
  repeats = 10,
  linout = TRUE,
  trace = FALSE,
  maxit = 500,
  MaxNWts = 1000
)
## Warning: executing %dopar% sequentially: no parallel backend registered
avnneModel
## Model Averaged Neural Network 
## 
## 200 samples
##  10 predictor
## 
## Pre-processing: centered (10), scaled (10) 
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results across tuning parameters:
## 
##   size  decay  RMSE      Rsquared   MAE     
##   50    0.001  2.140345  0.8230452  1.710449
##   50    0.010  2.080200  0.8279477  1.610662
##   50    0.100  2.108104  0.8277432  1.673913
##   75    0.001  2.051253  0.8324886  1.591372
##   75    0.010  1.992239  0.8377156  1.555732
##   75    0.100  2.034022  0.8304961  1.555760
## 
## Tuning parameter 'bag' was held constant at a value of TRUE
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were size = 75, decay = 0.01 and bag = TRUE.

The model is then applied to the witheld data for test-set evaluation.

avnnePredict <- predict(avnneModel, newdata = testData$x)

avnneResults <- postResample(pred = avnnePredict, obs = testData$y)

avnneResults
##      RMSE  Rsquared       MAE 
## 2.0004490 0.8384509 1.5029571

MARS

A multivariate adaptive regression spline (MARS) model is fit below.

set.seed(200)

marsGrid <- expand.grid(.degree = 1:2, .nprune = 2:20)

marsModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "earth",
  tuneGrid = marsGrid,
  trControl = ctrl72
)

marsModel
## Multivariate Adaptive Regression Spline 
## 
## 200 samples
##  10 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results across tuning parameters:
## 
##   degree  nprune  RMSE      Rsquared   MAE      
##   1        2      4.424952  0.2202294  3.6480727
##   1        3      3.766496  0.4620687  3.0129706
##   1        4      2.607163  0.7285888  2.0891517
##   1        5      2.273994  0.7898513  1.7902514
##   1        6      2.253916  0.7922682  1.7355670
##   1        7      1.748504  0.8766718  1.3842402
##   1        8      1.717478  0.8814445  1.3394916
##   1        9      1.654711  0.8919469  1.2832738
##   1       10      1.652273  0.8930306  1.2910890
##   1       11      1.666754  0.8911933  1.2892857
##   1       12      1.636810  0.8952611  1.2766528
##   1       13      1.666115  0.8903460  1.3012236
##   1       14      1.664627  0.8902921  1.3002668
##   1       15      1.664627  0.8902921  1.3002668
##   1       16      1.664627  0.8902921  1.3002668
##   1       17      1.664627  0.8902921  1.3002668
##   1       18      1.664627  0.8902921  1.3002668
##   1       19      1.664627  0.8902921  1.3002668
##   1       20      1.664627  0.8902921  1.3002668
##   2        2      4.424952  0.2202294  3.6480727
##   2        3      3.766496  0.4620687  3.0129706
##   2        4      2.607163  0.7285888  2.0891517
##   2        5      2.275723  0.7888996  1.7693843
##   2        6      2.210736  0.7977084  1.6899880
##   2        7      1.759404  0.8735665  1.3867212
##   2        8      1.641039  0.8899526  1.2899374
##   2        9      1.449042  0.9125083  1.1393677
##   2       10      1.383324  0.9226123  1.0925164
##   2       11      1.273126  0.9350998  1.0152548
##   2       12      1.214703  0.9400638  0.9538853
##   2       13      1.210562  0.9404654  0.9642744
##   2       14      1.203251  0.9414395  0.9459766
##   2       15      1.206666  0.9410937  0.9458757
##   2       16      1.177486  0.9441118  0.9135369
##   2       17      1.183215  0.9436490  0.9176178
##   2       18      1.183215  0.9436490  0.9176178
##   2       19      1.183215  0.9436490  0.9176178
##   2       20      1.183215  0.9436490  0.9176178
## 
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were nprune = 16 and degree = 2.

The model is then applied to the witheld testing data.

set.seed(200)

marsPredict <- predict(marsModel, newdata = testData$x)

marsResults <- postResample(pred = marsPredict, obs = testData$y)

marsResults
##      RMSE  Rsquared       MAE 
## 1.1492504 0.9471145 0.9158382

SVM

Below a Support Vector Machines model is fit to the training data.

set.seed(200)

svmModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "svmRadial",
  preProcess = c('center','scale'),
  tuneLength = 14,
  trControl = ctrl72
)

svmModel
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 200 samples
##  10 predictor
## 
## Pre-processing: centered (10), scaled (10) 
## Resampling: Cross-Validated (5 fold) 
## Summary of sample sizes: 160, 160, 160, 160, 160 
## Resampling results across tuning parameters:
## 
##   C        RMSE      Rsquared   MAE     
##      0.25  2.598321  0.7819081  2.073604
##      0.50  2.317770  0.8010368  1.818724
##      1.00  2.153031  0.8184174  1.667403
##      2.00  2.022443  0.8368244  1.578313
##      4.00  1.956036  0.8473000  1.548489
##      8.00  1.929698  0.8517430  1.535875
##     16.00  1.925020  0.8530319  1.536033
##     32.00  1.925020  0.8530319  1.536033
##     64.00  1.925020  0.8530319  1.536033
##    128.00  1.925020  0.8530319  1.536033
##    256.00  1.925020  0.8530319  1.536033
##    512.00  1.925020  0.8530319  1.536033
##   1024.00  1.925020  0.8530319  1.536033
##   2048.00  1.925020  0.8530319  1.536033
## 
## Tuning parameter 'sigma' was held constant at a value of 0.06299324
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were sigma = 0.06299324 and C = 16.

The model is then applied to the withheld testing data.

set.seed(200)

svmPredict <- predict(svmModel, newdata = testData$x)

svmResults <- postResample(pred = svmPredict, obs = testData$y)

svmResults
##      RMSE  Rsquared       MAE 
## 2.0736997 0.8256573 1.5751967
set.seed(200)

results <- as.data.frame(rbind(
  knnResults, 
  nneResults,
  avnneResults,
  marsResults,
  svmResults)) |> arrange(RMSE)

results

The table above displays how each model performed when applied to the withheld testing set. With this data and these hyperparameters, the MARS model performed the best overall with the lowest RMSE and the highest R2 indicating that the model explains most of the variance in the data.

The following code shows the MARS model’s predictors and their importance values.

mars_importance <- varImp(marsModel)

mars_importance
## earth variable importance
## 
##    Overall
## X1  100.00
## X4   75.23
## X2   48.73
## X5   15.52
## X3    0.00

The MARS model does select the informative predictors of X1-X5.

7.5

Loading, imputing and splitting the data into training and test sets.

library(AppliedPredictiveModeling)
library(impute)
data(ChemicalManufacturingProcess)

chem <- ChemicalManufacturingProcess

set.seed(200)

split <- createDataPartition(
  y = chem$Yield,
  p = 0.8,
  list = FALSE
)

train_chem <- chem[split,]
test_chem <- chem[-split,]

X_train = train_chem |> select(-Yield)
y_train = train_chem$Yield

X_test = test_chem |>  select(-Yield)
y_test = test_chem$Yield

Establish a consistent cross-validation method:

set.seed(200)

folds75 <- createFolds(
  y_train,
  k = 10,
  returnTrain = TRUE
)

ctrl75 <- trainControl(
  method = "cv",
  number = 10,
  index = folds75,
  allowParallel = FALSE
)
MARS

Fit a MARS model to the data, and then applies it to the withheld testing data.

set.seed(200)

marsGrid <- expand.grid(.degree = 1:2, .nprune = 2:17)

marsModel <- train(
  x = X_train,
  y = y_train,
  preProcess = c("zv","knnImpute", "center", "scale"),
  method = "earth",
  tuneGrid = marsGrid,
  trControl = ctrl75
)

marsPredict <- predict(marsModel, newdata = X_test)

marsResults <- postResample(pred = marsPredict, obs = y_test)

marsModel
## Multivariate Adaptive Regression Spline 
## 
## 144 samples
##  57 predictor
## 
## Pre-processing: nearest neighbor imputation (57), centered (57), scaled (57) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 131, 129, 129, 129, 130, 130, ... 
## Resampling results across tuning parameters:
## 
##   degree  nprune  RMSE      Rsquared   MAE      
##   1        2      1.354834  0.4724446  1.0668857
##   1        3      1.230036  0.5559614  1.0035529
##   1        4      1.228099  0.5611592  1.0035877
##   1        5      1.208571  0.5770716  0.9819023
##   1        6      1.228627  0.5584009  1.0072348
##   1        7      1.232565  0.5605999  1.0021640
##   1        8      1.244671  0.5542553  1.0074135
##   1        9      1.250628  0.5555221  0.9736156
##   1       10      1.274106  0.5536108  1.0123565
##   1       11      1.255030  0.5612946  1.0002058
##   1       12      1.195947  0.5918154  0.9569875
##   1       13      1.201658  0.5943086  0.9577096
##   1       14      1.192812  0.6024507  0.9365914
##   1       15      1.204400  0.6002635  0.9444477
##   1       16      1.206063  0.5942439  0.9414269
##   1       17      1.187185  0.5997785  0.9268973
##   2        2      1.354834  0.4724446  1.0668857
##   2        3      1.217192  0.5643805  0.9803751
##   2        4      1.229900  0.5520767  0.9907865
##   2        5      1.235716  0.5551905  0.9973403
##   2        6      1.294018  0.5239019  1.0357332
##   2        7      1.288685  0.5361712  1.0389083
##   2        8      1.251149  0.5644033  1.0170757
##   2        9      1.269877  0.5779509  1.0224877
##   2       10      1.302995  0.5594170  1.0330423
##   2       11      1.295322  0.5668564  1.0262513
##   2       12      1.331835  0.5555177  1.0343807
##   2       13      1.353433  0.5323816  1.0553688
##   2       14      1.327149  0.5521222  1.0404810
##   2       15      1.331401  0.5475030  1.0434075
##   2       16      1.309747  0.5654181  1.0313725
##   2       17      1.286864  0.5747038  1.0048306
## 
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were nprune = 17 and degree = 1.
marsResults
##      RMSE  Rsquared       MAE 
## 1.1886471 0.6927459 0.9718061
NNE

Below a neural network model is fit to the data, and it is then applied to the withheld portion of the data.

set.seed(200)

nneModel <- train(
  x = X_train,
  y = y_train,
  method = "nnet",
  preProc = c("zv","knnImpute", "center", "scale"),
  tuneGrid = expand.grid(size = c(1, 3, 5, 7, 9), decay = c(0, 0.001, 0.01, 0.1, 1, 10, 20)),
  trControl = ctrl75,
  linout = TRUE,
  trace = FALSE,
  maxit = 500,
  MaxNWts = 1000
)

nnePredict <- predict(nneModel, newdata = X_test)

nneResults <- postResample(pred = nnePredict, obs = y_test)

nneModel
## Neural Network 
## 
## 144 samples
##  57 predictor
## 
## Pre-processing: nearest neighbor imputation (57), centered (57), scaled (57) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 131, 129, 129, 129, 130, 130, ... 
## Resampling results across tuning parameters:
## 
##   size  decay  RMSE      Rsquared   MAE      
##   1     0e+00  1.671393  0.2342954  1.3220894
##   1     1e-03  1.793103  0.2769899  1.4256473
##   1     1e-02  1.817708  0.2840825  1.4882470
##   1     1e-01  1.492419  0.4158841  1.1923261
##   1     1e+00  1.655448  0.4029841  1.2720626
##   1     1e+01  2.157112  0.4842891  1.7007936
##   1     2e+01  3.448652  0.3909502  2.9804237
##   3     0e+00  2.704970  0.3522961  2.1086050
##   3     1e-03  2.186250  0.3156352  1.6940361
##   3     1e-02  2.593273  0.2831346  2.0412120
##   3     1e-01  2.732455  0.2489663  1.9968593
##   3     1e+00  1.886151  0.3764644  1.4068166
##   3     1e+01  1.506111  0.5618233  1.2024371
##   3     2e+01  2.083796  0.5739115  1.6836189
##   5     0e+00  3.045886  0.1786286  2.4900937
##   5     1e-03  2.774522  0.2776753  2.1382056
##   5     1e-02  2.636041  0.2622693  2.0420338
##   5     1e-01  2.308066  0.3753383  1.7854707
##   5     1e+00  1.477165  0.5005336  1.2029397
##   5     1e+01  1.329570  0.5807133  1.0700516
##   5     2e+01  1.652188  0.5860473  1.3345455
##   7     0e+00  5.235903  0.2231852  3.8404747
##   7     1e-03  3.077130  0.2545167  2.3258548
##   7     1e-02  2.643710  0.3025703  1.9473729
##   7     1e-01  2.387804  0.2489305  1.7864070
##   7     1e+00  1.498257  0.5101301  1.1890778
##   7     1e+01  1.269263  0.5872498  1.0168245
##   7     2e+01  1.466552  0.5919075  1.1920407
##   9     0e+00  6.058443  0.1275601  4.3206442
##   9     1e-03  4.739341  0.2545520  3.3937707
##   9     1e-02  3.048860  0.2304751  2.3367865
##   9     1e-01  2.743193  0.1324591  2.0988434
##   9     1e+00  1.620680  0.4335232  1.2706896
##   9     1e+01  1.248128  0.5905409  0.9951484
##   9     2e+01  1.376768  0.5950312  1.1214352
## 
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were size = 9 and decay = 10.
nneResults
##     RMSE Rsquared      MAE 
## 2.986672 0.153515 1.596242
SVM

Below a support vector machine model is fit to the data and then used to predict values base don the withheld data.

set.seed(200)

svmModel <- train(
  x = X_train,
  y = y_train,
  method = "svmRadial",
  preProcess = c("zv","knnImpute", "center", "scale"),
  tuneLength = 14,
  trControl = ctrl75
)

svmPredict <- predict(svmModel, newdata = X_test)

svmResults <- postResample(pred = svmPredict, obs = y_test)

svmModel
## Support Vector Machines with Radial Basis Function Kernel 
## 
## 144 samples
##  57 predictor
## 
## Pre-processing: nearest neighbor imputation (57), centered (57), scaled (57) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 131, 129, 129, 129, 130, 130, ... 
## Resampling results across tuning parameters:
## 
##   C        RMSE      Rsquared   MAE      
##      0.25  1.351947  0.5145456  1.0973241
##      0.50  1.255559  0.5619327  1.0087005
##      1.00  1.156991  0.6342514  0.9245480
##      2.00  1.099387  0.6750334  0.8869933
##      4.00  1.084030  0.6869060  0.8810549
##      8.00  1.075064  0.6889293  0.8747455
##     16.00  1.069741  0.6926211  0.8705992
##     32.00  1.069741  0.6926211  0.8705992
##     64.00  1.069741  0.6926211  0.8705992
##    128.00  1.069741  0.6926211  0.8705992
##    256.00  1.069741  0.6926211  0.8705992
##    512.00  1.069741  0.6926211  0.8705992
##   1024.00  1.069741  0.6926211  0.8705992
##   2048.00  1.069741  0.6926211  0.8705992
## 
## Tuning parameter 'sigma' was held constant at a value of 0.01356073
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were sigma = 0.01356073 and C = 16.
svmResults
##      RMSE  Rsquared       MAE 
## 1.3447732 0.6142816 1.0039691
KNN

A KNN model is fit to the data and then used to predict values for the withheld data.

set.seed(200)

knnModel <- train(x = X_train,
                  y = y_train,
                  method = "knn",
                  preProc = c("zv","knnImpute", "center", "scale"),
                  trControl = ctrl75,
                  tuneLength = 10
                  )

knnPred <- predict(knnModel, newdata = X_test)

knnResults <- postResample(pred = knnPred, obs = y_test)

knnModel
## k-Nearest Neighbors 
## 
## 144 samples
##  57 predictor
## 
## Pre-processing: nearest neighbor imputation (57), centered (57), scaled (57) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 131, 129, 129, 129, 130, 130, ... 
## Resampling results across tuning parameters:
## 
##   k   RMSE      Rsquared   MAE      
##    5  1.177570  0.5846202  0.9103973
##    7  1.212272  0.5606186  0.9733490
##    9  1.254775  0.5277336  1.0091729
##   11  1.291067  0.5077988  1.0467505
##   13  1.301841  0.5103401  1.0430786
##   15  1.319641  0.5022610  1.0446832
##   17  1.334194  0.4917434  1.0616371
##   19  1.347888  0.4908697  1.0739816
##   21  1.367743  0.4765887  1.1036922
##   23  1.380631  0.4638328  1.1054711
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was k = 5.
knnResults
##      RMSE  Rsquared       MAE 
## 1.5937742 0.4424488 1.2695625

a.

The optimal results from each resampled set are combined into a dataframe below.

marsBestTrain <- marsModel$results[which.min(marsModel$results$RMSE),]
nneBestTrain <- nneModel$results[which.min(nneModel$results$RMSE),]
svmBestTrain <- svmModel$results[which.min(svmModel$results$RMSE),]
knnBestTrain <- knnModel$results[which.min(knnModel$results$RMSE),]

bestTrain <- as.data.frame(bind_rows(MARS = marsBestTrain, 
                                    NNE = nneBestTrain, 
                                    SVM = svmBestTrain, 
                                    KNN = knnBestTrain,
                                    .id = "Model")) |> 
  arrange(RMSE)

bestTrain

Inspecting the dataframe, the SVM model performed the best on the cross validation among the models and settings tested.

Below, the model results from the test sets are combined into a dataframe.

bestTest <- as.data.frame(bind_rows(MARS = marsResults,
                                   SVM = svmResults,
                                   KNN = knnResults,
                                   NNE = nneResults,
                                   .id = "Model")) |> 
  arrange(RMSE)

bestTest

On the test sets though, the MARS model performed the best, with an improved R2 when compared to the resampling results. Since the MARS model performed the best among these tests and setting on unseen data, it will be used to continue to compare with the linear models from previous assignments.

b.

Variable importance is collected and listed below. From the MARS model there are eight features, six Manufacturing, and two Biological.

non_linear_importance <- varImp(marsModel)

non_linear_importance_df <- non_linear_importance$importance |> 
  rownames_to_column("Feature") |> arrange(desc(Overall)) |> 
  head(10)

non_linear_importance_df
lambdas <- 10^seq(3, -3, length.out = 100)

set.seed(200)

chem_ridge_model <- train(
  x = train_chem |> select(-Yield),
  y = train_chem$Yield,
  method = "glmnet",
  trControl = ctrl75,
  preProcess = c('zv', 'knnImpute'),
  tuneGrid = expand.grid(alpha = 0, lambda = lambdas),
  lambda = lambdas
)

chem_pred <- predict(
  chem_ridge_model,
  newdata = test_chem |> select(-Yield)
)

postResample(pred = as.numeric(chem_pred), obs = test_chem$Yield)
##      RMSE  Rsquared       MAE 
## 3.5921787 0.1076416 1.5860509
linear_importance <- varImp(chem_ridge_model)

linear_importance_df <- linear_importance$importance |> 
  rownames_to_column("Feature") |> 
  arrange(desc(Overall)) |> 
  head(10)

linear_importance_df

Comparing the top eight non-linear and top ten linear features by importance, 2 of the features exist in both lists, while six are unique to the non-linear model.

c.

The following code selects the features that are unique to the non-linear model.

unique_non_linear_results <- anti_join(
  non_linear_importance_df, 
  linear_importance_df, 
  by = "Feature")

unique_non_linear_results
features <- unique_non_linear_results$Feature

for (feature in features){
  
  p <- ggplot(train_chem, aes(x = .data[[feature]], y = Yield)) +
        geom_point(alpha = 0.6, na.rm = TRUE) +
        geom_smooth(method = "loess", formula = y ~ x, se = FALSE, na.rm = TRUE) +
        labs(x = feature, y = "Yield") +
        theme_minimal()
  
  print(p)
}

For all plots of the most important features that are unique to the best non-linear model when compared with the most important features of the best linear model, there are no indicators of a linear relationship between the process and Yield.

Generally, the values are clustered within a range or at intervals across the range of Yield outputs. As a result, they do not appear linear. When fit with a Loess line several appear curved, further indicating the non-linear aspect to these features.