Mars function

Author

Guibril Ramde

Exercise 7.2

#install.packages('kernlab')
library(mlbench)
Warning: package 'mlbench' was built under R version 4.5.2
library(tidyverse)
Warning: package 'ggplot2' was built under R version 4.5.2
Warning: package 'tibble' was built under R version 4.5.2
Warning: package 'tidyr' was built under R version 4.5.2
Warning: package 'readr' was built under R version 4.5.2
Warning: package 'purrr' was built under R version 4.5.2
Warning: package 'dplyr' was built under R version 4.5.2
── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.2.0     ✔ readr     2.1.6
✔ forcats   1.0.1     ✔ stringr   1.6.0
✔ ggplot2   4.0.2     ✔ tibble    3.3.1
✔ lubridate 1.9.4     ✔ tidyr     1.3.2
✔ purrr     1.2.1     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(ggplot2)
library(caret)
Loading required package: lattice

Attaching package: 'caret'

The following object is masked from 'package:purrr':

    lift
library(earth)
Warning: package 'earth' was built under R version 4.5.2
Loading required package: Formula
Loading required package: plotmo
Warning: package 'plotmo' was built under R version 4.5.2
Loading required package: plotrix
Warning: package 'plotrix' was built under R version 4.5.2
library(ipred)
Warning: package 'ipred' was built under R version 4.5.2
library(e1071)
Warning: package 'e1071' was built under R version 4.5.2

Attaching package: 'e1071'

The following object is masked from 'package:ggplot2':

    element
library(kernlab)

Attaching package: 'kernlab'

The following object is masked from 'package:purrr':

    cross

The following object is masked from 'package:ggplot2':

    alpha
#library(randomForest)

set.seed(200)

# Training dataset: 200 observations
trainingData <- mlbench.friedman1(200, sd = 1)
trainingData$x <- as.data.frame(trainingData$x)

# Testing dataset: 5000 observations
testData <- mlbench.friedman1(5000, sd = 1)
testData$x <- as.data.frame(testData$x)

# Exploratory analysis
dim(trainingData$x)
[1] 200  10
summary(trainingData$x)
       V1                 V2                  V3                 V4         
 Min.   :0.002806   Min.   :0.0004409   Min.   :0.003296   Min.   :0.00819  
 1st Qu.:0.217222   1st Qu.:0.2795877   1st Qu.:0.284872   1st Qu.:0.23178  
 Median :0.513935   Median :0.5106664   Median :0.537307   Median :0.44458  
 Mean   :0.481461   Mean   :0.5126936   Mean   :0.508570   Mean   :0.46939  
 3rd Qu.:0.680814   3rd Qu.:0.7326078   3rd Qu.:0.746155   3rd Qu.:0.68454  
 Max.   :0.998992   Max.   :0.9840194   Max.   :0.999923   Max.   :0.99929  
       V5                 V6                 V7                  V8          
 Min.   :0.001756   Min.   :0.002901   Min.   :0.0003388   Min.   :0.004698  
 1st Qu.:0.285408   1st Qu.:0.276341   1st Qu.:0.2092393   1st Qu.:0.289413  
 Median :0.534330   Median :0.497598   Median :0.4688035   Median :0.497961  
 Mean   :0.517948   Mean   :0.496841   Mean   :0.4636166   Mean   :0.512865  
 3rd Qu.:0.748180   3rd Qu.:0.749616   3rd Qu.:0.6752972   3rd Qu.:0.727624  
 Max.   :0.991577   Max.   :0.998987   Max.   :0.9943477   Max.   :0.999561  
       V9              V10          
 Min.   :0.0176   Min.   :0.003172  
 1st Qu.:0.2304   1st Qu.:0.317826  
 Median :0.5289   Median :0.535922  
 Mean   :0.5016   Mean   :0.542409  
 3rd Qu.:0.7218   3rd Qu.:0.789736  
 Max.   :0.9910   Max.   :0.999992  
summary(trainingData$y)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
  3.556  10.756  14.556  14.416  17.970  28.382 
# Examine predictors and response
featurePlot(
  x = trainingData$x,
  y = trainingData$y
)

folds <- createFolds(
  trainingData$y,
  k = 10,
  returnTrain = TRUE
)

control <- trainControl(
  method = "cv",
  number = 10,
  index = folds
)
# tune several models on these data
set.seed(200)

knnModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "knn",
  preProcess = c("center", "scale"),
  trControl = control,
  tuneLength = 10
)

knnModel
k-Nearest Neighbors 

200 samples
 10 predictor

Pre-processing: centered (10), scaled (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 180, 180, 180, 180, 180, 180, ... 
Resampling results across tuning parameters:

  k   RMSE      Rsquared   MAE     
   5  3.288773  0.5889898  2.725944
   7  3.144376  0.6434361  2.577807
   9  3.124264  0.6689980  2.517219
  11  3.077985  0.6915901  2.489324
  13  3.103767  0.7026394  2.506703
  15  3.127439  0.7062228  2.513829
  17  3.149433  0.7114210  2.526318
  19  3.153927  0.7249773  2.528158
  21  3.121909  0.7450302  2.523428
  23  3.171824  0.7389961  2.565383

RMSE was used to select the optimal model using the smallest value.
The final value used for the model was k = 11.
# Predictions
knnPred <- predict(
  knnModel,
  newdata = testData$x
)

# Evaluation
postResample(
  pred = knnPred,
  obs = testData$y
)
     RMSE  Rsquared       MAE 
3.1222641 0.6690472 2.4963650 
set.seed(200)

folds <- createFolds(
  trainingData$y,
  k = 10,
  returnTrain = TRUE
)

control <- trainControl(
  method = "cv",
  number = 10,
  index = folds
)
set.seed(200)

marsModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "earth",
  trControl = control,
  tuneGrid = expand.grid(
    nprune = 2:20,
    degree = 1:2
  )
)

marsModel
Multivariate Adaptive Regression Spline 

200 samples
 10 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 180, 180, 180, 180, 180, 180, ... 
Resampling results across tuning parameters:

  degree  nprune  RMSE      Rsquared   MAE      
  1        2      4.432989  0.2226534  3.6555837
  1        3      3.765358  0.4460270  3.0673378
  1        4      2.614445  0.7244605  2.1232450
  1        5      2.386299  0.7694607  1.9272623
  1        6      2.289910  0.7937064  1.8181594
  1        7      1.762028  0.8762780  1.3886668
  1        8      1.648775  0.8846873  1.2789524
  1        9      1.642300  0.8885518  1.2893087
  1       10      1.594610  0.8955287  1.2724429
  1       11      1.577498  0.8967307  1.2622284
  1       12      1.571859  0.8969154  1.2395206
  1       13      1.586838  0.8953683  1.2503627
  1       14      1.588146  0.8956292  1.2541297
  1       15      1.588146  0.8956292  1.2541297
  1       16      1.588146  0.8956292  1.2541297
  1       17      1.588146  0.8956292  1.2541297
  1       18      1.588146  0.8956292  1.2541297
  1       19      1.588146  0.8956292  1.2541297
  1       20      1.588146  0.8956292  1.2541297
  2        2      4.432989  0.2226534  3.6555837
  2        3      3.765756  0.4461630  3.0670826
  2        4      2.614662  0.7243956  2.1234522
  2        5      2.306530  0.7854392  1.8535277
  2        6      2.300339  0.7910586  1.7901324
  2        7      1.742326  0.8784813  1.4008423
  2        8      1.559211  0.9001201  1.2275562
  2        9      1.474509  0.9094802  1.1501028
  2       10      1.373899  0.9211506  1.1027803
  2       11      1.285318  0.9350943  1.0306267
  2       12      1.232765  0.9422446  0.9784531
  2       13      1.201863  0.9440998  0.9454771
  2       14      1.250165  0.9407303  0.9771774
  2       15      1.266927  0.9399250  0.9795399
  2       16      1.268891  0.9398187  0.9866665
  2       17      1.267066  0.9401562  0.9869553
  2       18      1.267066  0.9401562  0.9869553
  2       19      1.267066  0.9401562  0.9869553
  2       20      1.267066  0.9401562  0.9869553

RMSE was used to select the optimal model using the smallest value.
The final values used for the model were nprune = 13 and degree = 2.
# Predictions
marsPred <- predict(
  marsModel,
  newdata = testData$x
)

# Evaluation
postResample(
  pred = marsPred,
  obs = testData$y
)
     RMSE  Rsquared       MAE 
1.2803060 0.9335241 1.0168673 
# Variable importance
varImp(marsModel)
earth variable importance

   Overall
V1  100.00
V4   75.33
V2   48.88
V5   15.63
V3    0.00
# Plot variable importance
plot(varImp(marsModel))

# MARS model summary
summary(marsModel$finalModel)
Call: earth(x=data.frame[200,10], y=c(18.46,16.1,17...), keepxy=TRUE, degree=2,
            nprune=13)

                                coefficients
(Intercept)                        22.050690
h(0.621722-V1)                    -15.001651
h(V1-0.621722)                     10.878737
h(0.601063-V2)                    -18.830135
h(0.447442-V3)                      9.940077
h(V3-0.606015)                     12.999390
h(0.734892-V4)                     -9.877554
h(V4-0.734892)                     10.414930
h(0.850094-V5)                     -5.604897
h(V1-0.621722) * h(V2-0.295997)   -43.245766
h(0.649253-V1) * h(0.601063-V2)    26.218297

Selected 11 of 18 terms, and 5 of 10 predictors (nprune=13)
Termination condition: Reached nk 21
Importance: V1, V4, V2, V5, V3, V6-unused, V7-unused, V8-unused, V9-unused, ...
Number of terms at each degree of interaction: 1 8 2
GCV 1.747495    RSS 264.5358    GRSq 0.929051    RSq 0.9457576
# SVM 
set.seed(200)

svmLinearModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "svmLinear",
  preProcess = c("center", "scale"),
  trControl = control,
  tuneLength = 10
)

svmLinearModel
Support Vector Machines with Linear Kernel 

200 samples
 10 predictor

Pre-processing: centered (10), scaled (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 180, 180, 180, 180, 180, 180, ... 
Resampling results:

  RMSE      Rsquared  MAE    
  2.432316  0.765497  1.96225

Tuning parameter 'C' was held constant at a value of 1
# Predictions
svmLinearPred <- predict(
  svmLinearModel,
  newdata = testData$x
)

postResample(
  pred = svmLinearPred,
  obs = testData$y
)
     RMSE  Rsquared       MAE 
2.7633860 0.6973384 2.0970616 
set.seed(200)

svmRadialModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "svmRadial",
  preProcess = c("center", "scale"),
  trControl = control,
  tuneLength = 10
)

svmRadialModel
Support Vector Machines with Radial Basis Function Kernel 

200 samples
 10 predictor

Pre-processing: centered (10), scaled (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 180, 180, 180, 180, 180, 180, ... 
Resampling results across tuning parameters:

  C       RMSE      Rsquared   MAE     
    0.25  2.502240  0.7991647  1.996420
    0.50  2.251774  0.8122993  1.775708
    1.00  2.071314  0.8353324  1.624308
    2.00  1.954727  0.8525581  1.515420
    4.00  1.884376  0.8615784  1.470561
    8.00  1.861357  0.8643203  1.475030
   16.00  1.852625  0.8668207  1.476040
   32.00  1.852625  0.8668207  1.476040
   64.00  1.852625  0.8668207  1.476040
  128.00  1.852625  0.8668207  1.476040

Tuning parameter 'sigma' was held constant at a value of 0.06299324
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were sigma = 0.06299324 and C = 16.
# Predictions
svmRadialPred <- predict(
  svmRadialModel,
  newdata = testData$x
)

postResample(
  pred = svmRadialPred,
  obs = testData$y
)
     RMSE  Rsquared       MAE 
2.0736997 0.8256573 1.5751967 
#Neural Networks
set.seed(200)

nnetModel <- train(
  x = trainingData$x,
  y = trainingData$y,
  method = "nnet",
  preProcess = c("center", "scale"),
  trControl = control,
  tuneGrid = expand.grid(
    size = c(1, 3, 5, 7, 10),
    decay = c(0, 0.001, 0.01, 0.1)
  ),
  linout = TRUE,
  trace = FALSE,
  maxit = 500,
  MaxNWts = 2000
)

nnetModel
Neural Network 

200 samples
 10 predictor

Pre-processing: centered (10), scaled (10) 
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 180, 180, 180, 180, 180, 180, ... 
Resampling results across tuning parameters:

  size  decay  RMSE      Rsquared   MAE     
   1    0.000  2.416185  0.7679397  1.890174
   1    0.001  2.414178  0.7683682  1.888124
   1    0.010  2.412928  0.7686759  1.884140
   1    0.100  2.423494  0.7672732  1.888849
   3    0.000  2.441179  0.7796700  1.968065
   3    0.001  2.467861  0.7599315  1.992184
   3    0.010  2.483551  0.7585101  1.981780
   3    0.100  2.425833  0.7733648  1.928426
   5    0.000  2.864127  0.7149238  2.336369
   5    0.001  2.619389  0.7549897  2.046358
   5    0.010  2.577369  0.7424200  2.026045
   5    0.100  2.515321  0.7522710  2.007962
   7    0.000  9.717315  0.4612940  4.641975
   7    0.001  3.342916  0.6048640  2.567427
   7    0.010  2.944651  0.7018668  2.257427
   7    0.100  2.834740  0.7049757  2.232351
  10    0.000  3.623267  0.5962795  2.879838
  10    0.001  3.633105  0.5611956  2.923946
  10    0.010  3.465451  0.5926746  2.834971
  10    0.100  3.001043  0.6955901  2.482799

RMSE was used to select the optimal model using the smallest value.
The final values used for the model were size = 1 and decay = 0.01.
# Predictions
nnetPred <- predict(
  nnetModel,
  newdata = testData$x
)

postResample(
  pred = nnetPred,
  obs = testData$y
)
     RMSE  Rsquared       MAE 
2.6433534 0.7194358 2.0232768 

##Compare all five models

# Collect all predictions
predictions <- list(
  KNN = knnPred,
  MARS = marsPred,
  SVM_Linear = svmLinearPred,
  SVM_Radial = svmRadialPred,
  Neural_Network = nnetPred
 
)

# Calculate evaluation metrics
model_results <- lapply(
  predictions,
  function(pred) {
    postResample(
      pred = pred,
      obs = testData$y
    )
  }
)

# Convert results to a data frame
comparison <- bind_rows(
  model_results,
  .id = "Model"
) |>
  arrange(RMSE)

comparison
# A tibble: 5 × 4
  Model           RMSE Rsquared   MAE
  <chr>          <dbl>    <dbl> <dbl>
1 MARS            1.28    0.934  1.02
2 SVM_Radial      2.07    0.826  1.58
3 Neural_Network  2.64    0.719  2.02
4 SVM_Linear      2.76    0.697  2.10
5 KNN             3.12    0.669  2.50

##Visualize the models performance

ggplot(
  comparison,
  aes(
    x = reorder(Model, RMSE),
    y = RMSE,
    fill = Model
  )
) +
  geom_col() +
  coord_flip() +
  labs(
    title = "RMSE Comparison Across Regression Models",
    x = "Model",
    y = "Test RMSE"
  ) +
  theme_minimal() +
  theme(legend.position = "none")

# Visualize the R-square 
ggplot(
  comparison,
  aes(
    x = reorder(Model, Rsquared),
    y = Rsquared,
    fill = Model
  )
) +
  geom_col() +
  coord_flip() +
  labs(
    title = "R-squared Comparison Across Models",
    x = "Model",
    y = "Test R-squared"
  ) +
  theme_minimal() +
  theme(legend.position = "none")

#Comapre the predicted vs the actual
predicted_data <- bind_rows(
  lapply(
    names(predictions),
    function(name) {
      tibble(
        Model = name,
        Actual = testData$y,
        Predicted = predictions[[name]]
      )
    }
  )
)

ggplot(
  predicted_data,
  aes(x = Actual, y = Predicted)
) +
  geom_point(alpha = 0.2) +
  geom_abline(
    slope = 1,
    intercept = 0,
    color = "red",
    linewidth = 1
  ) +
  facet_wrap(~ Model, scales = "fixed") +
  labs(
    title = "Actual vs Predicted Values",
    x = "Actual Values",
    y = "Predicted Values"
  ) +
  theme_minimal()

# Best Model
best_model <- comparison |>
  slice_min(
    order_by = RMSE,
    n = 1,
    with_ties = FALSE
  )

best_model
# A tibble: 1 × 4
  Model  RMSE Rsquared   MAE
  <chr> <dbl>    <dbl> <dbl>
1 MARS   1.28    0.934  1.02

Interpretation

The MARS model performed better than the other five regression models because it achieved the lowest RMSE on the test dataset. This indicates that MARS produced more accurate predictions compared to KNN, linear SVM, radial SVM, Neural Network.

Additionally, the actual versus predicted plot shows that the MARS predictions are closely aligned with the actual values, indicating a good model fit. Since the Friedman dataset contains nonlinear relationships and interactions between predictors, MARS can effectively capture these patterns using piecewise regression functions.

Based on the RMSE results and the actual versus predicted plot, MARS is the best-performing model among the six models evaluated in this analysis.

##Exercise 7.5

Exercise 6.3 describes data for a chemical manufacturing process. Use the same data imputation, data splitting, and pre-processing steps as before and train several nonlinear regression models.

library(tidyverse)
library(AppliedPredictiveModeling)
Warning: package 'AppliedPredictiveModeling' was built under R version 4.5.2
library(caret)
library(earth)
library(kernlab)
library(nnet)
Warning: package 'nnet' was built under R version 4.5.2
set.seed(42)

data(ChemicalManufacturingProcess)

# Dataset exploration
head(ChemicalManufacturingProcess)
  Yield BiologicalMaterial01 BiologicalMaterial02 BiologicalMaterial03
1 38.00                 6.25                49.58                56.97
2 42.44                 8.01                60.97                67.48
3 42.03                 8.01                60.97                67.48
4 41.42                 8.01                60.97                67.48
5 42.49                 7.47                63.33                72.25
6 43.57                 6.12                58.36                65.31
  BiologicalMaterial04 BiologicalMaterial05 BiologicalMaterial06
1                12.74                19.51                43.73
2                14.65                19.36                53.14
3                14.65                19.36                53.14
4                14.65                19.36                53.14
5                14.02                17.91                54.66
6                15.17                21.79                51.23
  BiologicalMaterial07 BiologicalMaterial08 BiologicalMaterial09
1                  100                16.66                11.44
2                  100                19.04                12.55
3                  100                19.04                12.55
4                  100                19.04                12.55
5                  100                18.22                12.80
6                  100                18.30                12.13
  BiologicalMaterial10 BiologicalMaterial11 BiologicalMaterial12
1                 3.46               138.09                18.83
2                 3.46               153.67                21.05
3                 3.46               153.67                21.05
4                 3.46               153.67                21.05
5                 3.05               147.61                21.05
6                 3.78               151.88                20.76
  ManufacturingProcess01 ManufacturingProcess02 ManufacturingProcess03
1                     NA                     NA                     NA
2                    0.0                      0                     NA
3                    0.0                      0                     NA
4                    0.0                      0                     NA
5                   10.7                      0                     NA
6                   12.0                      0                     NA
  ManufacturingProcess04 ManufacturingProcess05 ManufacturingProcess06
1                     NA                     NA                     NA
2                    917                 1032.2                  210.0
3                    912                 1003.6                  207.1
4                    911                 1014.6                  213.3
5                    918                 1027.5                  205.7
6                    924                 1016.8                  208.9
  ManufacturingProcess07 ManufacturingProcess08 ManufacturingProcess09
1                     NA                     NA                  43.00
2                    177                    178                  46.57
3                    178                    178                  45.07
4                    177                    177                  44.92
5                    178                    178                  44.96
6                    178                    178                  45.32
  ManufacturingProcess10 ManufacturingProcess11 ManufacturingProcess12
1                     NA                     NA                     NA
2                     NA                     NA                      0
3                     NA                     NA                      0
4                     NA                     NA                      0
5                     NA                     NA                      0
6                     NA                     NA                      0
  ManufacturingProcess13 ManufacturingProcess14 ManufacturingProcess15
1                   35.5                   4898                   6108
2                   34.0                   4869                   6095
3                   34.8                   4878                   6087
4                   34.8                   4897                   6102
5                   34.6                   4992                   6233
6                   34.0                   4985                   6222
  ManufacturingProcess16 ManufacturingProcess17 ManufacturingProcess18
1                   4682                   35.5                   4865
2                   4617                   34.0                   4867
3                   4617                   34.8                   4877
4                   4635                   34.8                   4872
5                   4733                   33.9                   4886
6                   4786                   33.4                   4862
  ManufacturingProcess19 ManufacturingProcess20 ManufacturingProcess21
1                   6049                   4665                    0.0
2                   6097                   4621                    0.0
3                   6078                   4621                    0.0
4                   6073                   4611                    0.0
5                   6102                   4659                   -0.7
6                   6115                   4696                   -0.6
  ManufacturingProcess22 ManufacturingProcess23 ManufacturingProcess24
1                     NA                     NA                     NA
2                      3                      0                      3
3                      4                      1                      4
4                      5                      2                      5
5                      8                      4                     18
6                      9                      1                      1
  ManufacturingProcess25 ManufacturingProcess26 ManufacturingProcess27
1                   4873                   6074                   4685
2                   4869                   6107                   4630
3                   4897                   6116                   4637
4                   4892                   6111                   4630
5                   4930                   6151                   4684
6                   4871                   6128                   4687
  ManufacturingProcess28 ManufacturingProcess29 ManufacturingProcess30
1                   10.7                   21.0                    9.9
2                   11.2                   21.4                    9.9
3                   11.1                   21.3                    9.4
4                   11.1                   21.3                    9.4
5                   11.3                   21.6                    9.0
6                   11.4                   21.7                   10.1
  ManufacturingProcess31 ManufacturingProcess32 ManufacturingProcess33
1                   69.1                    156                     66
2                   68.7                    169                     66
3                   69.3                    173                     66
4                   69.3                    171                     68
5                   69.4                    171                     70
6                   68.2                    173                     70
  ManufacturingProcess34 ManufacturingProcess35 ManufacturingProcess36
1                    2.4                    486                  0.019
2                    2.6                    508                  0.019
3                    2.6                    509                  0.018
4                    2.5                    496                  0.018
5                    2.5                    468                  0.017
6                    2.5                    490                  0.018
  ManufacturingProcess37 ManufacturingProcess38 ManufacturingProcess39
1                    0.5                      3                    7.2
2                    2.0                      2                    7.2
3                    0.7                      2                    7.2
4                    1.2                      2                    7.2
5                    0.2                      2                    7.3
6                    0.4                      2                    7.2
  ManufacturingProcess40 ManufacturingProcess41 ManufacturingProcess42
1                     NA                     NA                   11.6
2                    0.1                   0.15                   11.1
3                    0.0                   0.00                   12.0
4                    0.0                   0.00                   10.6
5                    0.0                   0.00                   11.0
6                    0.0                   0.00                   11.5
  ManufacturingProcess43 ManufacturingProcess44 ManufacturingProcess45
1                    3.0                    1.8                    2.4
2                    0.9                    1.9                    2.2
3                    1.0                    1.8                    2.3
4                    1.1                    1.8                    2.1
5                    1.1                    1.7                    2.1
6                    2.2                    1.8                    2.0
dim(ChemicalManufacturingProcess)
[1] 176  58
str(ChemicalManufacturingProcess)
'data.frame':   176 obs. of  58 variables:
 $ Yield                 : num  38 42.4 42 41.4 42.5 ...
 $ BiologicalMaterial01  : num  6.25 8.01 8.01 8.01 7.47 6.12 7.48 6.94 6.94 6.94 ...
 $ BiologicalMaterial02  : num  49.6 61 61 61 63.3 ...
 $ BiologicalMaterial03  : num  57 67.5 67.5 67.5 72.2 ...
 $ BiologicalMaterial04  : num  12.7 14.7 14.7 14.7 14 ...
 $ BiologicalMaterial05  : num  19.5 19.4 19.4 19.4 17.9 ...
 $ BiologicalMaterial06  : num  43.7 53.1 53.1 53.1 54.7 ...
 $ BiologicalMaterial07  : num  100 100 100 100 100 100 100 100 100 100 ...
 $ BiologicalMaterial08  : num  16.7 19 19 19 18.2 ...
 $ BiologicalMaterial09  : num  11.4 12.6 12.6 12.6 12.8 ...
 $ BiologicalMaterial10  : num  3.46 3.46 3.46 3.46 3.05 3.78 3.04 3.85 3.85 3.85 ...
 $ BiologicalMaterial11  : num  138 154 154 154 148 ...
 $ BiologicalMaterial12  : num  18.8 21.1 21.1 21.1 21.1 ...
 $ ManufacturingProcess01: num  NA 0 0 0 10.7 12 11.5 12 12 12 ...
 $ ManufacturingProcess02: num  NA 0 0 0 0 0 0 0 0 0 ...
 $ ManufacturingProcess03: num  NA NA NA NA NA NA 1.56 1.55 1.56 1.55 ...
 $ ManufacturingProcess04: num  NA 917 912 911 918 924 933 929 928 938 ...
 $ ManufacturingProcess05: num  NA 1032 1004 1015 1028 ...
 $ ManufacturingProcess06: num  NA 210 207 213 206 ...
 $ ManufacturingProcess07: num  NA 177 178 177 178 178 177 178 177 177 ...
 $ ManufacturingProcess08: num  NA 178 178 177 178 178 178 178 177 177 ...
 $ ManufacturingProcess09: num  43 46.6 45.1 44.9 45 ...
 $ ManufacturingProcess10: num  NA NA NA NA NA NA 11.6 10.2 9.7 10.1 ...
 $ ManufacturingProcess11: num  NA NA NA NA NA NA 11.5 11.3 11.1 10.2 ...
 $ ManufacturingProcess12: num  NA 0 0 0 0 0 0 0 0 0 ...
 $ ManufacturingProcess13: num  35.5 34 34.8 34.8 34.6 34 32.4 33.6 33.9 34.3 ...
 $ ManufacturingProcess14: num  4898 4869 4878 4897 4992 ...
 $ ManufacturingProcess15: num  6108 6095 6087 6102 6233 ...
 $ ManufacturingProcess16: num  4682 4617 4617 4635 4733 ...
 $ ManufacturingProcess17: num  35.5 34 34.8 34.8 33.9 33.4 33.8 33.6 33.9 35.3 ...
 $ ManufacturingProcess18: num  4865 4867 4877 4872 4886 ...
 $ ManufacturingProcess19: num  6049 6097 6078 6073 6102 ...
 $ ManufacturingProcess20: num  4665 4621 4621 4611 4659 ...
 $ ManufacturingProcess21: num  0 0 0 0 -0.7 -0.6 1.4 0 0 1 ...
 $ ManufacturingProcess22: num  NA 3 4 5 8 9 1 2 3 4 ...
 $ ManufacturingProcess23: num  NA 0 1 2 4 1 1 2 3 1 ...
 $ ManufacturingProcess24: num  NA 3 4 5 18 1 1 2 3 4 ...
 $ ManufacturingProcess25: num  4873 4869 4897 4892 4930 ...
 $ ManufacturingProcess26: num  6074 6107 6116 6111 6151 ...
 $ ManufacturingProcess27: num  4685 4630 4637 4630 4684 ...
 $ ManufacturingProcess28: num  10.7 11.2 11.1 11.1 11.3 11.4 11.2 11.1 11.3 11.4 ...
 $ ManufacturingProcess29: num  21 21.4 21.3 21.3 21.6 21.7 21.2 21.2 21.5 21.7 ...
 $ ManufacturingProcess30: num  9.9 9.9 9.4 9.4 9 10.1 11.2 10.9 10.5 9.8 ...
 $ ManufacturingProcess31: num  69.1 68.7 69.3 69.3 69.4 68.2 67.6 67.9 68 68.5 ...
 $ ManufacturingProcess32: num  156 169 173 171 171 173 159 161 160 164 ...
 $ ManufacturingProcess33: num  66 66 66 68 70 70 65 65 65 66 ...
 $ ManufacturingProcess34: num  2.4 2.6 2.6 2.5 2.5 2.5 2.5 2.5 2.5 2.5 ...
 $ ManufacturingProcess35: num  486 508 509 496 468 490 475 478 491 488 ...
 $ ManufacturingProcess36: num  0.019 0.019 0.018 0.018 0.017 0.018 0.019 0.019 0.019 0.019 ...
 $ ManufacturingProcess37: num  0.5 2 0.7 1.2 0.2 0.4 0.8 1 1.2 1.8 ...
 $ ManufacturingProcess38: num  3 2 2 2 2 2 2 2 3 3 ...
 $ ManufacturingProcess39: num  7.2 7.2 7.2 7.2 7.3 7.2 7.3 7.3 7.4 7.1 ...
 $ ManufacturingProcess40: num  NA 0.1 0 0 0 0 0 0 0 0 ...
 $ ManufacturingProcess41: num  NA 0.15 0 0 0 0 0 0 0 0 ...
 $ ManufacturingProcess42: num  11.6 11.1 12 10.6 11 11.5 11.7 11.4 11.4 11.3 ...
 $ ManufacturingProcess43: num  3 0.9 1 1.1 1.1 2.2 0.7 0.8 0.9 0.8 ...
 $ ManufacturingProcess44: num  1.8 1.9 1.8 1.8 1.7 1.8 2 2 1.9 1.9 ...
 $ ManufacturingProcess45: num  2.4 2.2 2.3 2.1 2.1 2 2.2 2.2 2.1 2.4 ...
summary(ChemicalManufacturingProcess)
     Yield       BiologicalMaterial01 BiologicalMaterial02 BiologicalMaterial03
 Min.   :35.25   Min.   :4.580        Min.   :46.87        Min.   :56.97       
 1st Qu.:38.75   1st Qu.:5.978        1st Qu.:52.68        1st Qu.:64.98       
 Median :39.97   Median :6.305        Median :55.09        Median :67.22       
 Mean   :40.18   Mean   :6.411        Mean   :55.69        Mean   :67.70       
 3rd Qu.:41.48   3rd Qu.:6.870        3rd Qu.:58.74        3rd Qu.:70.43       
 Max.   :46.34   Max.   :8.810        Max.   :64.75        Max.   :78.25       
                                                                               
 BiologicalMaterial04 BiologicalMaterial05 BiologicalMaterial06
 Min.   : 9.38        Min.   :13.24        Min.   :40.60       
 1st Qu.:11.24        1st Qu.:17.23        1st Qu.:46.05       
 Median :12.10        Median :18.49        Median :48.46       
 Mean   :12.35        Mean   :18.60        Mean   :48.91       
 3rd Qu.:13.22        3rd Qu.:19.90        3rd Qu.:51.34       
 Max.   :23.09        Max.   :24.85        Max.   :59.38       
                                                               
 BiologicalMaterial07 BiologicalMaterial08 BiologicalMaterial09
 Min.   :100.0        Min.   :15.88        Min.   :11.44       
 1st Qu.:100.0        1st Qu.:17.06        1st Qu.:12.60       
 Median :100.0        Median :17.51        Median :12.84       
 Mean   :100.0        Mean   :17.49        Mean   :12.85       
 3rd Qu.:100.0        3rd Qu.:17.88        3rd Qu.:13.13       
 Max.   :100.8        Max.   :19.14        Max.   :14.08       
                                                               
 BiologicalMaterial10 BiologicalMaterial11 BiologicalMaterial12
 Min.   :1.770        Min.   :135.8        Min.   :18.35       
 1st Qu.:2.460        1st Qu.:143.8        1st Qu.:19.73       
 Median :2.710        Median :146.1        Median :20.12       
 Mean   :2.801        Mean   :147.0        Mean   :20.20       
 3rd Qu.:2.990        3rd Qu.:149.6        3rd Qu.:20.75       
 Max.   :6.870        Max.   :158.7        Max.   :22.21       
                                                               
 ManufacturingProcess01 ManufacturingProcess02 ManufacturingProcess03
 Min.   : 0.00          Min.   : 0.00          Min.   :1.47          
 1st Qu.:10.80          1st Qu.:19.30          1st Qu.:1.53          
 Median :11.40          Median :21.00          Median :1.54          
 Mean   :11.21          Mean   :16.68          Mean   :1.54          
 3rd Qu.:12.15          3rd Qu.:21.50          3rd Qu.:1.55          
 Max.   :14.10          Max.   :22.50          Max.   :1.60          
 NA's   :1              NA's   :3              NA's   :15            
 ManufacturingProcess04 ManufacturingProcess05 ManufacturingProcess06
 Min.   :911.0          Min.   : 923.0         Min.   :203.0         
 1st Qu.:928.0          1st Qu.: 986.8         1st Qu.:205.7         
 Median :934.0          Median : 999.2         Median :206.8         
 Mean   :931.9          Mean   :1001.7         Mean   :207.4         
 3rd Qu.:936.0          3rd Qu.:1008.9         3rd Qu.:208.7         
 Max.   :946.0          Max.   :1175.3         Max.   :227.4         
 NA's   :1              NA's   :1              NA's   :2             
 ManufacturingProcess07 ManufacturingProcess08 ManufacturingProcess09
 Min.   :177.0          Min.   :177.0          Min.   :38.89         
 1st Qu.:177.0          1st Qu.:177.0          1st Qu.:44.89         
 Median :177.0          Median :178.0          Median :45.73         
 Mean   :177.5          Mean   :177.6          Mean   :45.66         
 3rd Qu.:178.0          3rd Qu.:178.0          3rd Qu.:46.52         
 Max.   :178.0          Max.   :178.0          Max.   :49.36         
 NA's   :1              NA's   :1                                    
 ManufacturingProcess10 ManufacturingProcess11 ManufacturingProcess12
 Min.   : 7.500         Min.   : 7.500         Min.   :   0.0        
 1st Qu.: 8.700         1st Qu.: 9.000         1st Qu.:   0.0        
 Median : 9.100         Median : 9.400         Median :   0.0        
 Mean   : 9.179         Mean   : 9.386         Mean   : 857.8        
 3rd Qu.: 9.550         3rd Qu.: 9.900         3rd Qu.:   0.0        
 Max.   :11.600         Max.   :11.500         Max.   :4549.0        
 NA's   :9              NA's   :10             NA's   :1             
 ManufacturingProcess13 ManufacturingProcess14 ManufacturingProcess15
 Min.   :32.10          Min.   :4701           Min.   :5904          
 1st Qu.:33.90          1st Qu.:4828           1st Qu.:6010          
 Median :34.60          Median :4856           Median :6032          
 Mean   :34.51          Mean   :4854           Mean   :6039          
 3rd Qu.:35.20          3rd Qu.:4882           3rd Qu.:6061          
 Max.   :38.60          Max.   :5055           Max.   :6233          
                        NA's   :1                                    
 ManufacturingProcess16 ManufacturingProcess17 ManufacturingProcess18
 Min.   :   0           Min.   :31.30          Min.   :   0          
 1st Qu.:4561           1st Qu.:33.50          1st Qu.:4813          
 Median :4588           Median :34.40          Median :4835          
 Mean   :4566           Mean   :34.34          Mean   :4810          
 3rd Qu.:4619           3rd Qu.:35.10          3rd Qu.:4862          
 Max.   :4852           Max.   :40.00          Max.   :4971          
                                                                     
 ManufacturingProcess19 ManufacturingProcess20 ManufacturingProcess21
 Min.   :5890           Min.   :   0           Min.   :-1.8000       
 1st Qu.:6001           1st Qu.:4553           1st Qu.:-0.6000       
 Median :6022           Median :4582           Median :-0.3000       
 Mean   :6028           Mean   :4556           Mean   :-0.1642       
 3rd Qu.:6050           3rd Qu.:4610           3rd Qu.: 0.0000       
 Max.   :6146           Max.   :4759           Max.   : 3.6000       
                                                                     
 ManufacturingProcess22 ManufacturingProcess23 ManufacturingProcess24
 Min.   : 0.000         Min.   :0.000          Min.   : 0.000        
 1st Qu.: 3.000         1st Qu.:2.000          1st Qu.: 4.000        
 Median : 5.000         Median :3.000          Median : 8.000        
 Mean   : 5.406         Mean   :3.017          Mean   : 8.834        
 3rd Qu.: 8.000         3rd Qu.:4.000          3rd Qu.:14.000        
 Max.   :12.000         Max.   :6.000          Max.   :23.000        
 NA's   :1              NA's   :1              NA's   :1             
 ManufacturingProcess25 ManufacturingProcess26 ManufacturingProcess27
 Min.   :   0           Min.   :   0           Min.   :   0          
 1st Qu.:4832           1st Qu.:6020           1st Qu.:4560          
 Median :4855           Median :6047           Median :4587          
 Mean   :4828           Mean   :6016           Mean   :4563          
 3rd Qu.:4877           3rd Qu.:6070           3rd Qu.:4609          
 Max.   :4990           Max.   :6161           Max.   :4710          
 NA's   :5              NA's   :5              NA's   :5             
 ManufacturingProcess28 ManufacturingProcess29 ManufacturingProcess30
 Min.   : 0.000         Min.   : 0.00          Min.   : 0.000        
 1st Qu.: 0.000         1st Qu.:19.70          1st Qu.: 8.800        
 Median :10.400         Median :19.90          Median : 9.100        
 Mean   : 6.592         Mean   :20.01          Mean   : 9.161        
 3rd Qu.:10.750         3rd Qu.:20.40          3rd Qu.: 9.700        
 Max.   :11.500         Max.   :22.00          Max.   :11.200        
 NA's   :5              NA's   :5              NA's   :5             
 ManufacturingProcess31 ManufacturingProcess32 ManufacturingProcess33
 Min.   : 0.00          Min.   :143.0          Min.   :56.00         
 1st Qu.:70.10          1st Qu.:155.0          1st Qu.:62.00         
 Median :70.80          Median :158.0          Median :64.00         
 Mean   :70.18          Mean   :158.5          Mean   :63.54         
 3rd Qu.:71.40          3rd Qu.:162.0          3rd Qu.:65.00         
 Max.   :72.50          Max.   :173.0          Max.   :70.00         
 NA's   :5                                     NA's   :5             
 ManufacturingProcess34 ManufacturingProcess35 ManufacturingProcess36
 Min.   :2.300          Min.   :463.0          Min.   :0.01700       
 1st Qu.:2.500          1st Qu.:490.0          1st Qu.:0.01900       
 Median :2.500          Median :495.0          Median :0.02000       
 Mean   :2.494          Mean   :495.6          Mean   :0.01957       
 3rd Qu.:2.500          3rd Qu.:501.5          3rd Qu.:0.02000       
 Max.   :2.600          Max.   :522.0          Max.   :0.02200       
 NA's   :5              NA's   :5              NA's   :5             
 ManufacturingProcess37 ManufacturingProcess38 ManufacturingProcess39
 Min.   :0.000          Min.   :0.000          Min.   :0.000         
 1st Qu.:0.700          1st Qu.:2.000          1st Qu.:7.100         
 Median :1.000          Median :3.000          Median :7.200         
 Mean   :1.014          Mean   :2.534          Mean   :6.851         
 3rd Qu.:1.300          3rd Qu.:3.000          3rd Qu.:7.300         
 Max.   :2.300          Max.   :3.000          Max.   :7.500         
                                                                     
 ManufacturingProcess40 ManufacturingProcess41 ManufacturingProcess42
 Min.   :0.00000        Min.   :0.00000        Min.   : 0.00         
 1st Qu.:0.00000        1st Qu.:0.00000        1st Qu.:11.40         
 Median :0.00000        Median :0.00000        Median :11.60         
 Mean   :0.01771        Mean   :0.02371        Mean   :11.21         
 3rd Qu.:0.00000        3rd Qu.:0.00000        3rd Qu.:11.70         
 Max.   :0.10000        Max.   :0.20000        Max.   :12.10         
 NA's   :1              NA's   :1                                    
 ManufacturingProcess43 ManufacturingProcess44 ManufacturingProcess45
 Min.   : 0.0000        Min.   :0.000          Min.   :0.000         
 1st Qu.: 0.6000        1st Qu.:1.800          1st Qu.:2.100         
 Median : 0.8000        Median :1.900          Median :2.200         
 Mean   : 0.9119        Mean   :1.805          Mean   :2.138         
 3rd Qu.: 1.0250        3rd Qu.:1.900          3rd Qu.:2.300         
 Max.   :11.0000        Max.   :2.100          Max.   :2.600         
                                                                     
# Missing values
sum(is.na(ChemicalManufacturingProcess))
[1] 106
colSums(is.na(ChemicalManufacturingProcess))
                 Yield   BiologicalMaterial01   BiologicalMaterial02 
                     0                      0                      0 
  BiologicalMaterial03   BiologicalMaterial04   BiologicalMaterial05 
                     0                      0                      0 
  BiologicalMaterial06   BiologicalMaterial07   BiologicalMaterial08 
                     0                      0                      0 
  BiologicalMaterial09   BiologicalMaterial10   BiologicalMaterial11 
                     0                      0                      0 
  BiologicalMaterial12 ManufacturingProcess01 ManufacturingProcess02 
                     0                      1                      3 
ManufacturingProcess03 ManufacturingProcess04 ManufacturingProcess05 
                    15                      1                      1 
ManufacturingProcess06 ManufacturingProcess07 ManufacturingProcess08 
                     2                      1                      1 
ManufacturingProcess09 ManufacturingProcess10 ManufacturingProcess11 
                     0                      9                     10 
ManufacturingProcess12 ManufacturingProcess13 ManufacturingProcess14 
                     1                      0                      1 
ManufacturingProcess15 ManufacturingProcess16 ManufacturingProcess17 
                     0                      0                      0 
ManufacturingProcess18 ManufacturingProcess19 ManufacturingProcess20 
                     0                      0                      0 
ManufacturingProcess21 ManufacturingProcess22 ManufacturingProcess23 
                     0                      1                      1 
ManufacturingProcess24 ManufacturingProcess25 ManufacturingProcess26 
                     1                      5                      5 
ManufacturingProcess27 ManufacturingProcess28 ManufacturingProcess29 
                     5                      5                      5 
ManufacturingProcess30 ManufacturingProcess31 ManufacturingProcess32 
                     5                      5                      0 
ManufacturingProcess33 ManufacturingProcess34 ManufacturingProcess35 
                     5                      5                      5 
ManufacturingProcess36 ManufacturingProcess37 ManufacturingProcess38 
                     5                      0                      0 
ManufacturingProcess39 ManufacturingProcess40 ManufacturingProcess41 
                     0                      1                      1 
ManufacturingProcess42 ManufacturingProcess43 ManufacturingProcess44 
                     0                      0                      0 
ManufacturingProcess45 
                     0 

##Exploratory data analysis

yield_plot <- ggplot(
  ChemicalManufacturingProcess,
  aes(x = Yield)
) +
  geom_histogram(
    bins = 20,
    fill = "steelblue",
    color = "black"
  ) +
  labs(
    title = "Distribution of Chemical Manufacturing Yield",
    x = "Yield (%)",
    y = "Frequency"
  ) +
  theme_minimal()

print(yield_plot)

ggsave(
  "MARS_yield_distribution.png",
  yield_plot,
  width = 9,
  height = 5.5,
  dpi = 300
)

# Missing values by predictor
missing_by_variable <- sort(
  colSums(is.na(ChemicalManufacturingProcess)),
  decreasing = TRUE
)

head(missing_by_variable, 15)
ManufacturingProcess03 ManufacturingProcess11 ManufacturingProcess10 
                    15                     10                      9 
ManufacturingProcess25 ManufacturingProcess26 ManufacturingProcess27 
                     5                      5                      5 
ManufacturingProcess28 ManufacturingProcess29 ManufacturingProcess30 
                     5                      5                      5 
ManufacturingProcess31 ManufacturingProcess33 ManufacturingProcess34 
                     5                      5                      5 
ManufacturingProcess35 ManufacturingProcess36 ManufacturingProcess02 
                     5                      5                      3 
# Remove observations with missing response
model_data <- ChemicalManufacturingProcess |>
  filter(!is.na(Yield))

set.seed(42)

train_index <- createDataPartition(
  model_data$Yield,
  p = 0.80,
  list = FALSE
)

train_data <- model_data[train_index, , drop = FALSE]
test_data <- model_data[-train_index, , drop = FALSE]

# Preprocessing
preprocess_model <- preProcess(
  train_data |> select(-Yield),
  method = c("medianImpute", "center", "scale")
)

x_train <- predict(
  preprocess_model,
  train_data |> select(-Yield)
)

x_test <- predict(
  preprocess_model,
  test_data |> select(-Yield)
)

train_processed <- bind_cols(
  tibble(Yield = train_data$Yield),
  as_tibble(x_train)
)

test_processed <- bind_cols(
  tibble(Yield = test_data$Yield),
  as_tibble(x_test)
)

# Check results
dim(train_processed)
[1] 144  58
dim(test_processed)
[1] 32 58
sum(is.na(train_processed))
[1] 0
sum(is.na(test_processed))
[1] 0
set.seed(42)

folds <- createFolds(
  train_processed$Yield,
  k = 10,
  returnTrain = TRUE
)

control <- trainControl(
  method = "cv",
  number = 10,
  index = folds
)
set.seed(42)

knnModel <- train(
  Yield ~ .,
  data = train_processed,
  method = "knn",
  trControl = control,
  tuneLength = 10
)

knnModel
k-Nearest Neighbors 

144 samples
 57 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 128, 129, 130, 130, 128, 130, ... 
Resampling results across tuning parameters:

  k   RMSE      Rsquared   MAE     
   5  1.244040  0.5408253  1.023276
   7  1.312593  0.4873704  1.080397
   9  1.364263  0.4542507  1.124819
  11  1.359890  0.4711554  1.126593
  13  1.365952  0.4563463  1.127534
  15  1.372466  0.4542979  1.137705
  17  1.394964  0.4353030  1.150754
  19  1.411442  0.4158501  1.161529
  21  1.410953  0.4260760  1.161424
  23  1.432978  0.4096841  1.172731

RMSE was used to select the optimal model using the smallest value.
The final value used for the model was k = 5.
# Predictions
knnPred <- predict(
  knnModel,
  newdata = test_processed
)

# Evaluation
postResample(
  pred = knnPred,
  obs = test_processed$Yield
)
     RMSE  Rsquared       MAE 
1.3738785 0.5488151 0.9719375 
# Mars model
set.seed(42)

marsModel <- train(
  Yield ~ .,
  data = train_processed,
  method = "earth",
  trControl = control,
  tuneGrid = expand.grid(
    nprune = 2:20,
    degree = 1:2
  )
)

marsModel
Multivariate Adaptive Regression Spline 

144 samples
 57 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 128, 129, 130, 130, 128, 130, ... 
Resampling results across tuning parameters:

  degree  nprune  RMSE      Rsquared   MAE      
  1        2      1.430727  0.3826690  1.1191546
  1        3      1.355812  0.4697185  1.0828022
  1        4      1.226361  0.5543103  0.9786656
  1        5      1.278567  0.5265214  1.0103382
  1        6      1.222883  0.5736879  0.9731662
  1        7      1.224336  0.5780914  0.9711594
  1        8      3.393737  0.5009559  1.6369648
  1        9      3.519738  0.5018088  1.6613063
  1       10      3.977436  0.4640684  1.8220879
  1       11      3.784288  0.4801820  1.7528770
  1       12      3.798681  0.4757895  1.7558176
  1       13      5.434205  0.4675057  2.6895487
  1       14      7.654754  0.4117972  3.3715988
  1       15      7.792785  0.4212262  3.4269729
  1       16      8.694713  0.4205350  3.6837728
  1       17      8.688167  0.4264467  3.6787547
  1       18      8.688167  0.4264467  3.6787547
  1       19      8.688167  0.4264467  3.6787547
  1       20      8.688167  0.4264467  3.6787547
  2        2      1.430727  0.3826690  1.1191546
  2        3      1.239775  0.5403935  1.0193025
  2        4      1.258777  0.5382232  0.9929899
  2        5      1.205064  0.5710730  0.9642827
  2        6      1.220159  0.5784284  0.9834451
  2        7      1.191374  0.5975614  0.9442587
  2        8      1.164015  0.6123447  0.9309712
  2        9      1.118314  0.6457536  0.8741773
  2       10      1.172970  0.6320988  0.8903833
  2       11      1.156911  0.6408960  0.8925057
  2       12      1.170155  0.6373095  0.9055093
  2       13      1.184116  0.6319821  0.9028183
  2       14      1.169376  0.6391493  0.8933261
  2       15      1.133577  0.6563297  0.8658121
  2       16      1.120200  0.6599094  0.8562704
  2       17      1.122063  0.6588935  0.8569760
  2       18      1.120973  0.6600411  0.8570232
  2       19      1.136961  0.6592668  0.8725969
  2       20      1.142755  0.6595158  0.8769442

RMSE was used to select the optimal model using the smallest value.
The final values used for the model were nprune = 9 and degree = 2.
# Predictions
marsPred <- predict(
  marsModel,
  newdata = test_processed
)

# Evaluation
postResample(
  pred = marsPred,
  obs = test_processed$Yield
)
     RMSE  Rsquared       MAE 
1.4248559 0.5534361 1.0203994 
# Final model
summary(marsModel$finalModel)
Call: earth(x=matrix[144,57], y=c(38,42.44,42.0...), keepxy=TRUE, degree=2,
            nprune=9)

                                                                           coefficients
(Intercept)                                                                   38.046565
h(-1.14897-ManufacturingProcess09)                                            -1.100176
h(ManufacturingProcess09- -1.14897)                                            0.872789
h(ManufacturingProcess32- -1.2965)                                             0.727688
h(ManufacturingProcess01- -0.830301) * h(-0.245576-ManufacturingProcess33)     0.559615
h(ManufacturingProcess04- -0.592305) * h(ManufacturingProcess32- -1.2965)      0.667772
h(ManufacturingProcess04- -0.275671) * h(ManufacturingProcess39-0.0172987)    -3.770619
h(0.273211-ManufacturingProcess06) * h(ManufacturingProcess09- -1.14897)      -0.825138
h(-0.410521-ManufacturingProcess17) * h(ManufacturingProcess39-0.0172987)      7.981566

Selected 9 of 54 terms, and 8 of 57 predictors (nprune=9)
Termination condition: GRSq -10 at 54 terms
Importance: ManufacturingProcess32, ManufacturingProcess09, ...
Number of terms at each degree of interaction: 1 3 5
GCV 0.8822112    RSS 92.68732    GRSq 0.7322299    RSq 0.8018928
# Coefficients
coef(marsModel$finalModel)
                                                               (Intercept) 
                                                                38.0465653 
                                        h(ManufacturingProcess32- -1.2965) 
                                                                 0.7276875 
                                       h(ManufacturingProcess09- -1.14897) 
                                                                 0.8727894 
                                        h(-1.14897-ManufacturingProcess09) 
                                                                -1.1001756 
  h(0.273211-ManufacturingProcess06) * h(ManufacturingProcess09- -1.14897) 
                                                                -0.8251380 
 h(-0.410521-ManufacturingProcess17) * h(ManufacturingProcess39-0.0172987) 
                                                                 7.9815657 
 h(ManufacturingProcess04- -0.592305) * h(ManufacturingProcess32- -1.2965) 
                                                                 0.6677719 
h(ManufacturingProcess04- -0.275671) * h(ManufacturingProcess39-0.0172987) 
                                                                -3.7706190 
h(ManufacturingProcess01- -0.830301) * h(-0.245576-ManufacturingProcess33) 
                                                                 0.5596154 
# Variable importance
mars_importance <- varImp(marsModel)

mars_importance
earth variable importance

                       Overall
ManufacturingProcess32  100.00
ManufacturingProcess09   64.82
ManufacturingProcess06   34.04
ManufacturingProcess17   27.75
ManufacturingProcess39   27.75
ManufacturingProcess04   21.71
ManufacturingProcess01    0.00
ManufacturingProcess33    0.00
plot(mars_importance)

# MARS diagnostic plots
plot(marsModel$finalModel)

# SVM
set.seed(42)

svmLinearModel <- train(
  Yield ~ .,
  data = train_processed,
  method = "svmLinear",
  trControl = control,
  tuneLength = 10
)

svmLinearModel
Support Vector Machines with Linear Kernel 

144 samples
 57 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 128, 129, 130, 130, 128, 130, ... 
Resampling results:

  RMSE      Rsquared   MAE     
  3.713475  0.4705089  1.810233

Tuning parameter 'C' was held constant at a value of 1
# Predictions
svmLinearPred <- predict(
  svmLinearModel,
  newdata = test_processed
)

postResample(
  pred = svmLinearPred,
  obs = test_processed$Yield
)
     RMSE  Rsquared       MAE 
1.2425622 0.6171401 0.9491937 
#SVM Radial
set.seed(42)

svmRadialModel <- train(
  Yield ~ .,
  data = train_processed,
  method = "svmRadial",
  trControl = control,
  tuneLength = 10
)

svmRadialModel
Support Vector Machines with Radial Basis Function Kernel 

144 samples
 57 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 128, 129, 130, 130, 128, 130, ... 
Resampling results across tuning parameters:

  C       RMSE      Rsquared   MAE      
    0.25  1.397369  0.4977627  1.1448942
    0.50  1.278128  0.5399648  1.0482201
    1.00  1.176753  0.5939419  0.9543719
    2.00  1.135787  0.6124316  0.9064516
    4.00  1.127997  0.6215421  0.9030751
    8.00  1.108598  0.6364725  0.8916759
   16.00  1.107539  0.6378048  0.8910765
   32.00  1.107539  0.6378048  0.8910765
   64.00  1.107539  0.6378048  0.8910765
  128.00  1.107539  0.6378048  0.8910765

Tuning parameter 'sigma' was held constant at a value of 0.01489089
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were sigma = 0.01489089 and C = 16.
# Predictions
svmRadialPred <- predict(
  svmRadialModel,
  newdata = test_processed
)

postResample(
  pred = svmRadialPred,
  obs = test_processed$Yield
)
     RMSE  Rsquared       MAE 
1.2884031 0.6073312 0.9405254 
# Neural Networks Model
set.seed(42)

nnetModel <- train(
  Yield ~ .,
  data = train_processed,
  method = "nnet",
  trControl = control,
  tuneGrid = expand.grid(
    size = c(1, 3, 5, 7, 10),
    decay = c(0, 0.001, 0.01, 0.1)
  ),
  linout = TRUE,
  trace = FALSE,
  maxit = 500,
  MaxNWts = 2000
)

nnetModel
Neural Network 

144 samples
 57 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 128, 129, 130, 130, 128, 130, ... 
Resampling results across tuning parameters:

  size  decay  RMSE      Rsquared    MAE     
   1    0.000  1.668605  0.20669650  1.348523
   1    0.001  1.648740  0.35994255  1.334224
   1    0.010  1.752132  0.31605341  1.397902
   1    0.100  1.661771  0.35372605  1.340637
   3    0.000  2.758737  0.20032773  2.125011
   3    0.001  2.653715  0.30761720  1.909724
   3    0.010  2.819903  0.21490586  2.136095
   3    0.100  3.088103  0.24953583  2.201376
   5    0.000  3.830514  0.14943312  2.974661
   5    0.001  3.260636  0.09846871  2.341122
   5    0.010  2.894665  0.31676335  2.187823
   5    0.100  2.767204  0.26741379  1.962434
   7    0.000  4.460707  0.16485850  3.328479
   7    0.001  2.920082  0.28792203  2.141823
   7    0.010  2.878847  0.23615515  2.161221
   7    0.100  3.171473  0.20632423  2.144890
  10    0.000  7.893853  0.06685100  4.971657
  10    0.001  6.268534  0.13864805  4.522672
  10    0.010  4.476432  0.15236623  3.130031
  10    0.100  2.802354  0.25052049  2.091162

RMSE was used to select the optimal model using the smallest value.
The final values used for the model were size = 1 and decay = 0.001.
# Predictions
nnetPred <- predict(
  nnetModel,
  newdata = test_processed
)

postResample(
  pred = nnetPred,
  obs = test_processed$Yield
)
     RMSE  Rsquared       MAE 
1.2259036 0.6448059 0.9553021 

##Compare all five

# Store predictions
predictions <- list(
  KNN = knnPred,
  MARS = marsPred,
  SVM_Linear = svmLinearPred,
  SVM_Radial = svmRadialPred,
  Neural_Network = nnetPred
  
)

# Calculate performance metrics
model_results <- lapply(
  predictions,
  function(pred) {
    postResample(
      pred = pred,
      obs = test_processed$Yield
    )
  }
)

# Create comparison table
comparison <- bind_rows(
  model_results,
  .id = "Model"
) |>
  arrange(RMSE)

comparison
# A tibble: 5 × 4
  Model           RMSE Rsquared   MAE
  <chr>          <dbl>    <dbl> <dbl>
1 Neural_Network  1.23    0.645 0.955
2 SVM_Linear      1.24    0.617 0.949
3 SVM_Radial      1.29    0.607 0.941
4 KNN             1.37    0.549 0.972
5 MARS            1.42    0.553 1.02 

##Visualize model performance

ggplot(
  comparison,
  aes(
    x = reorder(Model, RMSE),
    y = RMSE,
    fill = Model
  )
) +
  geom_col() +
  coord_flip() +
  labs(
    title = "RMSE Comparison of Five Regression Models",
    x = "Model",
    y = "Test RMSE"
  ) +
  theme_minimal() +
  theme(legend.position = "none")

ggplot(
  comparison,
  aes(
    x = reorder(Model, Rsquared),
    y = Rsquared,
    fill = Model
  )
) +
  geom_col() +
  coord_flip() +
  labs(
    title = "R-squared Comparison of Five Models",
    x = "Model",
    y = "Test R-squared"
  ) +
  theme_minimal() +
  theme(legend.position = "none")

predicted_data <- bind_rows(
  lapply(
    names(predictions),
    function(name) {
      tibble(
        Model = name,
        Actual = test_processed$Yield,
        Predicted = predictions[[name]]
      )
    }
  )
)

ggplot(
  predicted_data,
  aes(x = Actual, y = Predicted)
) +
  geom_point(alpha = 0.6) +
  geom_abline(
    slope = 1,
    intercept = 0,
    color = "red",
    linewidth = 1
  ) +
  facet_wrap(~ Model) +
  labs(
    title = "Actual vs Predicted Manufacturing Yield",
    x = "Actual Yield",
    y = "Predicted Yield"
  ) +
  theme_minimal()

##Compare cross-validation

model_resamples <- resamples(
  list(
    KNN = knnModel,
    MARS = marsModel,
    SVM_Linear = svmLinearModel,
    SVM_Radial = svmRadialModel,
    Neural_Network = nnetModel
  
  )
)

summary(model_resamples)

Call:
summary.resamples(object = model_resamples)

Models: KNN, MARS, SVM_Linear, SVM_Radial, Neural_Network 
Number of resamples: 10 

MAE 
                    Min.   1st Qu.    Median      Mean  3rd Qu.     Max. NA's
KNN            0.7346667 0.9422054 1.0523214 1.0232762 1.122676 1.184714    0
MARS           0.4940149 0.6900834 0.8515230 0.8741773 1.020462 1.296867    0
SVM_Linear     0.6800268 0.9847195 1.1429465 1.8102335 1.266112 8.347196    0
SVM_Radial     0.5597241 0.7363036 0.8353486 0.8910765 0.970552 1.352390    0
Neural_Network 0.8638228 1.1318587 1.3061836 1.3342240 1.585045 1.882866    0

RMSE 
                    Min.   1st Qu.   Median     Mean  3rd Qu.      Max. NA's
KNN            1.0317044 1.1116954 1.308224 1.244040 1.344813  1.423410    0
MARS           0.7240758 0.8792829 1.045028 1.118314 1.230480  1.720611    0
SVM_Linear     0.8261778 1.1847316 1.349527 3.713475 1.647619 25.196369    0
SVM_Radial     0.8033869 0.9383318 1.086007 1.107539 1.197420  1.620343    0
Neural_Network 1.0879501 1.3085580 1.691832 1.648740 1.977086  2.163170    0

Rsquared 
                      Min.   1st Qu.    Median      Mean   3rd Qu.      Max.
KNN            0.325974137 0.4497631 0.4848883 0.5408253 0.6880081 0.8062443
MARS           0.360329627 0.5849356 0.6789879 0.6457536 0.7434353 0.8452116
SVM_Linear     0.006677735 0.3463312 0.4765197 0.4705089 0.6331664 0.8455222
SVM_Radial     0.250318787 0.6004521 0.6730952 0.6378048 0.7608339 0.7971990
Neural_Network 0.046691351 0.1806138 0.3787999 0.3599425 0.5653289 0.6687371
               NA's
KNN               0
MARS              0
SVM_Linear        0
SVM_Radial        0
Neural_Network    0
bwplot(
  model_resamples,
  metric = "RMSE"
)

dotplot(
  model_resamples,
  metric = "RMSE"
)

# Exclude linear SVM from nonlinear model selection
nonlinear_comparison <- comparison |>
  filter(Model != "SVM_Linear") |>
  arrange(RMSE)

nonlinear_comparison
# A tibble: 4 × 4
  Model           RMSE Rsquared   MAE
  <chr>          <dbl>    <dbl> <dbl>
1 Neural_Network  1.23    0.645 0.955
2 SVM_Radial      1.29    0.607 0.941
3 KNN             1.37    0.549 0.972
4 MARS            1.42    0.553 1.02 
# Best nonlinear model
best_nonlinear <- nonlinear_comparison |>
  slice(1)

best_nonlinear
# A tibble: 1 × 4
  Model           RMSE Rsquared   MAE
  <chr>          <dbl>    <dbl> <dbl>
1 Neural_Network  1.23    0.645 0.955
# Best cross-validation RMSE for each nonlinear model
cv_results <- bind_rows(
  lapply(
    list(
      KNN = knnModel,
      MARS = marsModel,
      SVM_Radial = svmRadialModel,
      Neural_Network = nnetModel
      
    ),
    function(model) {
      model$results |>
        slice_min(RMSE, n = 1, with_ties = FALSE) |>
        select(RMSE, Rsquared, MAE)
    }
  ),
  .id = "Model"
) |>
  arrange(RMSE)

cv_results
           Model     RMSE  Rsquared       MAE
1     SVM_Radial 1.107539 0.6378048 0.8910765
2           MARS 1.118314 0.6457536 0.8741773
3            KNN 1.244040 0.5408253 1.0232762
4 Neural_Network 1.648740 0.3599425 1.3342240

(a) Optimal Nonlinear Regression Model

Four nonlinear regression models were evaluated to predict chemical manufacturing yield: KNN, MARS, SVM Radial, and Neural Network. SVM Linear was also included for comparison.

The models were trained using 10-fold cross-validation and evaluated on an independent test dataset.

Based on cross-validation performance, SVM Radial achieved the lowest RMSE of 1.1075, followed closely by MARS with an RMSE of 1.1183. KNN and Neural Network produced higher cross-validation errors.

However, on the independent test dataset, Neural Network achieved the lowest RMSE of 1.2259, with an R-squared of 0.6448 and an MAE of 0.9553.

These results indicate that the best-performing model depends on the evaluation method. SVM Radial performed best during cross-validation, while Neural Network performed best on the test dataset.

Therefore, SVM Radial was selected as the optimal nonlinear regression model based on cross-validation RMSE, with Neural Network providing the strongest test-set performance.

b. Which predictors are most important

library(caret)
library(dplyr)
library(ggplot2)

set.seed(42)

# Training predictors and response
x_train <- train_processed |>
  select(-Yield)

y_train <- train_processed$Yield

# Original predictions
original_pred <- predict(
  svmRadialModel,
  newdata = x_train
)

# Original RMSE
original_rmse <- RMSE(original_pred, y_train)

# Calculate permutation importance
permutation_importance <- function(model, data, response, nsim = 10) {
  
  baseline_pred <- predict(model, newdata = data)
  baseline_rmse <- RMSE(baseline_pred, response)
  
  importance <- sapply(names(data), function(variable) {
    
    rmse_values <- replicate(nsim, {
      
      permuted_data <- data
      
      # Shuffle one predictor
      permuted_data[[variable]] <- sample(
        permuted_data[[variable]]
      )
      
      # Predict with shuffled data
      permuted_pred <- predict(
        model,
        newdata = permuted_data
      )
      
      RMSE(permuted_pred, response) - baseline_rmse
    })
    
    mean(rmse_values)
  })
  
  data.frame(
    Variable = names(importance),
    Importance = as.numeric(importance)
  ) |>
    arrange(desc(Importance))
}

# permutation importance
set.seed(42)

svm_importance <- permutation_importance(
  model = svmRadialModel,
  data = train_processed |> select(-Yield),
  response = train_processed$Yield,
  nsim = 30
)

top10_nonlinear <- svm_importance |>
  slice_head(n = 10)

top10_nonlinear
                 Variable Importance
1  ManufacturingProcess32  0.6264861
2  ManufacturingProcess36  0.3298613
3  ManufacturingProcess09  0.2937248
4  ManufacturingProcess37  0.2843384
5  ManufacturingProcess13  0.2823769
6  ManufacturingProcess28  0.2441497
7  ManufacturingProcess11  0.2411713
8    BiologicalMaterial03  0.2379343
9  ManufacturingProcess38  0.2293577
10 ManufacturingProcess17  0.2237996
top10_nonlinear <- svm_importance |>
  slice_head(n = 10)

top10_nonlinear
                 Variable Importance
1  ManufacturingProcess32  0.6264861
2  ManufacturingProcess36  0.3298613
3  ManufacturingProcess09  0.2937248
4  ManufacturingProcess37  0.2843384
5  ManufacturingProcess13  0.2823769
6  ManufacturingProcess28  0.2441497
7  ManufacturingProcess11  0.2411713
8    BiologicalMaterial03  0.2379343
9  ManufacturingProcess38  0.2293577
10 ManufacturingProcess17  0.2237996
# Biological predictors
biological_vars <- names(
  ChemicalManufacturingProcess
)[2:13]

top10_nonlinear <- top10_nonlinear |>
  mutate(
    Variable_Type = if_else(
      Variable %in% biological_vars,
      "Biological",
      "Process"
    )
  )

top10_nonlinear
                 Variable Importance Variable_Type
1  ManufacturingProcess32  0.6264861       Process
2  ManufacturingProcess36  0.3298613       Process
3  ManufacturingProcess09  0.2937248       Process
4  ManufacturingProcess37  0.2843384       Process
5  ManufacturingProcess13  0.2823769       Process
6  ManufacturingProcess28  0.2441497       Process
7  ManufacturingProcess11  0.2411713       Process
8    BiologicalMaterial03  0.2379343    Biological
9  ManufacturingProcess38  0.2293577       Process
10 ManufacturingProcess17  0.2237996       Process
# Count biological vs process
top10_nonlinear |>
  count(Variable_Type)
  Variable_Type n
1    Biological 1
2       Process 9
ggplot(
  top10_nonlinear,
  aes(
    x = reorder(Variable, Importance),
    y = Importance,
    fill = Variable_Type
  )
) +
  geom_col() +
  coord_flip() +
  labs(
    title = "Top 10 Important Predictors - SVM Radial",
    x = "Predictor",
    y = "Increase in RMSE",
    fill = "Variable Type"
  ) +
  theme_minimal()

# Import Ridge results from Exercise 6.3
top10_linear <- read.csv("ridge_top10.csv")

top10_linear
                Predictor   Overall
1  ManufacturingProcess32 100.00000
2  ManufacturingProcess09  78.88595
3  ManufacturingProcess36  71.51112
4  ManufacturingProcess13  70.37738
5  ManufacturingProcess17  68.25682
6  ManufacturingProcess34  54.47594
7  ManufacturingProcess06  45.69522
8  ManufacturingProcess11  43.90333
9    BiologicalMaterial03  40.71388
10 ManufacturingProcess33  40.21446
# Predictors shared by both models
common_predictors <- intersect(
  top10_nonlinear$Variable,
  top10_linear$Predictor
)

common_predictors
[1] "ManufacturingProcess32" "ManufacturingProcess36" "ManufacturingProcess09"
[4] "ManufacturingProcess13" "ManufacturingProcess11" "BiologicalMaterial03"  
[7] "ManufacturingProcess17"
# Predictors unique to SVM Radial
unique_nonlinear <- setdiff(
  top10_nonlinear$Variable,
  top10_linear$Predictor
)

unique_nonlinear
[1] "ManufacturingProcess37" "ManufacturingProcess28" "ManufacturingProcess38"
# Predictors unique to Ridge
unique_linear <- setdiff(
  top10_linear$Predictor,
  top10_nonlinear$Variable
)

unique_linear
[1] "ManufacturingProcess34" "ManufacturingProcess06" "ManufacturingProcess33"
# Predictors shared by both models
common_predictors <- intersect(
  top10_nonlinear$Variable,
  top10_linear$Predictor
)

common_predictors
[1] "ManufacturingProcess32" "ManufacturingProcess36" "ManufacturingProcess09"
[4] "ManufacturingProcess13" "ManufacturingProcess11" "BiologicalMaterial03"  
[7] "ManufacturingProcess17"
# Predictors unique to SVM Radial
unique_nonlinear <- setdiff(
  top10_nonlinear$Variable,
  top10_linear$Predictor
)

unique_nonlinear
[1] "ManufacturingProcess37" "ManufacturingProcess28" "ManufacturingProcess38"
# Predictors unique to linear model
unique_linear <- setdiff(
  top10_linear$Predictor,
  top10_nonlinear$Variable
)

unique_linear
[1] "ManufacturingProcess34" "ManufacturingProcess06" "ManufacturingProcess33"

(b) Important Predictors

Permutation importance was used to identify the ten most influential predictors in the SVM Radial model.

ManufacturingProcess32 was the most important predictor, with an increase in RMSE of approximately 0.6265 when its values were randomly shuffled. ManufacturingProcess36 and ManufacturingProcess09 followed, with increases in RMSE of approximately 0.3299 and 0.2937, respectively.

Among the ten most important predictors, nine were manufacturing process variables and one was a biological material variable (BiologicalMaterial03).

These results suggest that manufacturing process variables contribute substantially to the SVM Radial model’s predictive performance. However, variable importance does not necessarily indicate a causal relationship with manufacturing yield.

c. Explore the relationships between the top predictors and the response for the predictors that are unique to the optimal nonlinear regression model. Do these plots reveal intuition about the biological or process predictors and their relationship with yield?

# Plot predictors unique to the nonlinear model
plot_data <- train_data |>
  select(Yield, all_of(unique_nonlinear)) |>
  pivot_longer(
    cols = all_of(unique_nonlinear),
    names_to = "Predictor",
    values_to = "Value"
  ) |>
  filter(!is.na(Value), !is.na(Yield))

ggplot(
  plot_data,
  aes(x = Value, y = Yield)
) +
  geom_point(
    alpha = 0.5,
    color = "steelblue"
  ) +
  geom_smooth(
    method = "lm",
    formula = y ~ x + I(x^2),
    se = TRUE,
    color = "red"
  ) +
  facet_wrap(
    ~ Predictor,
    scales = "free_x"
  ) +
  labs(
    title = "Unique  SVM Radial Predictors vs Yield",
    x = "Predictor Value",
    y = "Manufacturing Yield (%)"
  ) +
  theme_minimal()

(c) Relationships Between Unique Nonlinear Predictors and Yield

The relationships between manufacturing yield and three predictors appearing in the SVM Radial model’s top-ten importance list, but not in the Ridge model’s top-ten list, were examined using scatterplots and quadratic regression curves.

ManufacturingProcess28 shows a potentially nonlinear relationship with yield. However, observations are concentrated in two distinct groups, making it difficult to determine the relationship between them. The wide confidence interval indicates substantial uncertainty in the fitted curve.

ManufacturingProcess37 shows a slight decrease in yield as its value increases, followed by a relatively stable relationship. However, the observations are widely scattered, suggesting a weak individual association with yield.

ManufacturingProcess38 shows a slight curved relationship, but most observations are concentrated at values of 2 and 3. Therefore, the evidence for a nonlinear relationship is limited.

Overall, these plots suggest that some manufacturing process predictors may have nonlinear associations with yield. However, the relationships are not sufficiently strong or consistent to establish clear nonlinear effects. The results also suggest that relationships between manufacturing process variables and yield may be more complex than those captured by simple individual predictor plots.