Hw8
Tora Mullings
2023-11-12
library(mlbench)
library(nnet)
library(kernlab)
library(e1071)
library(earth)
library(caret)7.2
set.seed(200)
trainingData <- mlbench.friedman1(200, sd=1)
trainingData$x <- data.frame(trainingData$x)
featurePlot(trainingData$x, trainingData$y)
testData <- mlbench.friedman1(5000, sd=1)
testData$x <- data.frame(testData$x)knnModel <- train(x=trainingData$x,
y=trainingData$y,
method="knn",
preProc=c("center", "scale"),
tuneLength=10)
knnModel## k-Nearest Neighbors
##
## 200 samples
## 10 predictor
##
## Pre-processing: centered (10), scaled (10)
## Resampling: Bootstrapped (25 reps)
## Summary of sample sizes: 200, 200, 200, 200, 200, 200, ...
## Resampling results across tuning parameters:
##
## k RMSE Rsquared MAE
## 5 3.466085 0.5121775 2.816838
## 7 3.349428 0.5452823 2.727410
## 9 3.264276 0.5785990 2.660026
## 11 3.214216 0.6024244 2.603767
## 13 3.196510 0.6176570 2.591935
## 15 3.184173 0.6305506 2.577482
## 17 3.183130 0.6425367 2.567787
## 19 3.198752 0.6483184 2.592683
## 21 3.188993 0.6611428 2.588787
## 23 3.200458 0.6638353 2.604529
##
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was k = 17.knnPred <- predict(knnModel, newdata=testData$x)
postResample(pred=knnPred, obs=testData$y)## RMSE Rsquared MAE
## 3.2040595 0.6819919 2.5683461SVM model
svmRTuned <- train(trainingData$x, trainingData$y,
method="svmRadial",
preProc=c("center", "scale"),
tuneLength=14,
trControl=trainControl(method="cv"))
svmRTuned## Support Vector Machines with Radial Basis Function Kernel
##
## 200 samples
## 10 predictor
##
## Pre-processing: centered (10), scaled (10)
## Resampling: Cross-Validated (10 fold)
## Summary of sample sizes: 180, 180, 180, 180, 180, 180, ...
## Resampling results across tuning parameters:
##
## C RMSE Rsquared MAE
## 0.25 2.505383 0.8031869 1.999381
## 0.50 2.290725 0.8103140 1.829703
## 1.00 2.105086 0.8302040 1.677851
## 2.00 2.014620 0.8418576 1.598814
## 4.00 1.965196 0.8491165 1.567327
## 8.00 1.927649 0.8538945 1.542267
## 16.00 1.924262 0.8545293 1.539275
## 32.00 1.924262 0.8545293 1.539275
## 64.00 1.924262 0.8545293 1.539275
## 128.00 1.924262 0.8545293 1.539275
## 256.00 1.924262 0.8545293 1.539275
## 512.00 1.924262 0.8545293 1.539275
## 1024.00 1.924262 0.8545293 1.539275
## 2048.00 1.924262 0.8545293 1.539275
##
## Tuning parameter 'sigma' was held constant at a value of 0.06802164
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were sigma = 0.06802164 and C = 16.svmRTuned$finalModel## Support Vector Machine object of class "ksvm"
##
## SV type: eps-svr (regression)
## parameter : epsilon = 0.1 cost C = 16
##
## Gaussian Radial Basis kernel function.
## Hyperparameter : sigma = 0.0680216365076835
##
## Number of Support Vectors : 152
##
## Objective Function Value : -66.0924
## Training error : 0.008551svmPred <- predict(svmRTuned$finalModel, newdata=testData$x)
postResample(pred=svmPred, obs=testData$y)## RMSE Rsquared MAE
## 7.156075 0.654429 6.169075MARS
marsFit <- earth(trainingData$x, trainingData$y)
marsFit## Selected 12 of 18 terms, and 6 of 10 predictors
## Termination condition: Reached nk 21
## Importance: X1, X4, X2, X5, X3, X6, X7-unused, X8-unused, X9-unused, ...
## Number of terms at each degree of interaction: 1 11 (additive model)
## GCV 2.540556 RSS 397.9654 GRSq 0.8968524 RSq 0.9183982summary(marsFit)## Call: earth(x=trainingData$x, y=trainingData$y)
##
## coefficients
## (Intercept) 18.451984
## h(0.621722-X1) -11.074396
## h(0.601063-X2) -10.744225
## h(X3-0.281766) 20.607853
## h(0.447442-X3) 17.880232
## h(X3-0.447442) -23.282007
## h(X3-0.636458) 15.150350
## h(0.734892-X4) -10.027487
## h(X4-0.734892) 9.092045
## h(0.850094-X5) -4.723407
## h(X5-0.850094) 10.832932
## h(X6-0.361791) -1.956821
##
## Selected 12 of 18 terms, and 6 of 10 predictors
## Termination condition: Reached nk 21
## Importance: X1, X4, X2, X5, X3, X6, X7-unused, X8-unused, X9-unused, ...
## Number of terms at each degree of interaction: 1 11 (additive model)
## GCV 2.540556 RSS 397.9654 GRSq 0.8968524 RSq 0.9183982marsGrid <- expand.grid(.degree=1:2, .nprune=2:38)
marsTuned <- train(trainingData$x, trainingData$y,
method="earth",
tuneGrid=marsGrid,
trControl=trainControl(method="cv"))
marsTuned## Multivariate Adaptive Regression Spline
##
## 200 samples
## 10 predictor
##
## No pre-processing
## Resampling: Cross-Validated (10 fold)
## Summary of sample sizes: 180, 180, 180, 180, 180, 180, ...
## Resampling results across tuning parameters:
##
## degree nprune RMSE Rsquared MAE
## 1 2 4.555815 0.2211499 3.7894920
## 1 3 3.926474 0.3967924 3.1774338
## 1 4 2.590871 0.7259766 2.0669237
## 1 5 2.294241 0.7891189 1.8337749
## 1 6 2.199933 0.8070077 1.7339981
## 1 7 1.742343 0.8732965 1.3795558
## 1 8 1.666015 0.8845896 1.2982794
## 1 9 1.642477 0.8883691 1.2818259
## 1 10 1.648030 0.8895147 1.2904522
## 1 11 1.617092 0.8944590 1.2649496
## 1 12 1.588637 0.8989528 1.2362045
## 1 13 1.616912 0.8958467 1.2653165
## 1 14 1.617859 0.8959075 1.2598825
## 1 15 1.626588 0.8945949 1.2766998
## 1 16 1.626588 0.8945949 1.2766998
## 1 17 1.626588 0.8945949 1.2766998
## 1 18 1.626588 0.8945949 1.2766998
## 1 19 1.626588 0.8945949 1.2766998
## 1 20 1.626588 0.8945949 1.2766998
## 1 21 1.626588 0.8945949 1.2766998
## 1 22 1.626588 0.8945949 1.2766998
## 1 23 1.626588 0.8945949 1.2766998
## 1 24 1.626588 0.8945949 1.2766998
## 1 25 1.626588 0.8945949 1.2766998
## 1 26 1.626588 0.8945949 1.2766998
## 1 27 1.626588 0.8945949 1.2766998
## 1 28 1.626588 0.8945949 1.2766998
## 1 29 1.626588 0.8945949 1.2766998
## 1 30 1.626588 0.8945949 1.2766998
## 1 31 1.626588 0.8945949 1.2766998
## 1 32 1.626588 0.8945949 1.2766998
## 1 33 1.626588 0.8945949 1.2766998
## 1 34 1.626588 0.8945949 1.2766998
## 1 35 1.626588 0.8945949 1.2766998
## 1 36 1.626588 0.8945949 1.2766998
## 1 37 1.626588 0.8945949 1.2766998
## 1 38 1.626588 0.8945949 1.2766998
## 2 2 4.555815 0.2211499 3.7894920
## 2 3 3.926474 0.3967924 3.1774338
## 2 4 2.590871 0.7259766 2.0669237
## 2 5 2.294241 0.7891189 1.8337749
## 2 6 2.246301 0.7992542 1.8222167
## 2 7 1.753824 0.8708046 1.3826524
## 2 8 1.633425 0.8903708 1.2489141
## 2 9 1.466904 0.9112713 1.1203186
## 2 10 1.324280 0.9278383 1.0547093
## 2 11 1.292964 0.9324603 1.0267734
## 2 12 1.228052 0.9379272 0.9722978
## 2 13 1.217714 0.9392455 0.9649610
## 2 14 1.209324 0.9396066 0.9701953
## 2 15 1.228813 0.9386482 0.9870788
## 2 16 1.217098 0.9392339 0.9809606
## 2 17 1.233891 0.9378794 0.9877503
## 2 18 1.233891 0.9378794 0.9877503
## 2 19 1.233891 0.9378794 0.9877503
## 2 20 1.233891 0.9378794 0.9877503
## 2 21 1.233891 0.9378794 0.9877503
## 2 22 1.233891 0.9378794 0.9877503
## 2 23 1.233891 0.9378794 0.9877503
## 2 24 1.233891 0.9378794 0.9877503
## 2 25 1.233891 0.9378794 0.9877503
## 2 26 1.233891 0.9378794 0.9877503
## 2 27 1.233891 0.9378794 0.9877503
## 2 28 1.233891 0.9378794 0.9877503
## 2 29 1.233891 0.9378794 0.9877503
## 2 30 1.233891 0.9378794 0.9877503
## 2 31 1.233891 0.9378794 0.9877503
## 2 32 1.233891 0.9378794 0.9877503
## 2 33 1.233891 0.9378794 0.9877503
## 2 34 1.233891 0.9378794 0.9877503
## 2 35 1.233891 0.9378794 0.9877503
## 2 36 1.233891 0.9378794 0.9877503
## 2 37 1.233891 0.9378794 0.9877503
## 2 38 1.233891 0.9378794 0.9877503
##
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were nprune = 14 and degree = 2.marsPred <- predict(marsTuned, newdata=testData$x)
postResample(pred=marsPred, obs=testData$y)## RMSE Rsquared MAE
## 1.1722635 0.9448890 0.9324923The MARS model has the highest Rsquared, at 0.94.
varImp(marsTuned)## earth variable importance
##
## Overall
## X1 100.00
## X4 75.24
## X2 48.74
## X5 15.53
## X3 0.00MARS selects the informative predictors, X1-X5.
7.5
library(AppliedPredictiveModeling)
data("ChemicalManufacturingProcess")
X <- ChemicalManufacturingProcess[2:58]
y <- ChemicalManufacturingProcess[1]
sum(is.na(X))## [1] 106prep <- preProcess(X, method=c("knnImpute"))
prep## Created from 152 samples and 57 variables
##
## Pre-processing:
## - centered (57)
## - ignored (0)
## - 5 nearest neighbor imputation (57)
## - scaled (57)Impute missing values.
X_imputed <- predict(prep, X)
sum(is.na(X_imputed))## [1] 0sample <- sample(c(TRUE, FALSE), nrow(X_imputed), replace=TRUE, prob=c(0.7,0.3))
X_imputed_train <- X_imputed[sample, ]
X_imputed_test <- X_imputed[!sample, ]
y_train <- y[sample, ]
y_test <- y[!sample, ]- Neural network
nnetAvgFit <- nnet(X_imputed_train, y_train,
size=5,
decay=0.01,
repeats=5,
linout=TRUE,
trace=FALSE,
maxit=500,
MaxNWts=5*(ncol(X_imputed_train)+1)+5+1)tooHigh <- findCorrelation(cor(X_imputed_train), cutoff=0.75)
trainXnnet <- X_imputed_train[, -tooHigh]
testXnnet <- X_imputed_test[, -tooHigh]
nnetGrid <- expand.grid(.decay=c(0,0.01,0.1),
.size=c(1:10),
.bag=FALSE)
nnetTune <- train(trainXnnet, y_train,
method="avNNet",
tuneGrid=nnetGrid,
trControl = trainControl(method="cv", number=10),
preProc = c("center", "scale"),
linout=TRUE,
trace=FALSE,
MaxNWts=2 * (ncol(trainXnnet)+1) + 2 +1,
maxit = 500)
nnetPred <- predict(nnetTune, newdata=testXnnet)
postResample(pred=nnetPred, obs=y_test)## RMSE Rsquared MAE
## 1.9434967 0.2432922 1.2256910MARS
marsGrid1 <- expand.grid(.degree=1:2, .nprune=2:38)
marsTuned1 <- train(trainXnnet, y_train,
method="earth",
tuneGrid=marsGrid1,
trControl=trainControl(method="cv"))
marsPred1 <- predict(marsTuned1, newdata=testXnnet)
postResample(pred=marsPred1, obs=y_test)## RMSE Rsquared MAE
## 1.2172127 0.5250973 0.9503113SVM
svmRTuned1 <- train(trainXnnet, y_train,
method="svmRadial",
preProc=c("center", "scale"),
tuneLength=14,
trControl=trainControl(method="cv"))
svmPred1 <- predict(svmRTuned1$finalModel, newdata=testXnnet)
postResample(pred=svmPred1, obs=y_test)## RMSE Rsquared MAE
## 1.1990532 0.5070212 0.9435136KNN
knnModel1 <- train(x=trainXnnet,
y=y_train,
method="knn",
preProc=c("center", "scale"),
tuneLength=10)## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07
## Warning in preProcess.default(thresh = 0.95, k = 5, freqCut = 19, uniqueCut =
## 10, : These variables have zero variances: BiologicalMaterial07knnPred1 <- predict(knnModel1, newdata=testXnnet)
postResample(pred=knnPred1, obs=y_test)## RMSE Rsquared MAE
## 1.3707688 0.3149447 1.0517262This model had “zero variance” warnings for one of the predictors, BiologicalMaterial07. Maybe the Rsquared would have been better without it. The MARS model has the best Rsquared, at 0.545.
caret::varImp(marsTuned1)## earth variable importance
##
## Overall
## ManufacturingProcess36 100.00
## ManufacturingProcess17 51.27
## BiologicalMaterial03 0.00
## ManufacturingProcess28 0.00The ManufacturingProcess predictors are definitely more important than the BiologicalMaterial predictors.
summary(marsTuned1)## Call: earth(x=data.frame[120,36], y=c(42.44,43.57,4...), keepxy=TRUE, degree=1,
## nprune=5)
##
## coefficients
## (Intercept) 41.046398
## h(0.658575-BiologicalMaterial03) -0.758838
## h(ManufacturingProcess17- -1.23678) -0.856594
## h(ManufacturingProcess28-0.763612) 14.037712
## h(0.488487-ManufacturingProcess36) 0.777073
##
## Selected 5 of 22 terms, and 4 of 36 predictors (nprune=5)
## Termination condition: RSq changed by less than 0.001 at 22 terms
## Importance: ManufacturingProcess36, ManufacturingProcess17, ...
## Number of terms at each degree of interaction: 1 4 (additive model)
## GCV 1.613195 RSS 165.6348 GRSq 0.5701742 RSq 0.6260233tops <- c("ManufacturingProcess36","ManufacturingProcess17","ManufacturingProcess28",
"BiologicalMaterial03", "ManufacturingProcess37")
cors <- c(cor(testXnnet$ManufacturingProcess36, marsPred1),
cor(testXnnet$ManufacturingProcess17, marsPred1),
cor(testXnnet$ManufacturingProcess28, marsPred1),
cor(testXnnet$BiologicalMaterial03, marsPred1),
cor(testXnnet$ManufacturingProcess37, marsPred1))
data.frame(predictor=tops, response_correlation=cors)## predictor response_correlation
## 1 ManufacturingProcess36 -0.7687756
## 2 ManufacturingProcess17 -0.4682771
## 3 ManufacturingProcess28 0.3728255
## 4 BiologicalMaterial03 0.6032868
## 5 ManufacturingProcess37 -0.1204466There is only one BiologicalMaterial predictor in the top 5 most important predictors for this MARS model. It has a strong positive correlation with the Yield. Of the four remaining predictors, three of them have negative correlation with the yield.