library(mlbench)
## Warning: package 'mlbench' was built under R version 4.3.3
library(caret)
## Warning: package 'caret' was built under R version 4.3.3
## Loading required package: ggplot2
## Warning: package 'ggplot2' was built under R version 4.3.3
## Loading required package: lattice
data("PimaIndiansDiabetes")
# 5 Training models
# Prepare training scheme
trainControl <-trainControl(method="repeatedcv", number=10, repeats=3)
# 1. CART
set.seed(7)
fit.cart <- train(diabetes~., data=PimaIndiansDiabetes, method="rpart", trControl=trainControl)
# 2. LDA
set.seed(7)
fit.lda <- train(diabetes~., data=PimaIndiansDiabetes, method="lda", trControl=trainControl)
# 3. SVM
set.seed(7)
fit.svm<- train(diabetes~., data=PimaIndiansDiabetes, method="svmRadial", trControl=trainControl)
# 4. KNN
set.seed(7)
fit.knn<- train(diabetes~., data=PimaIndiansDiabetes, method="knn", trControl=trainControl)
# 5. Random Forest
library(randomForest)
## Warning: package 'randomForest' was built under R version 4.3.3
## randomForest 4.7-1.1
## Type rfNews() to see new features/changes/bug fixes.
##
## Attaching package: 'randomForest'
## The following object is masked from 'package:ggplot2':
##
## margin
set.seed(7)
fit.rf<- train(diabetes~., data=PimaIndiansDiabetes, method="rf", trControl=trainControl)
# Collect results
results <- resamples(list(CART=fit.cart, LDA=fit.lda, SVM=fit.svm, KNN=fit.knn, RF=fit.rf))
summary(results)
##
## Call:
## summary.resamples(object = results)
##
## Models: CART, LDA, SVM, KNN, RF
## Number of resamples: 30
##
## Accuracy
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## CART 0.6753247 0.7272727 0.7532468 0.7469697 0.7662338 0.7922078 0
## LDA 0.7142857 0.7508117 0.7662338 0.7791069 0.8000256 0.9078947 0
## SVM 0.7236842 0.7508117 0.7631579 0.7712919 0.7915243 0.8947368 0
## KNN 0.6753247 0.7036056 0.7272727 0.7369503 0.7662338 0.8311688 0
## RF 0.6842105 0.7305195 0.7597403 0.7638528 0.8019481 0.8421053 0
##
## Kappa
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## CART 0.2762566 0.3620724 0.4241878 0.4151867 0.4861107 0.5250000 0
## LDA 0.3011551 0.4192537 0.4662541 0.4862025 0.5308596 0.7812500 0
## SVM 0.3391908 0.3997116 0.4460612 0.4621585 0.5234605 0.7475083 0
## KNN 0.2553191 0.3406000 0.3841761 0.3984995 0.4539789 0.6195363 0
## RF 0.2951613 0.3778304 0.4640696 0.4630809 0.5447483 0.6426332 0
# Box Plot
table(PimaIndiansDiabetes$diabetes)
##
## neg pos
## 500 268
prop.table(table(PimaIndiansDiabetes$diabetes))
##
## neg pos
## 0.6510417 0.3489583
scales<- list(X=list(relation="free"), y=list(relation="free"))
bwplot(results, scales=scales)
# Density Plot
table(PimaIndiansDiabetes$diabetes)
##
## neg pos
## 500 268
prop.table(table(PimaIndiansDiabetes$diabetes))
##
## neg pos
## 0.6510417 0.3489583
scales<- list(X=list(relation="free"), y=list(relation="free"))
densityplot(results, scales=scales)
# Dot Plot
table(PimaIndiansDiabetes$diabetes)
##
## neg pos
## 500 268
prop.table(table(PimaIndiansDiabetes$diabetes))
##
## neg pos
## 0.6510417 0.3489583
scales<- list(X=list(relation="free"), y=list(relation="free"))
dotplot(results, scales=scales)
# Scatter Plot
splom(results)
# Calculate and summarize statistical significance
# difference in model predictions
diffs<- diff(results)
# Summarize p-values for pairwise comparisons
summary(diffs)
##
## Call:
## summary.diff.resamples(object = diffs)
##
## p-value adjustment: bonferroni
## Upper diagonal: estimates of the difference
## Lower diagonal: p-value for H0: difference = 0
##
## Accuracy
## CART LDA SVM KNN RF
## CART -0.032137 -0.024322 0.010019 -0.016883
## LDA 0.0011862 0.007815 0.042157 0.015254
## SVM 0.0116401 0.9156892 0.034342 0.007439
## KNN 1.0000000 6.68e-05 0.0002941 -0.026902
## RF 0.2727542 0.4490617 1.0000000 0.0183793
##
## Kappa
## CART LDA SVM KNN RF
## CART -0.0710158 -0.0469717 0.0166872 -0.0478942
## LDA 0.0008086 0.0240440 0.0877029 0.0231215
## SVM 0.0258079 0.3562734 0.0636589 -0.0009225
## KNN 1.0000000 0.0003858 0.0040823 -0.0645814
## RF 0.0211763 1.0000000 1.0000000 0.0158974