# comaring ML models 

library(mlbench)
library(caret)
## Loading required package: ggplot2
## Loading required package: lattice
data("PimaIndiansDiabetes")
 

traincontrol<-trainControl(method = "repeatedcv",number = 10, repeats = 3)
set.seed(7)
fit<-train(diabetes~., data=PimaIndiansDiabetes,trControl=traincontrol,method="rf")

#CART
set.seed(7)
fit.cart<-train(diabetes~., data=PimaIndiansDiabetes,trControl=traincontrol,method="rpart")


#LDA
set.seed(7)
fit.lda<-train(diabetes~., data=PimaIndiansDiabetes,trControl=traincontrol,method="lda")

#svm
set.seed(7)
fit.svm<-train(diabetes~., data=PimaIndiansDiabetes,trControl=traincontrol,method="svmRadial")


#KNN
set.seed(7)
fit.KNN<-train(diabetes~., data=PimaIndiansDiabetes,trControl=traincontrol,method="knn")


# to pu thtem all in a table 
results<-resamples(list(CART=fit.cart, LDA=fit.lda, SVM=fit.svm, KNN=fit.KNN, RF= fit))
summary(results)
## 
## Call:
## summary.resamples(object = results)
## 
## Models: CART, LDA, SVM, KNN, RF 
## Number of resamples: 30 
## 
## Accuracy 
##           Min.   1st Qu.    Median      Mean   3rd Qu.      Max. NA's
## CART 0.6753247 0.7272727 0.7532468 0.7469697 0.7662338 0.7922078    0
## LDA  0.7142857 0.7508117 0.7662338 0.7791069 0.8000256 0.9078947    0
## SVM  0.7236842 0.7508117 0.7631579 0.7712919 0.7915243 0.8947368    0
## KNN  0.6753247 0.7036056 0.7272727 0.7369503 0.7662338 0.8311688    0
## RF   0.6842105 0.7305195 0.7662338 0.7638528 0.8019481 0.8421053    0
## 
## Kappa 
##           Min.   1st Qu.    Median      Mean   3rd Qu.      Max. NA's
## CART 0.2762566 0.3620724 0.4241878 0.4151867 0.4861107 0.5250000    0
## LDA  0.3011551 0.4192537 0.4662541 0.4862025 0.5308596 0.7812500    0
## SVM  0.3391908 0.3997116 0.4460612 0.4621585 0.5234605 0.7475083    0
## KNN  0.2553191 0.3406000 0.3841761 0.3984995 0.4539789 0.6195363    0
## RF   0.2951613 0.3682680 0.4685583 0.4620226 0.5447483 0.6426332    0
#box and whisker plot
#used to show how far our predictions are set form the mean

prop.table(table(PimaIndiansDiabetes$diabetes))
## 
##       neg       pos 
## 0.6510417 0.3489583
scales<-list(x=list(relation="free"), y = list(relation="free"))

bwplot(results, scales=scales)

densityplot(results, scales=scales, pch="|")

splom(results)

#diffrence in model prediction 
diffs<-diff(results)
summary(diffs)
## 
## Call:
## summary.diff.resamples(object = diffs)
## 
## p-value adjustment: bonferroni 
## Upper diagonal: estimates of the difference
## Lower diagonal: p-value for H0: difference = 0
## 
## Accuracy 
##      CART      LDA       SVM       KNN       RF       
## CART           -0.032137 -0.024322  0.010019 -0.016883
## LDA  0.0011862            0.007815  0.042157  0.015254
## SVM  0.0116401 0.9156892            0.034342  0.007439
## KNN  1.0000000 6.68e-05  0.0002941           -0.026902
## RF   0.2521949 0.3914281 1.0000000 0.0101387          
## 
## Kappa 
##      CART      LDA        SVM        KNN        RF        
## CART           -0.0710158 -0.0469717  0.0166872 -0.0468359
## LDA  0.0008086             0.0240440  0.0877029  0.0241799
## SVM  0.0258079 0.3562734              0.0636589  0.0001359
## KNN  1.0000000 0.0003858  0.0040823             -0.0635230
## RF   0.0250500 1.0000000  1.0000000  0.0099222

You can also embed plots, for example:

Note that the echo = FALSE parameter was added to the code chunk to prevent printing of the R code that generated the plot.