Final Project CART
Syed Abdullah
2024-05-09
R Markdown
This is an R Markdown document. Markdown is a simple formatting syntax for authoring HTML, PDF, and MS Word documents. For more details on using R Markdown see http://rmarkdown.rstudio.com.
When you click the Knit button a document will be generated that includes both content as well as the output of any embedded R code chunks within the document. You can embed an R code chunk like this:
library(mlbench)## Warning: package 'mlbench' was built under R version 4.2.3library(caret)## Warning: package 'caret' was built under R version 4.2.3## Loading required package: ggplot2## Warning: package 'ggplot2' was built under R version 4.2.3## Loading required package: latticelibrary(klaR)## Warning: package 'klaR' was built under R version 4.2.3## Loading required package: MASSlibrary(e1071)## Warning: package 'e1071' was built under R version 4.2.3library(randomForest)## Warning: package 'randomForest' was built under R version 4.2.3## randomForest 4.7-1.1## Type rfNews() to see new features/changes/bug fixes.##
## Attaching package: 'randomForest'## The following object is masked from 'package:ggplot2':
##
## marginproject <- read.csv("C:\\Users\\hussain7108\\Desktop\\churn (1).csv")
project$Churn. <- as.factor(project$Churn.)
project$Int.l.Plan <- as.factor(project$Int.l.Plan)
project$VMail.Plan <- as.factor(project$VMail.Plan)
# Select the specified columns from the project data frame
project <- project[, c("Area.Code", "Day.Mins", "Day.Calls", "Day.Charge",
"Eve.Mins", "Eve.Calls", "Eve.Charge", "Night.Calls",
"Night.Mins", "Night.Charge",
"Intl.Mins", "Intl.Calls", "Intl.Charge",
"Int.l.Plan", "VMail.Plan", "Churn.")]
summary(project)## Area.Code Day.Mins Day.Calls Day.Charge
## Min. :408.0 Min. : 0.0 Min. : 0.0 Min. : 0.00
## 1st Qu.:408.0 1st Qu.:143.7 1st Qu.: 87.0 1st Qu.:24.43
## Median :415.0 Median :179.4 Median :101.0 Median :30.50
## Mean :437.2 Mean :179.8 Mean :100.4 Mean :30.56
## 3rd Qu.:510.0 3rd Qu.:216.4 3rd Qu.:114.0 3rd Qu.:36.79
## Max. :510.0 Max. :350.8 Max. :165.0 Max. :59.64
## Eve.Mins Eve.Calls Eve.Charge Night.Calls
## Min. : 0.0 Min. : 0.0 Min. : 0.00 Min. : 33.0
## 1st Qu.:166.6 1st Qu.: 87.0 1st Qu.:14.16 1st Qu.: 87.0
## Median :201.4 Median :100.0 Median :17.12 Median :100.0
## Mean :201.0 Mean :100.1 Mean :17.08 Mean :100.1
## 3rd Qu.:235.3 3rd Qu.:114.0 3rd Qu.:20.00 3rd Qu.:113.0
## Max. :363.7 Max. :170.0 Max. :30.91 Max. :175.0
## Night.Mins Night.Charge Intl.Mins Intl.Calls
## Min. : 23.2 Min. : 1.040 Min. : 0.00 Min. : 0.000
## 1st Qu.:167.0 1st Qu.: 7.520 1st Qu.: 8.50 1st Qu.: 3.000
## Median :201.2 Median : 9.050 Median :10.30 Median : 4.000
## Mean :200.9 Mean : 9.039 Mean :10.24 Mean : 4.479
## 3rd Qu.:235.3 3rd Qu.:10.590 3rd Qu.:12.10 3rd Qu.: 6.000
## Max. :395.0 Max. :17.770 Max. :20.00 Max. :20.000
## Intl.Charge Int.l.Plan VMail.Plan Churn.
## Min. :0.000 no :3010 no :2411 False.:2850
## 1st Qu.:2.300 yes: 323 yes: 922 True. : 483
## Median :2.780
## Mean :2.765
## 3rd Qu.:3.270
## Max. :5.400str(project)## 'data.frame': 3333 obs. of 16 variables:
## $ Area.Code : int 415 415 415 408 415 510 510 415 408 415 ...
## $ Day.Mins : num 265 162 243 299 167 ...
## $ Day.Calls : int 110 123 114 71 113 98 88 79 97 84 ...
## $ Day.Charge : num 45.1 27.5 41.4 50.9 28.3 ...
## $ Eve.Mins : num 197.4 195.5 121.2 61.9 148.3 ...
## $ Eve.Calls : int 99 103 110 88 122 101 108 94 80 111 ...
## $ Eve.Charge : num 16.78 16.62 10.3 5.26 12.61 ...
## $ Night.Calls : int 91 103 104 89 121 118 118 96 90 97 ...
## $ Night.Mins : num 245 254 163 197 187 ...
## $ Night.Charge: num 11.01 11.45 7.32 8.86 8.41 ...
## $ Intl.Mins : num 10 13.7 12.2 6.6 10.1 6.3 7.5 7.1 8.7 11.2 ...
## $ Intl.Calls : int 3 3 5 7 3 6 7 6 4 5 ...
## $ Intl.Charge : num 2.7 3.7 3.29 1.78 2.73 1.7 2.03 1.92 2.35 3.02 ...
## $ Int.l.Plan : Factor w/ 2 levels "no","yes": 1 1 1 2 2 2 1 2 1 2 ...
## $ VMail.Plan : Factor w/ 2 levels "no","yes": 2 2 1 1 1 1 2 1 1 2 ...
## $ Churn. : Factor w/ 2 levels "False.","True.": 1 1 1 1 1 1 1 1 1 1 ...project[1:3,]## Area.Code Day.Mins Day.Calls Day.Charge Eve.Mins Eve.Calls Eve.Charge
## 1 415 265.1 110 45.07 197.4 99 16.78
## 2 415 161.6 123 27.47 195.5 103 16.62
## 3 415 243.4 114 41.38 121.2 110 10.30
## Night.Calls Night.Mins Night.Charge Intl.Mins Intl.Calls Intl.Charge
## 1 91 244.7 11.01 10.0 3 2.70
## 2 103 254.4 11.45 13.7 3 3.70
## 3 104 162.6 7.32 12.2 5 3.29
## Int.l.Plan VMail.Plan Churn.
## 1 no yes False.
## 2 no yes False.
## 3 no no False.#Split Data into training and testing
trainIndex<-createDataPartition(project$Churn.,p=0.80,list=FALSE)
dataTrain<-project[trainIndex,]
dataTest<-project
suppressWarnings({
#Naive Bayes
fit<-NaiveBayes(Churn.~., data=dataTrain)
predictions<-predict(fit,dataTest[,1:15])
})
confusionMatrix(predictions$class,dataTest$Churn.)## Confusion Matrix and Statistics
##
## Reference
## Prediction False. True.
## False. 2759 293
## True. 91 190
##
## Accuracy : 0.8848
## 95% CI : (0.8735, 0.8954)
## No Information Rate : 0.8551
## P-Value [Acc > NIR] : 2.994e-07
##
## Kappa : 0.4374
##
## Mcnemar's Test P-Value : < 2.2e-16
##
## Sensitivity : 0.9681
## Specificity : 0.3934
## Pos Pred Value : 0.9040
## Neg Pred Value : 0.6762
## Prevalence : 0.8551
## Detection Rate : 0.8278
## Detection Prevalence : 0.9157
## Balanced Accuracy : 0.6807
##
## 'Positive' Class : False.
## suppressWarnings({
trainControl<- trainControl(method="boot", number=100)
fit<-train(Churn.~., data=project,trControl=trainControl,method="nb")
print(fit)
trainControl<- trainControl(method="cv", number=10)
fit<-train(Churn.~., data=project,trControl=trainControl,method="nb")
print(fit)
trainControl<- trainControl(method="repeatedcv", number=10, repeats=3)
fit<-train(Churn.~., data=project,trControl=trainControl,method="nb")
print(fit)
trainControl<- trainControl(method="LOOCV")
fit<-train(Churn.~., data=project,trControl=trainControl,method="nb")
print(fit)
})## Naive Bayes
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Bootstrapped (100 reps)
## Summary of sample sizes: 3333, 3333, 3333, 3333, 3333, 3333, ...
## Resampling results across tuning parameters:
##
## usekernel Accuracy Kappa
## FALSE 0.8610877 0.4119172
## TRUE 0.8720511 0.2666513
##
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
## parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
## = 1.
## Naive Bayes
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold)
## Summary of sample sizes: 3000, 3000, 3000, 3000, 2999, 2999, ...
## Resampling results across tuning parameters:
##
## usekernel Accuracy Kappa
## FALSE 0.8628823 0.4236142
## TRUE 0.8676889 0.2284382
##
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
## parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
## = 1.
## Naive Bayes
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 2999, 3000, ...
## Resampling results across tuning parameters:
##
## usekernel Accuracy Kappa
## FALSE 0.8641774 0.4306549
## TRUE 0.8711862 0.2288507
##
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
## parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
## = 1.
## Naive Bayes
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Leave-One-Out Cross-Validation
## Summary of sample sizes: 3332, 3332, 3332, 3332, 3332, 3332, ...
## Resampling results across tuning parameters:
##
## usekernel Accuracy Kappa
## FALSE 0.8640864 0.4305291
## TRUE 0.8733873 0.2692043
##
## Tuning parameter 'fL' was held constant at a value of 0
## Tuning
## parameter 'adjust' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were fL = 0, usekernel = TRUE and adjust
## = 1.# Set train control
set.seed(7)
ctrl <- trainControl(method = "repeatedcv", number = 10, repeats = 3)
# CART model
set.seed(7)
cart_model <- train(Churn. ~ ., data = project, method = "rpart", trControl = ctrl)
# LDA model
set.seed(7)
lda_model <- train(Churn. ~ ., data = project, method = "lda", trControl = ctrl)
# SVM model
set.seed(7)
svm_model <- train(Churn. ~ ., data = project, method = "svmLinear", trControl = ctrl)
# KNN model
set.seed(7)
knn_model <- train(Churn. ~ ., data =project, method = "knn", trControl = ctrl)
# Random Forest model
set.seed(7)
rf_model <- train(Churn. ~ ., data = project, method = "rf", trControl = ctrl)
# Predictions
cart_pred <- predict(cart_model, newdata = dataTest)
lda_pred <- predict(lda_model, newdata = dataTest)
svm_pred <- predict(svm_model, newdata = dataTest)
knn_pred <- predict(knn_model, newdata = dataTest)
rf_pred <- predict(rf_model, newdata = dataTest)
# Model accuracies
cart_accuracy <- sum(cart_pred == dataTest$Churn.) / nrow(dataTest)
lda_accuracy <- sum(lda_pred == dataTest$Churn.) / nrow(dataTest)
svm_accuracy <- sum(svm_pred == dataTest$Churn.) / nrow(dataTest)
knn_accuracy <- sum(knn_pred == dataTest$Churn.) / nrow(dataTest)
rf_accuracy <- sum(rf_pred == dataTest$Churn.) / nrow(dataTest)
# Print accuracies
cat("CART Accuracy:", cart_accuracy, "\n")## CART Accuracy: 0.880288cat("LDA Accuracy:", lda_accuracy, "\n")## LDA Accuracy: 0.8508851cat("SVM Accuracy:", svm_accuracy, "\n")## SVM Accuracy: 0.8550855cat("KNN Accuracy:", knn_accuracy, "\n")## KNN Accuracy: 0.8811881cat("Random Forest Accuracy:", rf_accuracy, "\n")## Random Forest Accuracy: 1# Calculate accuracies
accuracies <- c( CART = cart_accuracy, LDA = lda_accuracy, SVM = svm_accuracy, KNN = knn_accuracy, RF = rf_accuracy)
# Find the difference in accuracies
accuracy_diff <- diff(accuracies)
# Print the difference in accuracies
print(accuracy_diff)## LDA SVM KNN RF
## -0.02940294 0.00420042 0.02610261 0.11881188# Plot the difference in accuracies
barplot(accuracy_diff, main = "Difference in Model Accuracies", ylab = "Accuracy Difference", col = "skyblue")
print(cart_model)## CART
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 3000, 3000, ...
## Resampling results across tuning parameters:
##
## cp Accuracy Kappa
## 0.05797101 0.8916878 0.4153777
## 0.08488613 0.8683864 0.2750178
## 0.08902692 0.8579855 0.1323372
##
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was cp = 0.05797101.print(lda_model)## Linear Discriminant Analysis
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 3000, 3000, ...
## Resampling results:
##
## Accuracy Kappa
## 0.8487898 0.2134685print(svm_model)## Support Vector Machines with Linear Kernel
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 3000, 3000, ...
## Resampling results:
##
## Accuracy Kappa
## 0.8550871 0
##
## Tuning parameter 'C' was held constant at a value of 1print(knn_model)## k-Nearest Neighbors
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 3000, 3000, ...
## Resampling results across tuning parameters:
##
## k Accuracy Kappa
## 5 0.8685869 0.2871832
## 7 0.8706884 0.2743266
## 9 0.8737879 0.2792865
##
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was k = 9.print(rf_model)## Random Forest
##
## 3333 samples
## 15 predictor
## 2 classes: 'False.', 'True.'
##
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 3 times)
## Summary of sample sizes: 3000, 2999, 3000, 3000, 3000, 3000, ...
## Resampling results across tuning parameters:
##
## mtry Accuracy Kappa
## 2 0.9223943 0.6025762
## 8 0.9267954 0.6382736
## 15 0.9267945 0.6406599
##
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was mtry = 8.results<-resamples(list(CART=cart_model,LDA=lda_model,SVM=svm_model,KNN=knn_model,RF=rf_model))
print(results)##
## Call:
## resamples.default(x = list(CART = cart_model, LDA = lda_model, SVM =
## svm_model, KNN = knn_model, RF = rf_model))
##
## Models: CART, LDA, SVM, KNN, RF
## Number of resamples: 30
## Performance metrics: Accuracy, Kappa
## Time estimates for: everything, final model fit#summarize the results
summary(results)##
## Call:
## summary.resamples(object = results)
##
## Models: CART, LDA, SVM, KNN, RF
## Number of resamples: 30
##
## Accuracy
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## CART 0.8678679 0.8828829 0.8920537 0.8916878 0.9011234 0.9129129 0
## LDA 0.8258258 0.8389527 0.8483483 0.8487898 0.8588589 0.8798799 0
## SVM 0.8532934 0.8532934 0.8558559 0.8550871 0.8558559 0.8558559 0
## KNN 0.8558559 0.8682635 0.8738739 0.8737879 0.8798799 0.8888889 0
## RF 0.9071856 0.9189796 0.9265388 0.9267954 0.9339339 0.9519520 0
##
## Kappa
## Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
## CART 0.20169990 0.3392650 0.4149430 0.4153777 0.4984253 0.5489027 0
## LDA 0.06803706 0.1667786 0.2139315 0.2134685 0.2594521 0.3716388 0
## SVM 0.00000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0
## KNN 0.15105163 0.2306095 0.2754119 0.2792865 0.3219840 0.3985942 0
## RF 0.53000454 0.5995513 0.6300468 0.6382736 0.6816463 0.7783878 0table(project$Churn.)##
## False. True.
## 2850 483prop.table(table(project$Churn.))##
## False. True.
## 0.8550855 0.1449145scales<-list(x=list(relation="free"), y=list(relation="free"))
bwplot(results,scales=scales)
densityplot(results, scales=scales)
dotplot(results, scales=scales)
splom(results)
diffs<-diff(results)
summary(diffs)##
## Call:
## summary.diff.resamples(object = diffs)
##
## p-value adjustment: bonferroni
## Upper diagonal: estimates of the difference
## Lower diagonal: p-value for H0: difference = 0
##
## Accuracy
## CART LDA SVM KNN RF
## CART 0.042898 0.036601 0.017900 -0.035108
## LDA 2.680e-10 -0.006297 -0.024998 -0.078006
## SVM 1.547e-14 0.2238 -0.018701 -0.071708
## KNN 3.931e-07 4.032e-07 7.607e-12 -0.053007
## RF 4.374e-10 < 2.2e-16 < 2.2e-16 < 2.2e-16
##
## Kappa
## CART LDA SVM KNN RF
## CART 0.20191 0.41538 0.13609 -0.22290
## LDA 1.444e-07 0.21347 -0.06582 -0.42481
## SVM < 2.2e-16 2.867e-14 -0.27929 -0.63827
## KNN 2.160e-07 0.009525 < 2.2e-16 -0.35899
## RF 1.798e-09 < 2.2e-16 < 2.2e-16 < 2.2e-16