Knn Lab1
2024-10-13
#setup
library(caret)## Warning: package 'caret' was built under R version 4.3.3## Loading required package: ggplot2## Warning: package 'ggplot2' was built under R version 4.3.3## Loading required package: latticebank <- read.csv("UniversalBank.csv")
names(bank)## [1] "ID" "Age" "Experience"
## [4] "Income" "ZIP.Code" "Family"
## [7] "CCAvg" "Education" "Mortgage"
## [10] "Personal.Loan" "Securities.Account" "CD.Account"
## [13] "Online" "CreditCard"head(bank,10)## ID Age Experience Income ZIP.Code Family CCAvg Education Mortgage
## 1 1 25 1 49 91107 4 1.6 1 0
## 2 2 45 19 34 90089 3 1.5 1 0
## 3 3 39 15 11 94720 1 1.0 1 0
## 4 4 35 9 100 94112 1 2.7 2 0
## 5 5 35 8 45 91330 4 1.0 2 0
## 6 6 37 13 29 92121 4 0.4 2 155
## 7 7 53 27 72 91711 2 1.5 2 0
## 8 8 50 24 22 93943 1 0.3 3 0
## 9 9 35 10 81 90089 3 0.6 2 104
## 10 10 34 9 180 93023 1 8.9 3 0
## Personal.Loan Securities.Account CD.Account Online CreditCard
## 1 0 1 0 0 0
## 2 0 1 0 0 0
## 3 0 0 0 0 0
## 4 0 0 0 0 0
## 5 0 0 0 0 1
## 6 0 0 0 1 0
## 7 0 0 0 1 0
## 8 0 0 0 0 1
## 9 0 0 0 1 0
## 10 1 0 0 0 0str(bank)## 'data.frame': 5000 obs. of 14 variables:
## $ ID : int 1 2 3 4 5 6 7 8 9 10 ...
## $ Age : int 25 45 39 35 35 37 53 50 35 34 ...
## $ Experience : int 1 19 15 9 8 13 27 24 10 9 ...
## $ Income : int 49 34 11 100 45 29 72 22 81 180 ...
## $ ZIP.Code : int 91107 90089 94720 94112 91330 92121 91711 93943 90089 93023 ...
## $ Family : int 4 3 1 1 4 4 2 1 3 1 ...
## $ CCAvg : num 1.6 1.5 1 2.7 1 0.4 1.5 0.3 0.6 8.9 ...
## $ Education : int 1 1 1 2 2 2 2 3 2 3 ...
## $ Mortgage : int 0 0 0 0 0 155 0 0 104 0 ...
## $ Personal.Loan : int 0 0 0 0 0 0 0 0 0 1 ...
## $ Securities.Account: int 1 1 0 0 0 0 0 0 0 0 ...
## $ CD.Account : int 0 0 0 0 0 0 0 0 0 0 ...
## $ Online : int 0 0 0 0 0 1 1 0 1 0 ...
## $ CreditCard : int 0 0 0 0 1 0 0 1 0 0 ...# Drop ID and zip code columns.
bank <- bank[ , -c(1, 5)]
names(bank)## [1] "Age" "Experience" "Income"
## [4] "Family" "CCAvg" "Education"
## [7] "Mortgage" "Personal.Loan" "Securities.Account"
## [10] "CD.Account" "Online" "CreditCard"# Reorder variables. Put the response last.
bank <- bank[ , c(1:7, 9:12, 8)]
head(bank)## Age Experience Income Family CCAvg Education Mortgage Securities.Account
## 1 25 1 49 4 1.6 1 0 1
## 2 45 19 34 3 1.5 1 0 1
## 3 39 15 11 1 1.0 1 0 0
## 4 35 9 100 1 2.7 2 0 0
## 5 35 8 45 4 1.0 2 0 0
## 6 37 13 29 4 0.4 2 155 0
## CD.Account Online CreditCard Personal.Loan
## 1 0 0 0 0
## 2 0 0 0 0
## 3 0 0 0 0
## 4 0 0 0 0
## 5 0 0 1 0
## 6 0 1 0 0# Set categorical variables as factor.
bank$Education <- as.factor(bank$Education)
bank$Securities.Account <- as.factor(bank$Securities.Account)
bank$CD.Account <- as.factor(bank$CD.Account)
bank$Online <- as.factor(bank$Online)
bank$CreditCard <- as.factor(bank$CreditCard)
# Rename outcome variable values
bank$Personal.Loan <- factor(bank$Personal.Loan,
levels = c("0", "1"),
labels = c("No", "Yes"))
table(bank$Personal.Loan)##
## No Yes
## 4520 480#Set training and validation sets
set.seed(8)
train_index <- sample(1:nrow(bank), 0.6 * nrow(bank))
valid_index <- setdiff(1:nrow(bank), train_index)
train <- bank[train_index, ]
valid <- bank[valid_index, ]
train_norm <- train
valid_norm <- valid
norm_values <- preProcess(train[, -c(12)],
method = c("center",
"scale"))
train[, -c(12)] <- predict(norm_values,
train[, -c(12)])
nrow(train)## [1] 3000nrow(valid)## [1] 2000#Define new customer
new_cust <- data.frame(Age = 40,
Experience = 10,
Income = 84,
Family = 2,
CCAvg = 2,
Education = 2,
Mortgage = 0,
Securities.Account = 0,
CD.Account = 0,
Online = 1,
CreditCard = 1)
new_cust## Age Experience Income Family CCAvg Education Mortgage Securities.Account
## 1 40 10 84 2 2 2 0 0
## CD.Account Online CreditCard
## 1 0 1 1# Set categorical variables as factor.
new_cust$Education <- as.factor(new_cust$Education)
new_cust$Securities.Account <- as.factor(new_cust$Securities.Account)
new_cust$CD.Account <- as.factor(new_cust$CD.Account)
new_cust$Online <- as.factor(new_cust$Online)
new_cust$CreditCard <- as.factor(new_cust$CreditCard)
new_cust## Age Experience Income Family CCAvg Education Mortgage Securities.Account
## 1 40 10 84 2 2 2 0 0
## CD.Account Online CreditCard
## 1 0 1 1# 5. prepare for kNN. -------------------------------------------------------------
# Normalisation, only for numerical variables
norm_values <- preProcess(train[, -c(6, 8:12)],
method = c("center",
"scale"))# Train k = 5
knn_model_k5 <- caret::knn3(Personal.Loan ~ .,
data = train_norm, k = 5)
knn_model_k5## 5-nearest neighbor model
## Training set outcome distribution:
##
## No Yes
## 2722 278#Predict training set
knn_pred_k5_train <- predict(knn_model_k5,
newdata = train_norm[, -c(12)],
type = "class")
head(knn_pred_k5_train)## [1] No No No No No No
## Levels: No Yes# Evaluate training confusion matrix
confusionMatrix(knn_pred_k5_train, as.factor(train_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 2677 150
## Yes 45 128
##
## Accuracy : 0.935
## 95% CI : (0.9256, 0.9436)
## No Information Rate : 0.9073
## P-Value [Acc > NIR] : 2.496e-08
##
## Kappa : 0.5345
##
## Mcnemar's Test P-Value : 9.506e-14
##
## Sensitivity : 0.46043
## Specificity : 0.98347
## Pos Pred Value : 0.73988
## Neg Pred Value : 0.94694
## Prevalence : 0.09267
## Detection Rate : 0.04267
## Detection Prevalence : 0.05767
## Balanced Accuracy : 0.72195
##
## 'Positive' Class : Yes
## # Train k = 7
knn_model_k7 <- caret::knn3(Personal.Loan ~ .,
data = train_norm, k = 7)
knn_model_k7## 7-nearest neighbor model
## Training set outcome distribution:
##
## No Yes
## 2722 278#Predict training set
knn_pred_k7_train <- predict(knn_model_k7,
newdata = train_norm[, -c(12)],
type = "class")
head(knn_pred_k7_train)## [1] No No No No No No
## Levels: No Yes# Evaluate training confusion matrix
confusionMatrix(knn_pred_k7_train, as.factor(train_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 2673 158
## Yes 49 120
##
## Accuracy : 0.931
## 95% CI : (0.9213, 0.9398)
## No Information Rate : 0.9073
## P-Value [Acc > NIR] : 1.920e-06
##
## Kappa : 0.502
##
## Mcnemar's Test P-Value : 6.072e-14
##
## Sensitivity : 0.43165
## Specificity : 0.98200
## Pos Pred Value : 0.71006
## Neg Pred Value : 0.94419
## Prevalence : 0.09267
## Detection Rate : 0.04000
## Detection Prevalence : 0.05633
## Balanced Accuracy : 0.70683
##
## 'Positive' Class : Yes
## # Train k = 3
knn_model_k3 <- caret::knn3(Personal.Loan ~ .,
data = train_norm, k = 3)
knn_model_k3## 3-nearest neighbor model
## Training set outcome distribution:
##
## No Yes
## 2722 278#Predict training set
knn_pred_k3_train <- predict(knn_model_k3,
newdata = train_norm[, -c(12)],
type = "class")
head(knn_pred_k3_train)## [1] No No No No No No
## Levels: No Yes# Evaluate training confusion matrix (BEST RESULT IN TRAINING)
confusionMatrix(knn_pred_k3_train, as.factor(train_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 2667 109
## Yes 55 169
##
## Accuracy : 0.9453
## 95% CI : (0.9366, 0.9532)
## No Information Rate : 0.9073
## P-Value [Acc > NIR] : 8.322e-15
##
## Kappa : 0.6439
##
## Mcnemar's Test P-Value : 3.494e-05
##
## Sensitivity : 0.60791
## Specificity : 0.97979
## Pos Pred Value : 0.75446
## Neg Pred Value : 0.96073
## Prevalence : 0.09267
## Detection Rate : 0.05633
## Detection Prevalence : 0.07467
## Balanced Accuracy : 0.79385
##
## 'Positive' Class : Yes
## # 7.4 predict validation set
# use k = 5
# Predict training set
knn_pred_k5_valid <- predict(knn_model_k5,
newdata = valid_norm[, -c(12)],
type = "class")
head(knn_pred_k5_valid)## [1] No No No No No No
## Levels: No Yes# Evaluate
confusionMatrix(knn_pred_k5_valid, as.factor(valid_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 1746 138
## Yes 52 64
##
## Accuracy : 0.905
## 95% CI : (0.8913, 0.9175)
## No Information Rate : 0.899
## P-Value [Acc > NIR] : 0.1975
##
## Kappa : 0.355
##
## Mcnemar's Test P-Value : 6.98e-10
##
## Sensitivity : 0.3168
## Specificity : 0.9711
## Pos Pred Value : 0.5517
## Neg Pred Value : 0.9268
## Prevalence : 0.1010
## Detection Rate : 0.0320
## Detection Prevalence : 0.0580
## Balanced Accuracy : 0.6440
##
## 'Positive' Class : Yes
## library(ROSE)## Warning: package 'ROSE' was built under R version 4.3.3## Loaded ROSE 0.0-4ROSE::roc.curve(valid_norm$Personal.Loan,
knn_pred_k5_valid)
## Area under the curve (AUC): 0.644# 7.4 predict validation set ----------------------------------------------
# use k = 3 (BEST RESULT IN VALIDATION)
# Predict training set
knn_pred_k3_valid <- predict(knn_model_k3,
newdata = valid_norm[, -c(12)],
type = "class")
head(knn_pred_k3_valid)## [1] No No No No No No
## Levels: No Yes# Evaluate
confusionMatrix(knn_pred_k3_valid, as.factor(valid_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 1732 123
## Yes 66 79
##
## Accuracy : 0.9055
## 95% CI : (0.8918, 0.918)
## No Information Rate : 0.899
## P-Value [Acc > NIR] : 0.1771
##
## Kappa : 0.4051
##
## Mcnemar's Test P-Value : 4.633e-05
##
## Sensitivity : 0.3911
## Specificity : 0.9633
## Pos Pred Value : 0.5448
## Neg Pred Value : 0.9337
## Prevalence : 0.1010
## Detection Rate : 0.0395
## Detection Prevalence : 0.0725
## Balanced Accuracy : 0.6772
##
## 'Positive' Class : Yes
## ROSE::roc.curve(valid_norm$Personal.Loan,
knn_pred_k3_valid)
## Area under the curve (AUC): 0.677# 7.4 predict validation set ----------------------------------------------
# use k = 7
# Predict training set
knn_pred_k7_valid <- predict(knn_model_k7,
newdata = valid_norm[, -c(12)],
type = "class")
head(knn_pred_k7_valid)## [1] No No No No No No
## Levels: No Yes# Evaluate
confusionMatrix(knn_pred_k7_valid, as.factor(valid_norm[, 12]),
positive = "Yes")## Confusion Matrix and Statistics
##
## Reference
## Prediction No Yes
## No 1753 139
## Yes 45 63
##
## Accuracy : 0.908
## 95% CI : (0.8945, 0.9203)
## No Information Rate : 0.899
## P-Value [Acc > NIR] : 0.0958
##
## Kappa : 0.3615
##
## Mcnemar's Test P-Value : 7.079e-12
##
## Sensitivity : 0.3119
## Specificity : 0.9750
## Pos Pred Value : 0.5833
## Neg Pred Value : 0.9265
## Prevalence : 0.1010
## Detection Rate : 0.0315
## Detection Prevalence : 0.0540
## Balanced Accuracy : 0.6434
##
## 'Positive' Class : Yes
## ROSE::roc.curve(valid_norm$Personal.Loan,
knn_pred_k7_valid)
## Area under the curve (AUC): 0.643The best choice of k in this situation was 5. Overall, it had similar values to 3 and 7 for sensitivity, specificity, and pos pred value, but ultimately it had the highest positive pred value in the validation confusion matrices. The pos pred score tells us how good the model is at predicting the positive values. It also has a high neg pred value, which means that it is good at predicting the negative.
Based on my evaluation, the model is fairly good at predicting the negative with the specificity and neg pred value both being high in the 90s, but it is not the best at predicting the positives with a low sensitivity and relatively low pos pred value.