Confusion Matrix

Author

Aidan Ho

library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.2.1     ✔ readr     2.2.0
✔ forcats   1.0.1     ✔ stringr   1.6.0
✔ ggplot2   4.0.3     ✔ tibble    3.3.1
✔ lubridate 1.9.5     ✔ tidyr     1.3.2
✔ purrr     1.2.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(caret)

Loading required package: lattice

Attaching package: 'caret'

The following object is masked from 'package:purrr':

    lift

library(knitr)

# 1. Load Data
train_raw <- read.csv("C:/Users/aidan/Downloads/insurance-training-data2.csv")
test_raw  <- read.csv("C:/Users/aidan/Downloads/insurance-testing-data2.csv")

# 2. Basic Data Cleaning & Imputation
clean_data <- function(df) {
  df <- df %>%
    mutate(
      INCOME     = as.numeric(gsub("[\\$,]", "", INCOME)),
      HOME_VAL   = as.numeric(gsub("[\\$,]", "", HOME_VAL)),
      BLUEBOOK   = as.numeric(gsub("[\\$,]", "", BLUEBOOK)),
      OLDCLAIM   = as.numeric(gsub("[\\$,]", "", OLDCLAIM)),
      REVOKED    = ifelse(REVOKED == "Yes", 1, 0),
      URBANICITY = ifelse(grepl("Highly Urban", URBANICITY), 1, 0)
    )
  
  num_cols <- c("AGE", "YOJ", "INCOME", "HOME_VAL", "BLUEBOOK", "CAR_AGE")
  for (col in num_cols) {
    if (col %in% colnames(df)) {
      df[[col]][is.na(df[[col]])] <- median(df[[col]], na.rm = TRUE)
    }
  }
  return(df)
}

train <- clean_data(train_raw)
test  <- clean_data(test_raw)

# 3. Fit Logistic Model on Training Data
logit_model <- glm(TARGET_FLAG ~ AGE + KIDSDRIV + MVR_PTS + REVOKED + 
                     TIF + TRAVTIME + URBANICITY + BLUEBOOK + INCOME, 
                   data = train, family = binomial(link = "logit"))

# 4. Internal Validation Split (to generate Confusion Matrix & F1)
set.seed(42)
train_idx <- createDataPartition(train$TARGET_FLAG, p = 0.8, list = FALSE)
val_train <- train[train_idx, ]
val_test  <- train[-train_idx, ]

best_logit <- glm(formula(logit_model), data = val_train, family = binomial)
val_probs  <- predict(best_logit, newdata = val_test, type = "response")
val_preds  <- factor(ifelse(val_probs > 0.5, 1, 0), levels = c(0, 1))
val_actual <- factor(val_test$TARGET_FLAG, levels = c(0, 1))

# 5. Create Confusion Matrix & Report F1 Statistic
cm <- confusionMatrix(val_preds, val_actual, positive = "1")
print(cm)

Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 884 262
         1  64  95
                                          
               Accuracy : 0.7502          
                 95% CI : (0.7258, 0.7735)
    No Information Rate : 0.7264          
    P-Value [Acc > NIR] : 0.02828         
                                          
                  Kappa : 0.2401          
                                          
 Mcnemar's Test P-Value : < 2e-16         
                                          
            Sensitivity : 0.2661          
            Specificity : 0.9325          
         Pos Pred Value : 0.5975          
         Neg Pred Value : 0.7714          
             Prevalence : 0.2736          
         Detection Rate : 0.0728          
   Detection Prevalence : 0.1218          
      Balanced Accuracy : 0.5993          
                                          
       'Positive' Class : 1

f1_score <- cm$byClass["F1"]
cat("\nF1 Score for Best Model:", round(f1_score, 4), "\n")


F1 Score for Best Model: 0.3682

# 6. Predict onto Test Data & Save Results
test$P_TARGET_FLAG <- predict(logit_model, newdata = test, type = "response")
test$TARGET_FLAG   <- ifelse(test$P_TARGET_FLAG > 0.5, 1, 0)

# Save CSV File
predictions_summary <- test %>% select(INDEX, P_TARGET_FLAG, TARGET_FLAG)
write.csv(predictions_summary, "Evaluation_Dataset_Predictions.csv", row.names = FALSE)

# Display Table in Rendered Output
kable(head(predictions_summary, 10), caption = "First 10 Predictions")

First 10 Predictions
INDEX	P_TARGET_FLAG	TARGET_FLAG
5	0.1835118	0
8	0.3084215	0
26	0.4148967	0
40	0.3020215	0
45	0.0917245	0
55	0.4297917	0
61	0.8782225	1
66	0.1269640	0
67	0.4491373	0
71	0.3036976	0

# 1. Convert Confusion Matrix stats into a readable table
metrics_df <- data.frame(
  Metric = c("Accuracy", "Sensitivity (Recall)", "Specificity", "F1 Score", "Balanced Accuracy"),
  Value  = c(
    round(cm$overall["Accuracy"], 4),
    round(cm$byClass["Sensitivity"], 4),
    round(cm$byClass["Specificity"], 4),
    round(cm$byClass["F1"], 4),
    round(cm$byClass["Balanced Accuracy"], 4)
  )
)

# Display Metrics Table
kable(metrics_df, caption = "Model Classification Performance Metrics")

Model Classification Performance Metrics
	Metric	Value
Accuracy	Accuracy	0.7502
Sensitivity	Sensitivity (Recall)	0.2661
Specificity	Specificity	0.9325
F1	F1 Score	0.3682
Balanced Accuracy	Balanced Accuracy	0.5993

# 2. Display Predictions Table
kable(head(predictions_summary, 10), caption = "First 10 Evaluation Predictions")

First 10 Evaluation Predictions
INDEX	P_TARGET_FLAG	TARGET_FLAG
5	0.1835118	0
8	0.3084215	0
26	0.4148967	0
40	0.3020215	0
45	0.0917245	0
55	0.4297917	0
61	0.8782225	1
66	0.1269640	0
67	0.4491373	0
71	0.3036976	0