Goal: Enhance spam email prediction model by applying additional techniques

Import Data

library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ ggplot2   3.5.1     ✔ tibble    3.2.1
## ✔ lubridate 1.9.3     ✔ tidyr     1.3.1
## ✔ purrr     1.0.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(correlationfunnel)
## Warning: package 'correlationfunnel' was built under R version 4.4.2
## ══ Using correlationfunnel? ════════════════════════════════════════════════════
## You might also be interested in applied data science training for business.
## </> Learn more at - www.business-science.io </>
library(tidymodels)
## Warning: package 'tidymodels' was built under R version 4.4.2
## ── Attaching packages ────────────────────────────────────── tidymodels 1.2.0 ──
## ✔ broom        1.0.6     ✔ rsample      1.2.1
## ✔ dials        1.3.0     ✔ tune         1.2.1
## ✔ infer        1.0.7     ✔ workflows    1.1.4
## ✔ modeldata    1.4.0     ✔ workflowsets 1.1.0
## ✔ parsnip      1.2.1     ✔ yardstick    1.3.2
## ✔ recipes      1.1.0
## Warning: package 'dials' was built under R version 4.4.2
## Warning: package 'infer' was built under R version 4.4.2
## Warning: package 'modeldata' was built under R version 4.4.2
## Warning: package 'parsnip' was built under R version 4.4.2
## Warning: package 'tune' was built under R version 4.4.2
## Warning: package 'workflows' was built under R version 4.4.2
## Warning: package 'workflowsets' was built under R version 4.4.2
## Warning: package 'yardstick' was built under R version 4.4.2
## ── Conflicts ───────────────────────────────────────── tidymodels_conflicts() ──
## ✖ scales::discard() masks purrr::discard()
## ✖ dplyr::filter()   masks stats::filter()
## ✖ recipes::fixed()  masks stringr::fixed()
## ✖ dplyr::lag()      masks stats::lag()
## ✖ yardstick::spec() masks readr::spec()
## ✖ recipes::step()   masks stats::step()
## • Use tidymodels_prefer() to resolve common conflicts.
library(themis)
## Warning: package 'themis' was built under R version 4.4.3
library(vip)
## Warning: package 'vip' was built under R version 4.4.3
## 
## Attaching package: 'vip'
## 
## The following object is masked from 'package:utils':
## 
##     vi
spam <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/main/data/2023/2023-08-15/spam.csv')
## Rows: 4601 Columns: 7
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (1): yesno
## dbl (6): crl.tot, dollar, bang, money, n000, make
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Clean Data

spam_clean <- spam %>%
  mutate(yesno = factor(yesno, levels = c("y", "n")))

Explore Data

spam_clean %>% count(yesno)
## # A tibble: 2 × 2
##   yesno     n
##   <fct> <int>
## 1 y      1813
## 2 n      2788
ggplot(spam_clean, aes(yesno)) + geom_bar()

ggplot(spam_clean, aes(yesno, crl.tot)) + geom_boxplot()

Correlation Analysis

spam_binarized <- spam_clean %>% binarize()
spam_correlation <- spam_binarized %>% correlate(yesno__y)
spam_correlation_sorted <- spam_correlation %>% arrange(desc(correlation))

correlationfunnel::plot_correlation_funnel(spam_correlation_sorted)

Split Data

set.seed(1234)
spam_split <- initial_split(spam_clean, strata = yesno)
spam_train <- training(spam_split)
spam_test <- testing(spam_split)
spam_cv <- vfold_cv(spam_train, strata = yesno)

Preprocess Data

spam_recipe <- recipe(yesno ~ ., data = spam_train) %>%
    step_YeoJohnson(all_numeric_predictors()) %>%
    step_normalize(all_numeric_predictors()) %>%
    step_dummy(all_nominal_predictors()) %>%
    step_smote(yesno)

Model Specification

spam_xgboost_spec <- boost_tree(trees = tune(), tree_depth = tune()) %>% 
  set_mode("classification") %>% 
  set_engine("xgboost") 

spam_rf_spec <- rand_forest(trees = tune()) %>% 
  set_mode("classification") %>% 
  set_engine("ranger", importance = "impurity")

Workflow

spam_xgboost_workflow <- workflow() %>% add_recipe(spam_recipe) %>% add_model(spam_xgboost_spec)
spam_rf_workflow <- workflow() %>% add_recipe(spam_recipe) %>% add_model(spam_rf_spec)

Hyperparameter Tuning

doParallel::registerDoParallel()

xgboost_grid <- grid_regular(trees(), tree_depth(), levels = 5)
rf_grid <- grid_regular(trees(), levels = 5)

set.seed(43931)
spam_xgboost_tune <- tune_grid(spam_xgboost_workflow, resamples = spam_cv, grid = xgboost_grid, control = control_grid(save_pred = TRUE))
spam_rf_tune <- tune_grid(spam_rf_workflow, resamples = spam_cv, grid = rf_grid, control = control_grid(save_pred = TRUE))

Model Evaluation

xgboost_metrics <- collect_metrics(spam_xgboost_tune)
rf_metrics <- collect_metrics(spam_rf_tune)

xgboost_roc <- collect_predictions(spam_xgboost_tune) %>% group_by(id) %>% roc_curve(yesno, .pred_y) %>% autoplot()
rf_roc <- collect_predictions(spam_rf_tune) %>% group_by(id) %>% roc_curve(yesno, .pred_y) %>% autoplot()

list(xgboost_roc, rf_roc)
## [[1]]

## 
## [[2]]

Final Model Selection and Fitting

spam_xgboost_final <- spam_xgboost_workflow %>% finalize_workflow(select_best(spam_xgboost_tune, metric = "accuracy")) %>% last_fit(spam_split)
## Warning: package 'xgboost' was built under R version 4.4.2
spam_rf_final <- spam_rf_workflow %>% finalize_workflow(select_best(spam_rf_tune, metric = "accuracy")) %>% last_fit(spam_split)
## Warning: package 'ranger' was built under R version 4.4.2
final_metrics <- bind_rows(collect_metrics(spam_xgboost_final), collect_metrics(spam_rf_final))

Variable Importance

spam_xgboost_final %>% workflows::extract_fit_engine() %>% vip()

spam_rf_final %>% workflows::extract_fit_engine() %>% vip()

Conclusion

The initial model used YeoJohnson transformation. Enhancements included: * Normalization of numeric features. * SMOTE to balance the target variable. * Hyperparameter tuning for tree depth and number of trees. * Random Forest model comparison.

The final results showed: * XGBoost achieved an accuracy of 0.858 and ROC AUC of 0.914. * Random Forest achieved an accuracy of 0.880 and ROC AUC of 0.921.

This comparison provides insight into which model performs better for spam detection.